From 86beb71954950c3a8a1cf3b3f854a143161bb540 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 14:58:31 +0530 Subject: [PATCH 01/15] test(cli-e2e): add a standalone replay server entry point Co-Authored-By: Claude Sonnet 5.5 --- apps/cli-e2e/package.json | 1 + apps/cli-e2e/scripts/replay-serve.ts | 35 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 apps/cli-e2e/scripts/replay-serve.ts diff --git a/apps/cli-e2e/package.json b/apps/cli-e2e/package.json index 28e474eaa3..d5caeef214 100644 --- a/apps/cli-e2e/package.json +++ b/apps/cli-e2e/package.json @@ -7,6 +7,7 @@ "test": "pnpm run test:e2e", "test:e2e": "pnpm exec turbo run supabase#build && pnpm exec turbo run @supabase/cli-e2e#test:e2e:run --only --", "test:e2e:run": "bun --bun vitest run", + "replay:serve": "bun scripts/replay-serve.ts", "record": "RECORD=true pnpm run test:e2e:run", "types:check": "tsc --noEmit" }, diff --git a/apps/cli-e2e/scripts/replay-serve.ts b/apps/cli-e2e/scripts/replay-serve.ts new file mode 100644 index 0000000000..3d65892662 --- /dev/null +++ b/apps/cli-e2e/scripts/replay-serve.ts @@ -0,0 +1,35 @@ +import { startReplayServer } from "../src/server/replay-server.ts"; + +const FIXTURES_DIR = new URL("../fixtures", import.meta.url).pathname; + +function parsePort(argv: readonly string[]): number { + const index = argv.indexOf("--port"); + if (index === -1) return 0; + const port = Number(argv[index + 1]); + if (!Number.isInteger(port) || port < 0 || port > 65535) { + throw new Error(`--port expects an integer between 0 and 65535, got "${argv[index + 1]}"`); + } + return port; +} + +// Recording mode wipes the committed fixtures, so a standalone server never runs in it. +if (process.env["RECORD"] === "true") { + throw new Error("replay-serve does not support RECORD; unset it to serve the fixtures"); +} + +const server = await startReplayServer({ + fixturesDir: FIXTURES_DIR, + port: parsePort(process.argv.slice(2)), +}); + +let stopping = false; +const shutdown = async () => { + if (stopping) return; + stopping = true; + await server.stop(); + process.exit(0); +}; +process.on("SIGTERM", shutdown); +process.on("SIGINT", shutdown); + +console.log(JSON.stringify({ url: server.url })); From a4528b6d3f73a33614e42ac096b48245c6d88bf2 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 14:58:33 +0530 Subject: [PATCH 02/15] chore(cli): add workload manifest, replay control, and API call counts to bench-cli Adds a PR workload manifest, replay-server profile redirection and scenario control, per-run API call counts from traces, and a Markdown summary renderer. Co-Authored-By: Claude Sonnet 5.5 --- apps/cli/bench/workloads.pr.json | 33 ++ .../scripts/__fixtures__/api-call-trace.jsonl | 2 + apps/cli/scripts/bench-cli.ts | 520 ++++++++---------- apps/cli/scripts/bench-cli.unit.test.ts | 167 ++---- apps/cli/scripts/bench-report.ts | 506 +++++++++++++++++ apps/cli/scripts/bench-report.unit.test.ts | 359 ++++++++++++ apps/cli/scripts/bench-summary.ts | 181 ++++++ apps/cli/scripts/bench-summary.unit.test.ts | 83 +++ apps/cli/scripts/bench-workloads.unit.test.ts | 39 ++ .../replay-control.integration.test.ts | 74 +++ apps/cli/scripts/replay-control.ts | 80 +++ apps/cli/scripts/trace-report.ts | 38 +- apps/cli/scripts/trace-report.unit.test.ts | 68 +++ 13 files changed, 1750 insertions(+), 400 deletions(-) create mode 100644 apps/cli/bench/workloads.pr.json create mode 100644 apps/cli/scripts/__fixtures__/api-call-trace.jsonl create mode 100644 apps/cli/scripts/bench-report.ts create mode 100644 apps/cli/scripts/bench-report.unit.test.ts create mode 100644 apps/cli/scripts/bench-summary.ts create mode 100644 apps/cli/scripts/bench-summary.unit.test.ts create mode 100644 apps/cli/scripts/bench-workloads.unit.test.ts create mode 100644 apps/cli/scripts/replay-control.integration.test.ts create mode 100644 apps/cli/scripts/replay-control.ts diff --git a/apps/cli/bench/workloads.pr.json b/apps/cli/bench/workloads.pr.json new file mode 100644 index 0000000000..68d9c733ec --- /dev/null +++ b/apps/cli/bench/workloads.pr.json @@ -0,0 +1,33 @@ +{ + "workloads": [ + { "id": "version", "args": ["--version"], "cwdSetup": "none" }, + { "id": "help", "args": ["--help"], "cwdSetup": "none" }, + { "id": "init", "args": ["init"], "cwdSetup": "none" }, + { "id": "migration-new", "args": ["migration", "new", "bench"], "cwdSetup": "init" }, + { + "id": "db-query", + "args": ["db", "query", "--linked", "SELECT 1"], + "cwdSetup": "linked", + "projectRef": "replayfixtureprojref", + "scenario": "db-query-linked-returns-select-1-result-in-table-format" + }, + { + "id": "gen-types", + "args": ["gen", "types", "--lang", "typescript", "--project-id", "replayfixtureprojref"], + "cwdSetup": "none", + "scenario": "gen-types-generates-typescript-types-from-project" + }, + { + "id": "projects-list", + "args": ["projects", "list"], + "cwdSetup": "none", + "scenario": "projects-list-renders-project-list" + }, + { + "id": "functions-list", + "args": ["functions", "list", "--project-ref", "replayfixtureprojref"], + "cwdSetup": "none", + "scenario": "functions-list-renders-fixture-data-in-output" + } + ] +} diff --git a/apps/cli/scripts/__fixtures__/api-call-trace.jsonl b/apps/cli/scripts/__fixtures__/api-call-trace.jsonl new file mode 100644 index 0000000000..da0943e55b --- /dev/null +++ b/apps/cli/scripts/__fixtures__/api-call-trace.jsonl @@ -0,0 +1,2 @@ +{"resourceSpans":[{"scopeSpans":[{"scope":{"name":"effect","version":"4.0.2"},"spans":[{"traceId":"8ced2df9b0dcb856be8a39ecde5572f1","spanId":"b4b54720ec88d11a","parentSpanId":"894f97211df62fba","name":"GET","kind":3,"startTimeUnixNano":"1793178856745192708","endTimeUnixNano":"1793178856748612583","attributes":[{"key":"http.request.method","value":{"stringValue":"GET"}},{"key":"url.full","value":{"stringValue":"http://127.0.0.1:62824/v1/projects"}},{"key":"url.path","value":{"stringValue":"/v1/projects"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.address","value":{"stringValue":"127.0.0.1"}},{"key":"server.port","value":{"intValue":62824}},{"key":"http.response.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"status":{"code":1},"links":[],"droppedLinksCount":0},{"traceId":"8ced2df9b0dcb856be8a39ecde5572f1","spanId":"894664602ca4b246","name":"cli.run","kind":1,"startTimeUnixNano":"1793178856730765750","endTimeUnixNano":"1793178856751636708","attributes":[{"key":"process.boot_ms","value":{"intValue":100}},{"key":"process.exit_code","value":{"intValue":0}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"status":{"code":1},"links":[],"droppedLinksCount":0}]}]}]} +{"resourceSpans":[{"scopeSpans":[{"scope":{"name":"effect","version":"4.0.2"},"spans":[{"traceId":"ab4e0d434b5b64d740d8c6b20839d5bc","spanId":"ba8b1abcbb3ace23","parentSpanId":"22c8e0027cbedc65","name":"GET","kind":3,"startTimeUnixNano":"1793178857188826250","endTimeUnixNano":"1793178857192480042","attributes":[{"key":"http.request.method","value":{"stringValue":"GET"}},{"key":"url.full","value":{"stringValue":"http://127.0.0.1:62824/v1/projects"}},{"key":"url.path","value":{"stringValue":"/v1/projects"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.address","value":{"stringValue":"127.0.0.1"}},{"key":"server.port","value":{"intValue":62824}},{"key":"http.response.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"status":{"code":1},"links":[],"droppedLinksCount":0},{"traceId":"ab4e0d434b5b64d740d8c6b20839d5bc","spanId":"8330e018c95efc64","name":"cli.run","kind":1,"startTimeUnixNano":"1793178857174426458","endTimeUnixNano":"1793178857195383917","attributes":[{"key":"process.boot_ms","value":{"intValue":118}},{"key":"process.exit_code","value":{"intValue":0}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"status":{"code":1},"links":[],"droppedLinksCount":0}]}]}]} diff --git a/apps/cli/scripts/bench-cli.ts b/apps/cli/scripts/bench-cli.ts index 6e1df06c16..8494fc8e92 100644 --- a/apps/cli/scripts/bench-cli.ts +++ b/apps/cli/scripts/bench-cli.ts @@ -1,32 +1,63 @@ /** * A/B wall-time benchmark for two CLI binaries. Alternates launches between `--base` and * `--branch` so neither build runs consistently warmer, isolates each launch's environment, and - * optionally compares per-span trace totals when `SUPABASE_TRACE_FILE` tracing is available. Run - * as `bun scripts/bench-cli.ts --base --branch [options]`. + * optionally records per-span trace totals and Management API call counts from + * `SUPABASE_TRACE_FILE` tracing. Run as + * `bun scripts/bench-cli.ts --base --branch [options]`. */ import { randomUUID } from "node:crypto"; import { tmpdir } from "node:os"; import process from "node:process"; import { parseArgs } from "node:util"; import { BunPath, BunRuntime, BunServices } from "@effect/platform-bun"; -import { Clock, Console, Effect, FileSystem, Path, Ref, Schema, Stdio } from "effect"; +import { + Clock, + Console, + Effect, + FileSystem, + Layer, + Option, + Path, + Ref, + Schema, + Stdio, +} from "effect"; +import { FetchHttpClient } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; -import { readSpans, type ReportSpan } from "./trace-report.ts"; +import { + analyzeWorkload, + BenchReport, + classifyHttpSpans, + collectFlags, + parseWorkloadManifest, + summarizeSpanRun, + validateWorkloads, + type Build, + type RunSample, + type SpanComparison, + type Workload, + type WorkloadAnalysis, + type WorkloadReport, +} from "./bench-report.ts"; +import { readRequestLogCount, resetReplay } from "./replay-control.ts"; +import { readSpans } from "./trace-report.ts"; const USAGE = - 'usage: bun scripts/bench-cli.ts --base --branch [--runs N] [--warmup N] [--command ""]... [--cwd-setup init] [--trace] [--json] [--out ] [--cpu-prof] [--update-check]\n' + - " --command splits on whitespace; it does not support shell quoting.\n" + + 'usage: bun scripts/bench-cli.ts --base --branch [--runs N] [--warmup N] [--command ""]... [--workloads ] [--api-url ] [--cwd-setup init] [--trace] [--json] [--out ] [--cpu-prof] [--update-check]\n' + + " --command splits on whitespace; it does not support shell quoting. Use --workloads for quoted arguments.\n" + + " --workloads reads a manifest of workloads and cannot be combined with --command or --cwd-setup.\n" + + " --api-url points every launch at a replay server through a profile file, and counts Management API calls; it requires --trace.\n" + " --update-check keeps the GitHub release check, which otherwise adds network time to every launch."; -export type Build = "base" | "branch"; - export interface BenchOptions { readonly base: string; readonly branch: string; readonly runs: number; readonly warmup: number; readonly commands: ReadonlyArray>; + readonly workloadsFile: string | undefined; readonly cwdSetupInit: boolean; + readonly apiUrl: string | undefined; readonly trace: boolean; readonly json: boolean; readonly out: string | undefined; @@ -34,10 +65,11 @@ export interface BenchOptions { readonly updateCheck: boolean; } -const ReportJson = Schema.fromJsonString(Schema.Unknown, { space: 2 }); - const DEFAULT_COMMANDS: ReadonlyArray> = [["--version"], ["--help"]]; +/** Dummy token for launches pointed at a replay server; it never reaches a real API. */ +const BENCH_ACCESS_TOKEN = `sbp_${"0".repeat(40)}`; + /** Whether `--help`/`-h` is passed as a flag rather than as a `--command` value. */ export function wantsHelp(argv: ReadonlyArray): boolean { return argv.some( @@ -45,7 +77,7 @@ export function wantsHelp(argv: ReadonlyArray): boolean { ); } -/** Parses argv into {@link BenchOptions}; throws on missing `--base`/`--branch` or bad numbers. */ +/** Parses argv into {@link BenchOptions}; throws on missing `--base`/`--branch` or bad values. */ export function parseBenchArgs(argv: ReadonlyArray): BenchOptions { // `parseArgs` rejects a separate value that starts with `-`, but CLI commands usually do. const args: Array = []; @@ -67,6 +99,8 @@ export function parseBenchArgs(argv: ReadonlyArray): BenchOptions { runs: { type: "string", default: "10" }, warmup: { type: "string", default: "2" }, command: { type: "string", multiple: true }, + workloads: { type: "string" }, + "api-url": { type: "string" }, "cwd-setup": { type: "string" }, trace: { type: "boolean", default: false }, json: { type: "boolean", default: false }, @@ -90,6 +124,15 @@ export function parseBenchArgs(argv: ReadonlyArray): BenchOptions { if (values["cwd-setup"] !== undefined && values["cwd-setup"] !== "init") { throw new Error('--cwd-setup only supports "init"'); } + const hasCommands = values.command !== undefined && values.command.length > 0; + if (values.workloads !== undefined && (hasCommands || values["cwd-setup"] !== undefined)) { + throw new Error("--workloads cannot be combined with --command or --cwd-setup"); + } + if (values["api-url"] !== undefined) { + if (!URL.canParse(values["api-url"])) throw new Error("--api-url must be a valid URL"); + if (!values.trace) + throw new Error("--api-url requires --trace, which supplies the call counts"); + } /* The host `Path` service (`node:path` itself), read synchronously for this sync parser. */ const hostPath = Effect.runSync( @@ -102,11 +145,12 @@ export function parseBenchArgs(argv: ReadonlyArray): BenchOptions { branch: hostPath.resolve(values.branch), runs, warmup, - commands: - values.command === undefined || values.command.length === 0 - ? DEFAULT_COMMANDS - : values.command.map((raw) => raw.split(/\s+/u).filter((part) => part.length > 0)), + commands: hasCommands + ? values.command!.map((raw) => raw.split(/\s+/u).filter((part) => part.length > 0)) + : DEFAULT_COMMANDS, + workloadsFile: values.workloads, cwdSetupInit: values["cwd-setup"] === "init", + apiUrl: values["api-url"], trace: values.trace ?? false, json: values.json ?? false, out: values.out, @@ -139,169 +183,37 @@ export interface PlannedRun { readonly iteration: number; } -/** Alternates base/branch launches per iteration; the first `warmup` iterations are discarded. */ +/** Launches base and branch once per iteration, swapping which goes first on odd iterations. */ export function buildRunPlan(runs: number, warmup: number): ReadonlyArray { const plan: Array = []; for (let iteration = 0; iteration < runs + warmup; iteration++) { const isWarmup = iteration < warmup; - plan.push({ build: "base", warmup: isWarmup, iteration }); - plan.push({ build: "branch", warmup: isWarmup, iteration }); + const order: ReadonlyArray = + iteration % 2 === 0 ? ["base", "branch"] : ["branch", "base"]; + for (const build of order) plan.push({ build, warmup: isWarmup, iteration }); } return plan; } -const round = (value: number): number => Math.round(value * 100) / 100; - -export function median(values: ReadonlyArray): number { - if (values.length === 0) return 0; - const sorted = [...values].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - return sorted.length % 2 === 0 ? (sorted[mid - 1]! + sorted[mid]!) / 2 : sorted[mid]!; -} - -export function percentile(values: ReadonlyArray, p: number): number { - if (values.length === 0) return 0; - const sorted = [...values].sort((a, b) => a - b); - const rank = Math.min(Math.max(Math.ceil((p / 100) * sorted.length) - 1, 0), sorted.length - 1); - return sorted[rank]!; -} - -export interface TimingSummary { - readonly median: number; - readonly min: number; - readonly max: number; - readonly p90: number; -} - -export function summarizeTimings(values: ReadonlyArray): TimingSummary { - const sorted = [...values].sort((a, b) => a - b); - return { - median: round(median(sorted)), - min: round(sorted[0] ?? 0), - max: round(sorted[sorted.length - 1] ?? 0), - p90: round(percentile(sorted, 90)), - }; -} - -export interface Delta { - readonly ms: number; - readonly pct: number; -} - -/** `branch - base`; `pct` is `0` when `baseMedian` is `0` rather than `Infinity`. */ -export function deltaOf(baseMedian: number, branchMedian: number): Delta { - const ms = branchMedian - baseMedian; - return { ms: round(ms), pct: baseMedian === 0 ? 0 : round((ms / baseMedian) * 100) }; -} - -export interface SpanRunSummary { - readonly totalMs: number; - readonly count: number; -} - -/** Sums duration and counts occurrences of each span name within one run's decoded spans. */ -export function summarizeSpanRun(spans: ReadonlyArray): Record { - const result: Record = {}; - for (const span of spans) { - const entry = result[span.name] ?? { totalMs: 0, count: 0 }; - result[span.name] = { - totalMs: entry.totalMs + (span.endMs - span.startMs), - count: entry.count + 1, - }; - } - return result; -} - -export interface SpanAggregate { - readonly medianMs: number; - readonly medianCount: number; -} - -/** Medians each span name's per-run total duration and count across runs; missing runs count as 0. */ -export function aggregateSpanRuns( - runs: ReadonlyArray>, -): Record { - const names = new Set(); - for (const run of runs) for (const name of Object.keys(run)) names.add(name); - const result: Record = {}; - for (const name of names) { - result[name] = { - medianMs: round(median(runs.map((run) => run[name]?.totalMs ?? 0))), - medianCount: round(median(runs.map((run) => run[name]?.count ?? 0))), - }; - } - return result; -} - -export interface SpanComparison { - readonly name: string; - readonly base: SpanAggregate | undefined; - readonly branch: SpanAggregate | undefined; - readonly deltaMs: number | undefined; - readonly onlyIn: Build | undefined; -} - -function spanSortKey(entry: SpanComparison): number { - return entry.deltaMs !== undefined - ? Math.abs(entry.deltaMs) - : (entry.branch?.medianMs ?? entry.base?.medianMs ?? 0); -} - -/** Pairs base/branch span aggregates by name, sorted by `|delta|` descending; flags one-sided spans. */ -export function compareSpans( - base: Record, - branch: Record, -): ReadonlyArray { - const names = new Set([...Object.keys(base), ...Object.keys(branch)]); - const comparisons = [...names].map((name): SpanComparison => { - const baseEntry = base[name]; - const branchEntry = branch[name]; - return { - name, - base: baseEntry, - branch: branchEntry, - deltaMs: - baseEntry !== undefined && branchEntry !== undefined - ? round(branchEntry.medianMs - baseEntry.medianMs) - : undefined, - onlyIn: baseEntry === undefined ? "branch" : branchEntry === undefined ? "base" : undefined, - }; - }); - return comparisons.sort((a, b) => spanSortKey(b) - spanSortKey(a)); -} - -/** Whether the builds exited differently, which makes their timings measure different work. */ -export function exitCodesDiffer( - base: ReadonlyArray, - branch: ReadonlyArray, -): boolean { - const codes = (values: ReadonlyArray) => - [...new Set(values)].sort((a, b) => a - b).join(","); - return codes(base) !== codes(branch); -} - -interface BuildTimingSummary extends TimingSummary { - readonly exitCodes: ReadonlyArray; -} - -export interface CommandReport { - readonly command: ReadonlyArray; - readonly base: BuildTimingSummary; - readonly branch: BuildTimingSummary; - readonly deltaMs: number; - readonly deltaPct: number; - readonly spans: ReadonlyArray | undefined; - readonly traceEmptyFor: ReadonlyArray; - readonly exitCodesDiffer: boolean; -} - -interface RawSample { - readonly command: ReadonlyArray; - readonly build: Build; - readonly wallMs: number; - readonly exitCode: number; - readonly spans: Record | undefined; -} +/** The workloads for a run: the manifest entries, or one per `--command` with `--cwd-setup`. */ +export const resolveWorkloads = Effect.fn("BenchCli.resolveWorkloads")(function* ( + options: BenchOptions, +) { + const workloads: ReadonlyArray = + options.workloadsFile === undefined + ? options.commands.map((args) => ({ + id: args.join(" ") || "(no args)", + args, + cwdSetup: options.cwdSetupInit ? ("init" as const) : ("none" as const), + })) + : yield* Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return yield* parseWorkloadManifest(yield* fs.readFileString(options.workloadsFile!)); + }); + const problems = validateWorkloads(workloads, { apiUrl: options.apiUrl }); + if (problems.length > 0) return yield* Effect.fail(problems.join("\n")); + return workloads; +}); interface LaunchResult { readonly wallMs: number; @@ -334,25 +246,60 @@ const runLaunch = (params: { } satisfies LaunchResult; }); -const baseLaunchEnv = (supabaseHome: string, updateCheck: boolean): Record => ({ +const PROFILE_FILE = "bench-profile.yaml"; + +/** A profile naming `apiUrl` as the Management API; the profile is the CLI's only API redirect. */ +export function profileYaml(apiUrl: string): string { + return [ + "name: bench", + `api_url: "${apiUrl}"`, + `dashboard_url: "${apiUrl}"`, + "project_host: localhost", + "", + ].join("\n"); +} + +const baseLaunchEnv = ( + options: BenchOptions, + supabaseHome: string, + profilePath: string, +): Record => ({ SUPABASE_HOME: supabaseHome, DO_NOT_TRACK: "1", SUPABASE_NO_KEYRING: "1", - ...(updateCheck ? {} : { SUPABASE_NO_UPDATE_NOTIFIER: "1" }), + ...(options.updateCheck ? {} : { SUPABASE_NO_UPDATE_NOTIFIER: "1" }), + ...(options.apiUrl === undefined + ? {} + : { + SUPABASE_PROFILE: profilePath, + SUPABASE_ACCESS_TOKEN: BENCH_ACCESS_TOKEN, + SUPABASE_TELEMETRY_DISABLED: "1", + // With a pooler-url file present, a set password skips the interactive login-role flow. + SUPABASE_DB_PASSWORD: "bench-placeholder-password", + }), }); -/** A fresh project directory and `SUPABASE_HOME`, initialized when `--cwd-setup init` is set. */ -const prepareLaunch = (options: BenchOptions, binary: string) => +/** + * A fresh project directory and `SUPABASE_HOME`, with the replay profile written and the + * workload's `cwdSetup` applied. + */ +const prepareLaunch = (options: BenchOptions, workload: Workload, binary: string) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const projectDir = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-bench-cwd-" }); const supabaseHome = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-bench-home-" }); - if (options.cwdSetupInit) { + const profilePath = path.join(supabaseHome, PROFILE_FILE); + if (options.apiUrl !== undefined) { + yield* fs.writeFileString(profilePath, profileYaml(options.apiUrl)); + } + const env = baseLaunchEnv(options, supabaseHome, profilePath); + if (workload.cwdSetup !== "none") { const init = yield* runLaunch({ binary, args: ["init"], cwd: projectDir, - env: isolatedEnv(process.env, baseLaunchEnv(supabaseHome, options.updateCheck)), + env: isolatedEnv(process.env, env), }); if (init.exitCode !== 0) { yield* Console.error( @@ -360,43 +307,48 @@ const prepareLaunch = (options: BenchOptions, binary: string) => ); } } - return { projectDir, supabaseHome }; + if (workload.cwdSetup === "linked" && workload.projectRef !== undefined) { + const tempDir = path.join(projectDir, "supabase", ".temp"); + yield* fs.makeDirectory(tempDir, { recursive: true }); + yield* fs.writeFileString(path.join(tempDir, "project-ref"), workload.projectRef); + yield* fs.writeFileString( + path.join(tempDir, "pooler-url"), + `postgresql://postgres.${workload.projectRef}:[YOUR-PASSWORD]@aws-0-eu-central-1.pooler.supabase.com:6543/postgres`, + ); + } + return { projectDir, env }; }); -const runCommandBench = ( - options: BenchOptions, - command: ReadonlyArray, - profiled: Ref.Ref, -) => +const runWorkloadBench = (options: BenchOptions, workload: Workload, profiled: Ref.Ref) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const plan = buildRunPlan(options.runs, options.warmup); - const samples: Record> = { base: [], branch: [] }; - const exitCodes: Record> = { base: [], branch: [] }; - const spanRuns: Record>> = { base: [], branch: [] }; - const raw: Array = []; + const samples: Record> = { base: [], branch: [] }; let cpuProfilePath: string | undefined; + // Resets the replay server after setup, so setup's own requests and a previous launch's + // consumed scenario never reach the launch that follows. + const resetServer = + options.apiUrl === undefined ? Effect.void : resetReplay(options.apiUrl, workload.scenario); + for (const planned of plan) { const binary = planned.build === "base" ? options.base : options.branch; yield* Effect.scoped( Effect.gen(function* () { - const { projectDir, supabaseHome } = yield* prepareLaunch(options, binary); + const prepared = yield* prepareLaunch(options, workload, binary); + yield* resetServer; const result = yield* runLaunch({ binary, - args: command, - cwd: projectDir, - env: isolatedEnv(process.env, baseLaunchEnv(supabaseHome, options.updateCheck)), + args: workload.args, + cwd: prepared.projectDir, + env: isolatedEnv(process.env, prepared.env), }); if (planned.warmup) return; - samples[planned.build].push(result.wallMs); - exitCodes[planned.build].push(result.exitCode); - // Tracing and profiling slow the launch they observe, so they run in an extra, untimed // launch in fresh directories. Both builds get traced launches; only one branch launch // is profiled. @@ -413,81 +365,64 @@ const runCommandBench = ( bunOptions = `--cpu-prof --cpu-prof-dir=${profileDir} --cpu-prof-name=${profileName}`; } } + + let trace: RunSample["trace"]; if (traceFile !== undefined || bunOptions !== undefined) { - const observed = yield* prepareLaunch(options, binary); - yield* runLaunch({ + const observed = yield* prepareLaunch(options, workload, binary); + yield* resetServer; + const traced = yield* runLaunch({ binary, - args: command, + args: workload.args, cwd: observed.projectDir, env: isolatedEnv(process.env, { - ...baseLaunchEnv(observed.supabaseHome, options.updateCheck), + ...observed.env, ...(traceFile === undefined ? {} : { SUPABASE_TRACE_FILE: traceFile }), ...(bunOptions === undefined ? {} : { BUN_OPTIONS: bunOptions }), }), }); + if (traceFile !== undefined) { + const requestLogCount = + options.apiUrl === undefined + ? undefined + : yield* readRequestLogCount(options.apiUrl); + // Trace data is optional; a missing or unreadable file must not discard timings. + const decoded = yield* readSpans(traceFile).pipe(Effect.option); + if (Option.isSome(decoded) && decoded.value.spans.length > 0) { + const { spans, skippedLines } = decoded.value; + const http = + options.apiUrl === undefined + ? undefined + : classifyHttpSpans(spans, options.apiUrl); + trace = { + exitCode: traced.exitCode, + skippedLines, + spans: summarizeSpanRun(spans), + ...(http === undefined || requestLogCount === undefined + ? {} + : { api: { ...http, requestLogCount } }), + }; + } + } } - const spans = - traceFile === undefined - ? undefined - : summarizeSpanRun( - // Trace data is optional; a missing or unreadable file must not discard timings. - (yield* readSpans(traceFile).pipe( - Effect.orElseSucceed(() => ({ spans: [], skippedLines: 0 })), - )).spans, - ); - if (spans !== undefined) spanRuns[planned.build].push(spans); - - raw.push({ - command, - build: planned.build, + samples[planned.build].push({ wallMs: result.wallMs, exitCode: result.exitCode, - spans, + ...(trace === undefined ? {} : { trace }), }); }), ); } - const base = summarizeTimings(samples.base); - const branch = summarizeTimings(samples.branch); - const delta = deltaOf(base.median, branch.median); - - const spans = options.trace - ? compareSpans(aggregateSpanRuns(spanRuns.base), aggregateSpanRuns(spanRuns.branch)) - : undefined; - const traceEmptyFor: ReadonlyArray = options.trace - ? (["base", "branch"] as const).filter( - (build) => - spanRuns[build].length > 0 && - spanRuns[build].every((run) => Object.keys(run).length === 0), - ) - : []; - - const report: CommandReport = { - command, - base: { ...base, exitCodes: exitCodes.base }, - branch: { ...branch, exitCodes: exitCodes.branch }, - deltaMs: delta.ms, - deltaPct: delta.pct, - spans, - traceEmptyFor, - exitCodesDiffer: exitCodesDiffer(exitCodes.base, exitCodes.branch), + const report: WorkloadReport = { + id: workload.id, + args: workload.args, + base: samples.base, + branch: samples.branch, }; - return { report, raw, cpuProfilePath }; + return { report, cpuProfilePath }; }); -interface Report { - readonly base: string; - readonly branch: string; - readonly runs: number; - readonly warmup: number; - readonly trace: boolean; - readonly cwdSetup: "init" | undefined; - readonly commands: ReadonlyArray; - readonly cpuProfilePath: string | undefined; -} - function formatSpanLine(span: SpanComparison): string { if (span.onlyIn !== undefined) { const only = span.onlyIn === "base" ? span.base : span.branch; @@ -500,52 +435,59 @@ function formatSpanLine(span: SpanComparison): string { ); } -function formatCommandReport(commandReport: CommandReport): ReadonlyArray { +function formatWorkload(analysis: WorkloadAnalysis): ReadonlyArray { const sign = (value: number) => (value >= 0 ? "+" : ""); + const line = (label: string, build: WorkloadAnalysis["base"]) => + ` ${label} median ${build.timing.median}ms min ${build.timing.min}ms max ${build.timing.max}ms p90 ${build.timing.p90}ms exits [${build.exitCodes.join(",")}]`; const lines = [ - `command: ${commandReport.command.join(" ") || "(no args)"}`, - ` base: median ${commandReport.base.median}ms min ${commandReport.base.min}ms max ${commandReport.base.max}ms p90 ${commandReport.base.p90}ms exits [${commandReport.base.exitCodes.join(",")}]`, - ` branch: median ${commandReport.branch.median}ms min ${commandReport.branch.min}ms max ${commandReport.branch.max}ms p90 ${commandReport.branch.p90}ms exits [${commandReport.branch.exitCodes.join(",")}]`, - ` delta: ${sign(commandReport.deltaMs)}${commandReport.deltaMs}ms (${sign(commandReport.deltaPct)}${commandReport.deltaPct}%)`, + `command: ${analysis.args.join(" ") || "(no args)"} [${analysis.id}]`, + line("base: ", analysis.base), + line("branch:", analysis.branch), + ` delta: ${sign(analysis.deltaMs)}${analysis.deltaMs}ms (${sign(analysis.deltaPct)}${analysis.deltaPct}%) ${analysis.verdict}`, ]; - if (commandReport.exitCodesDiffer) { - lines.push(" warning: exit codes differ between builds, so the timings are not comparable"); - } - for (const build of ["base", "branch"] as const) { - const codes = commandReport[build].exitCodes; - const failed = codes.filter((code) => code !== 0).length; - if (failed > 0) { - lines.push( - ` warning: ${build} exited non-zero in ${failed}/${codes.length} runs, so its timings measure a failing invocation`, - ); + if (analysis.base.api !== undefined || analysis.branch.api !== undefined) { + lines.push( + ` api calls: base ${analysis.base.api?.medianCalls ?? "n/a"} branch ${analysis.branch.api?.medianCalls ?? "n/a"}`, + ); + for (const change of analysis.endpointChanges) { + lines.push(` ${change.endpoint} base ${change.base} branch ${change.branch}`); } } - if (commandReport.traceEmptyFor.length > 0) { - lines.push(` trace: no spans captured for ${commandReport.traceEmptyFor.join(", ")}`); - } - if (commandReport.spans !== undefined && commandReport.spans.length > 0) { + for (const flag of collectFlags(analysis)) lines.push(` warning: ${flag.detail}`); + if (analysis.spans !== undefined && analysis.spans.length > 0) { lines.push(" spans (delta ms, or one-sided median ms x count):"); - for (const span of commandReport.spans) lines.push(formatSpanLine(span)); + for (const span of analysis.spans) lines.push(formatSpanLine(span)); } lines.push(""); return lines; } -function formatReport(report: Report): string { +function formatReport(report: BenchReport): string { const lines = [ `base: ${report.base}`, `branch: ${report.branch}`, `runs: ${report.runs} warmup: ${report.warmup}` + - (report.cwdSetup === undefined ? "" : ` cwd-setup: ${report.cwdSetup}`), + (report.apiUrl === undefined ? "" : ` api-url: ${report.apiUrl}`), "", ]; - for (const commandReport of report.commands) lines.push(...formatCommandReport(commandReport)); + for (const workload of report.workloads) { + lines.push(...formatWorkload(analyzeWorkload(workload, report.trace))); + } if (report.cpuProfilePath !== undefined) { lines.push(`cpu profile (one untimed branch launch): ${report.cpuProfilePath}`); } return lines.join("\n"); } +const encodeReport = Schema.encodeEffect(Schema.fromJsonString(BenchReport, { space: 2 })); + +const fileSize = (binary: string) => + FileSystem.FileSystem.use((fs) => fs.stat(binary)).pipe( + Effect.map((info) => Number(info.size)), + Effect.option, + Effect.map(Option.getOrUndefined), + ); + const main = Effect.gen(function* () { const stdio = yield* Stdio.Stdio; const argv = yield* stdio.args; @@ -570,42 +512,42 @@ const main = Effect.gen(function* () { ); if (options === undefined) return; + const workloads = yield* resolveWorkloads(options); const profiled = yield* Ref.make(false); - const commands: Array = []; - const samples: Array = []; + const reports: Array = []; let cpuProfilePath: string | undefined; - for (const command of options.commands) { - const result = yield* runCommandBench(options, command, profiled); - commands.push(result.report); - samples.push(...result.raw); + for (const workload of workloads) { + const result = yield* runWorkloadBench(options, workload, profiled); + reports.push(result.report); if (result.cpuProfilePath !== undefined) cpuProfilePath = result.cpuProfilePath; } - const report: Report = { + const baseSizeBytes = yield* fileSize(options.base); + const branchSizeBytes = yield* fileSize(options.branch); + const report: BenchReport = { base: options.base, branch: options.branch, + ...(baseSizeBytes === undefined ? {} : { baseSizeBytes }), + ...(branchSizeBytes === undefined ? {} : { branchSizeBytes }), runs: options.runs, warmup: options.warmup, trace: options.trace, - cwdSetup: options.cwdSetupInit ? "init" : undefined, - commands, - cpuProfilePath, + ...(options.apiUrl === undefined ? {} : { apiUrl: options.apiUrl }), + workloads: reports, + ...(cpuProfilePath === undefined ? {} : { cpuProfilePath }), }; if (options.out !== undefined) { const fs = yield* FileSystem.FileSystem; - yield* fs.writeFileString( - options.out, - yield* Schema.encodeEffect(ReportJson)({ report, samples }), - ); + yield* fs.writeFileString(options.out, yield* encodeReport(report)); } - yield* Console.log( - options.json ? yield* Schema.encodeEffect(ReportJson)(report) : formatReport(report), - ); + yield* Console.log(options.json ? yield* encodeReport(report) : formatReport(report)); }); if (import.meta.main) { - BunRuntime.runMain(main.pipe(Effect.provide(BunServices.layer))); + BunRuntime.runMain( + main.pipe(Effect.provide(Layer.mergeAll(FetchHttpClient.layer, BunServices.layer))), + ); } diff --git a/apps/cli/scripts/bench-cli.unit.test.ts b/apps/cli/scripts/bench-cli.unit.test.ts index d991afdc32..903031d96b 100644 --- a/apps/cli/scripts/bench-cli.unit.test.ts +++ b/apps/cli/scripts/bench-cli.unit.test.ts @@ -1,37 +1,21 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it } from "@effect/vitest"; +import { BunServices } from "@effect/platform-bun"; +import { Effect } from "effect"; import { - aggregateSpanRuns, buildRunPlan, - compareSpans, - deltaOf, - exitCodesDiffer, isolatedEnv, parseBenchArgs, + profileYaml, + resolveWorkloads, wantsHelp, - summarizeSpanRun, - summarizeTimings, } from "./bench-cli.ts"; -import type { SpanRunSummary } from "./bench-cli.ts"; -import type { ReportSpan } from "./trace-report.ts"; -const span = (name: string, startMs: number, endMs: number): ReportSpan => ({ - traceId: "t", - spanId: `${name}-${startMs}`, - parentSpanId: undefined, - name, - startMs, - endMs, - failed: false, - errorType: undefined, -}); +const required = ["--base", "/bin/a", "--branch", "/bin/b"]; describe("parseBenchArgs", () => { it("accepts commands that start with a dash and keeps update checks off by default", () => { const options = parseBenchArgs([ - "--base", - "/bin/a", - "--branch", - "/bin/b", + ...required, "--command", "--version", "--command", @@ -41,115 +25,77 @@ describe("parseBenchArgs", () => { expect(options.commands).toEqual([["--version"], ["db", "diff", "--local"]]); expect(options.updateCheck).toBe(false); }); + + it("rejects a manifest combined with --command or --cwd-setup", () => { + expect(() => + parseBenchArgs([...required, "--workloads", "w.json", "--command", "init"]), + ).toThrow("--workloads cannot be combined"); + expect(() => + parseBenchArgs([...required, "--workloads", "w.json", "--cwd-setup", "init"]), + ).toThrow("--workloads cannot be combined"); + }); + + it("requires --trace with --api-url because call counts come from the trace", () => { + expect(() => parseBenchArgs([...required, "--api-url", "http://127.0.0.1:1"])).toThrow( + "--api-url requires --trace", + ); + expect(parseBenchArgs([...required, "--api-url", "http://127.0.0.1:1", "--trace"]).apiUrl).toBe( + "http://127.0.0.1:1", + ); + }); +}); + +describe("resolveWorkloads", () => { + it.effect.each([ + { cwdSetup: ["--cwd-setup", "init"], expected: "init" }, + { cwdSetup: [], expected: "none" }, + ])("turns --command entries into workloads with cwdSetup $expected", ({ cwdSetup, expected }) => + Effect.gen(function* () { + const options = parseBenchArgs([ + ...required, + "--command", + "migration new bench", + ...cwdSetup, + ]); + + const workloads = yield* resolveWorkloads(options); + + expect(workloads).toEqual([ + { id: "migration new bench", args: ["migration", "new", "bench"], cwdSetup: expected }, + ]); + }).pipe(Effect.provide(BunServices.layer)), + ); }); describe("wantsHelp", () => { it("treats --help as a flag only when it is not a --command value", () => { expect(wantsHelp(["--help"])).toBe(true); - expect(wantsHelp(["--base", "/a", "--branch", "/b", "--command", "--help"])).toBe(false); - }); -}); - -describe("summarizeTimings", () => { - it("computes median, min, max, and p90", () => { - expect(summarizeTimings([10, 20, 30, 40, 50])).toEqual({ - median: 30, - min: 10, - max: 50, - p90: 50, - }); + expect(wantsHelp([...required, "--command", "--help"])).toBe(false); }); }); describe("buildRunPlan", () => { - it("alternates base and branch and discards the first warmup iterations", () => { + it("swaps the launch order every other iteration and discards the first warmup iterations", () => { const plan = buildRunPlan(2, 1); expect(plan.map((run) => [run.build, run.warmup])).toEqual([ ["base", true], ["branch", true], - ["base", false], ["branch", false], ["base", false], + ["base", false], ["branch", false], ]); }); }); -describe("exitCodesDiffer", () => { - it("flags builds that exit differently but not shared failures", () => { - expect(exitCodesDiffer([1, 1], [1, 1])).toBe(false); - expect(exitCodesDiffer([0, 0], [1, 0])).toBe(true); - }); -}); - -describe("deltaOf", () => { - it("reports branch minus base in ms and percent", () => { - expect(deltaOf(100, 110)).toEqual({ ms: 10, pct: 10 }); - }); +describe("profileYaml", () => { + it("points the API and dashboard at the replay server and keeps project hosts local", () => { + const yaml = profileYaml("http://127.0.0.1:9"); - it("reports a zero percent delta instead of Infinity when base is zero", () => { - expect(deltaOf(0, 5)).toEqual({ ms: 5, pct: 0 }); - }); -}); - -describe("span aggregation", () => { - it("sums duration and counts per span name within one run", () => { - const run = [span("Config.load", 0, 10), span("Db.query", 10, 20), span("Db.query", 20, 50)]; - - expect(summarizeSpanRun(run)).toEqual({ - "Config.load": { totalMs: 10, count: 1 }, - "Db.query": { totalMs: 40, count: 2 }, - }); - }); - - it("medians per-span totals and counts across runs, treating an absent span as zero", () => { - const runs: ReadonlyArray> = [ - { "Db.query": { totalMs: 40, count: 2 } }, - { "Db.query": { totalMs: 60, count: 2 } }, - {}, - ]; - - expect(aggregateSpanRuns(runs)).toEqual({ - "Db.query": { medianMs: 40, medianCount: 2 }, - }); - }); -}); - -describe("compareSpans", () => { - it("flags spans found in only one build and sorts by absolute delta", () => { - const base = { - "Config.load": { medianMs: 10, medianCount: 1 }, - "Credentials.load": { medianMs: 5, medianCount: 1 }, - }; - const branch = { - "Config.load": { medianMs: 40, medianCount: 1 }, - "Otlp.flush": { medianMs: 3, medianCount: 1 }, - }; - - expect(compareSpans(base, branch)).toEqual([ - { - name: "Config.load", - base: { medianMs: 10, medianCount: 1 }, - branch: { medianMs: 40, medianCount: 1 }, - deltaMs: 30, - onlyIn: undefined, - }, - { - name: "Credentials.load", - base: { medianMs: 5, medianCount: 1 }, - branch: undefined, - deltaMs: undefined, - onlyIn: "base", - }, - { - name: "Otlp.flush", - base: undefined, - branch: { medianMs: 3, medianCount: 1 }, - deltaMs: undefined, - onlyIn: "branch", - }, - ]); + expect(yaml).toContain('api_url: "http://127.0.0.1:9"'); + expect(yaml).toContain('dashboard_url: "http://127.0.0.1:9"'); + expect(yaml).toContain("project_host: localhost"); }); }); @@ -159,6 +105,7 @@ describe("isolatedEnv", () => { HOME: "/home/person", PATH: "/usr/bin", SUPABASE_ACCESS_TOKEN: "secret", + SUPABASE_API_URL: "https://elsewhere", OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector", TRACEPARENT: "00-...-01", }; diff --git a/apps/cli/scripts/bench-report.ts b/apps/cli/scripts/bench-report.ts new file mode 100644 index 0000000000..ad99ab2ecd --- /dev/null +++ b/apps/cli/scripts/bench-report.ts @@ -0,0 +1,506 @@ +/** + * Report schemas and the pure analysis shared by `bench-cli.ts` (which records raw per-run data) + * and `bench-summary.ts` (which derives flags and verdicts), so thresholds can change without + * re-running a benchmark. + */ +import { Effect, Schema } from "effect"; +import type { ReportSpan } from "./trace-report.ts"; + +export type Build = "base" | "branch"; + +const round = (value: number): number => Math.round(value * 100) / 100; + +export function median(values: ReadonlyArray): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1]! + sorted[mid]!) / 2 : sorted[mid]!; +} + +export function percentile(values: ReadonlyArray, p: number): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const rank = Math.min(Math.max(Math.ceil((p / 100) * sorted.length) - 1, 0), sorted.length - 1); + return sorted[rank]!; +} + +export interface TimingSummary { + readonly median: number; + readonly min: number; + readonly max: number; + readonly p90: number; +} + +export function summarizeTimings(values: ReadonlyArray): TimingSummary { + const sorted = [...values].sort((a, b) => a - b); + return { + median: round(median(sorted)), + min: round(sorted[0] ?? 0), + max: round(sorted[sorted.length - 1] ?? 0), + p90: round(percentile(sorted, 90)), + }; +} + +export interface Delta { + readonly ms: number; + readonly pct: number; +} + +/** `branch - base`; `pct` is `0` when `baseMedian` is `0` rather than `Infinity`. */ +export function deltaOf(baseMedian: number, branchMedian: number): Delta { + const ms = branchMedian - baseMedian; + return { ms: round(ms), pct: baseMedian === 0 ? 0 : round((ms / baseMedian) * 100) }; +} + +const SpanRunSummary = Schema.Struct({ totalMs: Schema.Finite, count: Schema.Finite }); +export type SpanRunSummary = typeof SpanRunSummary.Type; + +/** + * Sums duration and counts occurrences of each span name within one run's decoded spans. HTTP + * client spans are left out: the API-call section reports them per endpoint. + */ +export function summarizeSpanRun(spans: ReadonlyArray): Record { + const result: Record = {}; + for (const span of spans) { + if (span.http !== undefined) continue; + const entry = result[span.name] ?? { totalMs: 0, count: 0 }; + result[span.name] = { + totalMs: entry.totalMs + (span.endMs - span.startMs), + count: entry.count + 1, + }; + } + return result; +} + +export interface SpanAggregate { + readonly medianMs: number; + readonly medianCount: number; +} + +/** Medians each span name's per-run total duration and count across runs; missing runs count as 0. */ +export function aggregateSpanRuns( + runs: ReadonlyArray>, +): Record { + const names = new Set(); + for (const run of runs) for (const name of Object.keys(run)) names.add(name); + const result: Record = {}; + for (const name of names) { + result[name] = { + medianMs: round(median(runs.map((run) => run[name]?.totalMs ?? 0))), + medianCount: round(median(runs.map((run) => run[name]?.count ?? 0))), + }; + } + return result; +} + +export interface SpanComparison { + readonly name: string; + readonly base: SpanAggregate | undefined; + readonly branch: SpanAggregate | undefined; + readonly deltaMs: number | undefined; + readonly onlyIn: Build | undefined; +} + +function spanSortKey(entry: SpanComparison): number { + return entry.deltaMs !== undefined + ? Math.abs(entry.deltaMs) + : (entry.branch?.medianMs ?? entry.base?.medianMs ?? 0); +} + +/** Pairs base/branch span aggregates by name, sorted by `|delta|` descending; flags one-sided spans. */ +export function compareSpans( + base: Record, + branch: Record, +): ReadonlyArray { + const names = new Set([...Object.keys(base), ...Object.keys(branch)]); + const comparisons = [...names].map((name): SpanComparison => { + const baseEntry = base[name]; + const branchEntry = branch[name]; + return { + name, + base: baseEntry, + branch: branchEntry, + deltaMs: + baseEntry !== undefined && branchEntry !== undefined + ? round(branchEntry.medianMs - baseEntry.medianMs) + : undefined, + onlyIn: baseEntry === undefined ? "branch" : branchEntry === undefined ? "base" : undefined, + }; + }); + return comparisons.sort((a, b) => spanSortKey(b) - spanSortKey(a)); +} + +/** Whether the builds exited differently, which makes their timings measure different work. */ +export function exitCodesDiffer( + base: ReadonlyArray, + branch: ReadonlyArray, +): boolean { + const codes = (values: ReadonlyArray) => + [...new Set(values)].sort((a, b) => a - b).join(","); + return codes(base) !== codes(branch); +} + +/* Workload manifest */ + +const Workload = Schema.Struct({ + id: Schema.NonEmptyString, + args: Schema.Array(Schema.String), + cwdSetup: Schema.Literals(["none", "init", "linked"]), + scenario: Schema.optionalKey(Schema.String), + projectRef: Schema.optionalKey(Schema.String), +}); +export type Workload = typeof Workload.Type; + +const WorkloadManifest = Schema.Struct({ workloads: Schema.NonEmptyArray(Workload) }); + +const decodeManifest = Schema.decodeUnknownEffect(Schema.fromJsonString(WorkloadManifest)); + +/** Rules the schema cannot express; returns one message per violation. */ +export function validateWorkloads( + workloads: ReadonlyArray, + context: { readonly apiUrl: string | undefined }, +): ReadonlyArray { + const problems: Array = []; + const seen = new Set(); + for (const workload of workloads) { + if (seen.has(workload.id)) problems.push(`duplicate workload id "${workload.id}"`); + seen.add(workload.id); + if (workload.cwdSetup === "linked" && workload.projectRef === undefined) { + problems.push(`workload "${workload.id}" uses cwdSetup "linked" and needs a projectRef`); + } + if (workload.scenario !== undefined && context.apiUrl === undefined) { + problems.push(`workload "${workload.id}" names a scenario and needs --api-url`); + } + } + return problems; +} + +/** Decodes the text of a workload manifest file. */ +export const parseWorkloadManifest = (text: string) => + decodeManifest(text).pipe(Effect.map((manifest) => manifest.workloads)); + +/* HTTP call classification */ + +export interface HttpCallSummary { + readonly apiCalls: number; + readonly externalCalls: number; + readonly errorStatuses: number; + readonly endpoints: Record; +} + +/** + * Splits a run's HTTP client spans into calls to `apiUrl` (same hostname and port) and calls to + * any other host. `errorStatuses` counts API calls that got a 4xx or 5xx response. + */ +export function classifyHttpSpans( + spans: ReadonlyArray, + apiUrl: string, +): HttpCallSummary { + const url = new URL(apiUrl); + const apiPort = url.port === "" ? (url.protocol === "https:" ? 443 : 80) : Number(url.port); + let apiCalls = 0; + let externalCalls = 0; + let errorStatuses = 0; + const endpoints: Record = {}; + for (const span of spans) { + const http = span.http; + if (http === undefined) continue; + if (http.address !== url.hostname || (http.port !== undefined && http.port !== apiPort)) { + externalCalls += 1; + continue; + } + apiCalls += 1; + if (http.status !== undefined && http.status >= 400) errorStatuses += 1; + const endpoint = `${http.method} ${http.path ?? "(unknown path)"}`; + endpoints[endpoint] = (endpoints[endpoint] ?? 0) + 1; + } + return { apiCalls, externalCalls, errorStatuses, endpoints }; +} + +/* Report (raw, as written by `bench-cli.ts --out`) */ + +const ApiSample = Schema.Struct({ + apiCalls: Schema.Finite, + externalCalls: Schema.Finite, + errorStatuses: Schema.Finite, + endpoints: Schema.Record(Schema.String, Schema.Finite), + requestLogCount: Schema.Finite, +}); + +const TraceSample = Schema.Struct({ + exitCode: Schema.Finite, + skippedLines: Schema.Finite, + spans: Schema.Record(Schema.String, SpanRunSummary), + api: Schema.optionalKey(ApiSample), +}); + +const RunSample = Schema.Struct({ + wallMs: Schema.Finite, + exitCode: Schema.Finite, + trace: Schema.optionalKey(TraceSample), +}); +export type RunSample = typeof RunSample.Type; + +const WorkloadReport = Schema.Struct({ + id: Schema.String, + args: Schema.Array(Schema.String), + base: Schema.Array(RunSample), + branch: Schema.Array(RunSample), +}); +export type WorkloadReport = typeof WorkloadReport.Type; + +export const BenchReport = Schema.Struct({ + base: Schema.String, + branch: Schema.String, + baseSizeBytes: Schema.optionalKey(Schema.Finite), + branchSizeBytes: Schema.optionalKey(Schema.Finite), + runs: Schema.Finite, + warmup: Schema.Finite, + trace: Schema.Boolean, + apiUrl: Schema.optionalKey(Schema.String), + workloads: Schema.Array(WorkloadReport), + cpuProfilePath: Schema.optionalKey(Schema.String), +}); +export type BenchReport = typeof BenchReport.Type; + +export const decodeBenchReport = Schema.decodeUnknownEffect(Schema.fromJsonString(BenchReport)); + +/* Analysis */ + +export const VERDICT_MIN_PCT = 5; +export const VERDICT_MIN_MS = 5; +export const VERDICT_MIN_SAMPLES = 5; +export const PHASE_ROWS = 3; + +export type TimingVerdict = "slower" | "faster" | "within noise" | "not comparable"; + +/** + * "slower" or "faster" only when the branch median is outside the base min-max range and the + * change is at least {@link VERDICT_MIN_PCT} percent and {@link VERDICT_MIN_MS} ms. + */ +export function timingVerdict( + base: TimingSummary, + branch: TimingSummary, + context: { readonly comparable: boolean; readonly samples: number }, +): TimingVerdict { + if (!context.comparable || context.samples < VERDICT_MIN_SAMPLES) return "not comparable"; + const delta = deltaOf(base.median, branch.median); + if (Math.abs(delta.pct) < VERDICT_MIN_PCT || Math.abs(delta.ms) < VERDICT_MIN_MS) { + return "within noise"; + } + if (branch.median > base.max) return "slower"; + if (branch.median < base.min) return "faster"; + return "within noise"; +} + +export interface ApiSummary { + readonly medianCalls: number; + readonly runCalls: ReadonlyArray; + readonly nonDeterministic: boolean; + /** Median per-run count for each `METHOD path`; a run without the endpoint counts as 0. */ + readonly endpoints: Record; + readonly externalCalls: number; + /** Runs whose span-derived API call count differs from the replay server's request log. */ + readonly logMismatchRuns: number; + readonly errorStatuses: number; +} + +export interface BuildAnalysis { + readonly timing: TimingSummary; + readonly samples: number; + readonly exitCodes: ReadonlyArray; + readonly tracedExitCodes: ReadonlyArray; + readonly spans: Record | undefined; + readonly api: ApiSummary | undefined; + /** Measured runs without a usable trace when tracing was requested. */ + readonly traceMissingRuns: number; + readonly skippedTraceLines: number; +} + +function summarizeApi(samples: ReadonlyArray): ApiSummary | undefined { + const apis = samples.flatMap((sample) => + sample.trace?.api === undefined ? [] : [sample.trace.api], + ); + if (apis.length === 0) return undefined; + const runCalls = apis.map((api) => api.apiCalls); + const keys = new Set(apis.flatMap((api) => Object.keys(api.endpoints))); + const endpoints: Record = {}; + for (const key of keys) endpoints[key] = median(apis.map((api) => api.endpoints[key] ?? 0)); + const shape = (api: (typeof apis)[number]) => + JSON.stringify(Object.entries(api.endpoints).sort(([a], [b]) => a.localeCompare(b))); + return { + medianCalls: median(runCalls), + runCalls, + nonDeterministic: new Set(apis.map(shape)).size > 1, + endpoints, + externalCalls: Math.max(...apis.map((api) => api.externalCalls)), + logMismatchRuns: apis.filter((api) => api.apiCalls !== api.requestLogCount).length, + errorStatuses: Math.max(...apis.map((api) => api.errorStatuses)), + }; +} + +function summarizeBuild(samples: ReadonlyArray, traced: boolean): BuildAnalysis { + const withTrace = samples.flatMap((sample) => (sample.trace === undefined ? [] : [sample.trace])); + return { + timing: summarizeTimings(samples.map((sample) => sample.wallMs)), + samples: samples.length, + exitCodes: samples.map((sample) => sample.exitCode), + tracedExitCodes: withTrace.map((trace) => trace.exitCode), + spans: traced ? aggregateSpanRuns(withTrace.map((trace) => trace.spans)) : undefined, + api: summarizeApi(samples), + traceMissingRuns: traced ? samples.length - withTrace.length : 0, + skippedTraceLines: withTrace.reduce((sum, trace) => sum + trace.skippedLines, 0), + }; +} + +export interface EndpointChange { + readonly endpoint: string; + readonly base: number; + readonly branch: number; +} + +export interface WorkloadAnalysis { + readonly id: string; + readonly args: ReadonlyArray; + readonly base: BuildAnalysis; + readonly branch: BuildAnalysis; + readonly deltaMs: number; + readonly deltaPct: number; + readonly verdict: TimingVerdict; + readonly exitCodesDiffer: boolean; + readonly spans: ReadonlyArray | undefined; + /** Top phase deltas; empty unless the verdict is "slower" or "faster". */ + readonly phaseRows: ReadonlyArray; + readonly endpointChanges: ReadonlyArray; +} + +/** Whether every launch of the workload exited 0 and every API call got a non-error response. */ +function runsCleanly(build: BuildAnalysis): boolean { + return ( + [...build.exitCodes, ...build.tracedExitCodes].every((code) => code === 0) && + (build.api?.errorStatuses ?? 0) === 0 + ); +} + +function endpointChanges( + base: BuildAnalysis, + branch: BuildAnalysis, +): ReadonlyArray { + const keys = new Set([ + ...Object.keys(base.api?.endpoints ?? {}), + ...Object.keys(branch.api?.endpoints ?? {}), + ]); + return [...keys] + .sort() + .map((endpoint) => ({ + endpoint, + base: base.api?.endpoints[endpoint] ?? 0, + branch: branch.api?.endpoints[endpoint] ?? 0, + })) + .filter((change) => change.base !== change.branch); +} + +/** Derives medians, comparisons and the timing verdict for one workload from its raw samples. */ +export function analyzeWorkload(workload: WorkloadReport, traced: boolean): WorkloadAnalysis { + const base = summarizeBuild(workload.base, traced); + const branch = summarizeBuild(workload.branch, traced); + const delta = deltaOf(base.timing.median, branch.timing.median); + const differ = exitCodesDiffer(base.exitCodes, branch.exitCodes); + const verdict = timingVerdict(base.timing, branch.timing, { + comparable: !differ && runsCleanly(base) && runsCleanly(branch), + samples: Math.min(base.samples, branch.samples), + }); + const spans = + base.spans === undefined || branch.spans === undefined + ? undefined + : compareSpans(base.spans, branch.spans); + return { + id: workload.id, + args: workload.args, + base, + branch, + deltaMs: delta.ms, + deltaPct: delta.pct, + verdict, + exitCodesDiffer: differ, + spans, + phaseRows: + verdict === "slower" || verdict === "faster" ? (spans ?? []).slice(0, PHASE_ROWS) : [], + endpointChanges: endpointChanges(base, branch), + }; +} + +export type FlagKind = + | "new-api-calls" + | "api-calls-dropped" + | "external-calls" + | "non-deterministic" + | "request-log-mismatch" + | "error-status" + | "trace-incomplete" + | "non-zero-exit" + | "exit-codes-differ" + | "traced-exit-differs"; + +export interface Flag { + readonly workload: string; + readonly kind: FlagKind; + readonly detail: string; +} + +/** Everything about a workload that makes its comparison suspect or notable beyond timing. */ +export function collectFlags(analysis: WorkloadAnalysis): ReadonlyArray { + const flags: Array = []; + const add = (kind: FlagKind, detail: string) => + flags.push({ workload: analysis.id, kind, detail }); + const baseCalls = analysis.base.api?.medianCalls; + const branchCalls = analysis.branch.api?.medianCalls; + if (baseCalls === 0 && branchCalls !== undefined && branchCalls > 0) { + add("new-api-calls", `base makes 0 API calls, branch makes ${branchCalls}`); + } + if (baseCalls !== undefined && baseCalls > 0 && branchCalls === 0) { + add("api-calls-dropped", `base makes ${baseCalls} API calls, branch makes 0`); + } + if (analysis.exitCodesDiffer) { + add("exit-codes-differ", "exit codes differ between builds, so timings are not comparable"); + } + for (const build of ["base", "branch"] as const) { + const summary = analysis[build]; + const failed = summary.exitCodes.filter((code) => code !== 0).length; + if (failed > 0) { + add("non-zero-exit", `${build} exited non-zero in ${failed}/${summary.samples} runs`); + } + const timed = new Set(summary.exitCodes); + if (summary.tracedExitCodes.some((code) => !timed.has(code))) { + add( + "traced-exit-differs", + `${build} traced launch exited differently from its timed launches`, + ); + } + if (summary.traceMissingRuns > 0 || summary.skippedTraceLines > 0) { + add( + "trace-incomplete", + `${build} has ${summary.traceMissingRuns} runs without a trace and ${summary.skippedTraceLines} undecodable trace lines`, + ); + } + const api = summary.api; + if (api === undefined) continue; + if (api.externalCalls > 0) { + add("external-calls", `${build} made calls to hosts other than the replay server`); + } + if (api.nonDeterministic) add("non-deterministic", `${build} API calls differ between runs`); + if (api.logMismatchRuns > 0) { + add( + "request-log-mismatch", + `${build} spans disagree with the replay request log in ${api.logMismatchRuns} runs`, + ); + } + if (api.errorStatuses > 0) { + add( + "error-status", + `${build} got 4xx/5xx replay responses, so it did not follow the recorded scenario`, + ); + } + } + return flags; +} diff --git a/apps/cli/scripts/bench-report.unit.test.ts b/apps/cli/scripts/bench-report.unit.test.ts new file mode 100644 index 0000000000..44dcaa5a13 --- /dev/null +++ b/apps/cli/scripts/bench-report.unit.test.ts @@ -0,0 +1,359 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { + aggregateSpanRuns, + analyzeWorkload, + classifyHttpSpans, + collectFlags, + compareSpans, + deltaOf, + exitCodesDiffer, + parseWorkloadManifest, + summarizeSpanRun, + summarizeTimings, + timingVerdict, + validateWorkloads, + type RunSample, + type SpanRunSummary, + type WorkloadReport, +} from "./bench-report.ts"; +import type { ReportSpan } from "./trace-report.ts"; + +const span = (name: string, startMs: number, endMs: number): ReportSpan => ({ + traceId: "t", + spanId: `${name}-${startMs}`, + parentSpanId: undefined, + name, + startMs, + endMs, + failed: false, + errorType: undefined, +}); + +const httpSpan = ( + method: string, + path: string, + port: number, + status = 200, + address = "127.0.0.1", +): ReportSpan => ({ + ...span(method, 0, 5), + http: { method, path, address, port, status }, +}); + +describe("summarizeTimings", () => { + it("computes median, min, max, and p90", () => { + expect(summarizeTimings([10, 20, 30, 40, 50])).toEqual({ + median: 30, + min: 10, + max: 50, + p90: 50, + }); + }); +}); + +describe("exitCodesDiffer", () => { + it("flags builds that exit differently but not shared failures", () => { + expect(exitCodesDiffer([1, 1], [1, 1])).toBe(false); + expect(exitCodesDiffer([0, 0], [1, 0])).toBe(true); + }); +}); + +describe("deltaOf", () => { + it("reports branch minus base in ms and percent", () => { + expect(deltaOf(100, 110)).toEqual({ ms: 10, pct: 10 }); + }); + + it("reports a zero percent delta instead of Infinity when base is zero", () => { + expect(deltaOf(0, 5)).toEqual({ ms: 5, pct: 0 }); + }); +}); + +describe("span aggregation", () => { + it("sums duration and counts per span name within one run, leaving HTTP spans to the API section", () => { + const run = [ + span("Config.load", 0, 10), + span("Db.query", 10, 20), + span("Db.query", 20, 50), + httpSpan("GET", "/v1/projects", 1), + ]; + + expect(summarizeSpanRun(run)).toEqual({ + "Config.load": { totalMs: 10, count: 1 }, + "Db.query": { totalMs: 40, count: 2 }, + }); + }); + + it("medians per-span totals and counts across runs, treating an absent span as zero", () => { + const runs: ReadonlyArray> = [ + { "Db.query": { totalMs: 40, count: 2 } }, + { "Db.query": { totalMs: 60, count: 2 } }, + {}, + ]; + + expect(aggregateSpanRuns(runs)).toEqual({ "Db.query": { medianMs: 40, medianCount: 2 } }); + }); +}); + +describe("compareSpans", () => { + it("flags spans found in only one build and sorts by absolute delta", () => { + const base = { + "Config.load": { medianMs: 10, medianCount: 1 }, + "Credentials.load": { medianMs: 5, medianCount: 1 }, + }; + const branch = { + "Config.load": { medianMs: 40, medianCount: 1 }, + "Otlp.flush": { medianMs: 3, medianCount: 1 }, + }; + + expect( + compareSpans(base, branch).map((entry) => [entry.name, entry.deltaMs, entry.onlyIn]), + ).toEqual([ + ["Config.load", 30, undefined], + ["Credentials.load", undefined, "base"], + ["Otlp.flush", undefined, "branch"], + ]); + }); +}); + +describe("workload manifest", () => { + const entry = { id: "w", args: ["init"], cwdSetup: "none" }; + + it.effect("decodes entries and rejects an unknown cwdSetup", () => + Effect.gen(function* () { + const parsed = yield* parseWorkloadManifest( + '{"workloads":[{"id":"w","args":["init"],"cwdSetup":"none"}]}', + ); + expect(parsed).toEqual([entry]); + + const exit = yield* Effect.exit( + parseWorkloadManifest('{"workloads":[{"id":"w","args":["init"],"cwdSetup":"bogus"}]}'), + ); + expect(exit._tag).toBe("Failure"); + }), + ); + + it("requires a projectRef for linked workloads, --api-url for scenarios, and unique ids", () => { + const problems = validateWorkloads( + [ + { id: "a", args: [], cwdSetup: "linked" }, + { id: "b", args: [], cwdSetup: "none", scenario: "s" }, + { id: "b", args: [], cwdSetup: "none" }, + ], + { apiUrl: undefined }, + ); + + expect(problems).toEqual([ + 'workload "a" uses cwdSetup "linked" and needs a projectRef', + 'workload "b" names a scenario and needs --api-url', + 'duplicate workload id "b"', + ]); + expect( + validateWorkloads([{ ...entry, scenario: "s" }] as never, { apiUrl: "http://127.0.0.1:1" }), + ).toEqual([]); + }); +}); + +describe("classifyHttpSpans", () => { + const apiUrl = "http://127.0.0.1:4010"; + + it("counts calls to the API host and port per endpoint and separates other hosts and ports", () => { + const summary = classifyHttpSpans( + [ + httpSpan("GET", "/v1/projects", 4010), + httpSpan("GET", "/v1/projects", 4010), + httpSpan("POST", "/v1/x", 4010, 502), + httpSpan("GET", "/v1/projects", 4011), + httpSpan("GET", "/", 443, 200, "api.supabase.com"), + span("Config.load", 0, 1), + ], + apiUrl, + ); + + expect(summary).toEqual({ + apiCalls: 3, + externalCalls: 2, + errorStatuses: 1, + endpoints: { "GET /v1/projects": 2, "POST /v1/x": 1 }, + }); + }); +}); + +describe("timingVerdict", () => { + const base = { median: 100, min: 95, max: 108, p90: 107 }; + const context = { comparable: true, samples: 10 }; + + it.each([ + { branch: { median: 120, min: 115, max: 125, p90: 124 }, expected: "slower" }, + { branch: { median: 80, min: 78, max: 90, p90: 88 }, expected: "faster" }, + { branch: { median: 103, min: 99, max: 110, p90: 109 }, expected: "within noise" }, + { branch: { median: 107, min: 99, max: 118, p90: 117 }, expected: "within noise" }, + ])("calls a branch median of $branch.median $expected", ({ branch, expected }) => { + expect(timingVerdict(base, branch, context)).toBe(expected); + }); + + it("ignores a large percentage change that is under the absolute floor", () => { + expect( + timingVerdict( + { median: 10, min: 9, max: 11, p90: 11 }, + { median: 14, min: 13, max: 15, p90: 15 }, + context, + ), + ).toBe("within noise"); + }); + + it("declines to compare when runs are not comparable or too few", () => { + const slow = { median: 200, min: 190, max: 210, p90: 209 }; + expect(timingVerdict(base, slow, { comparable: false, samples: 10 })).toBe("not comparable"); + expect(timingVerdict(base, slow, { comparable: true, samples: 3 })).toBe("not comparable"); + }); +}); + +const sample = ( + wallMs: number, + api?: { + calls: number; + endpoints?: Record; + log?: number; + external?: number; + errors?: number; + }, + exitCode = 0, +): RunSample => ({ + wallMs, + exitCode, + trace: { + exitCode, + skippedLines: 0, + spans: { "cli.run": { totalMs: wallMs / 2, count: 1 } }, + ...(api === undefined + ? {} + : { + api: { + apiCalls: api.calls, + externalCalls: api.external ?? 0, + errorStatuses: api.errors ?? 0, + endpoints: api.endpoints ?? {}, + requestLogCount: api.log ?? api.calls, + }, + }), + }, +}); + +const workload = ( + base: ReadonlyArray, + branch: ReadonlyArray, +): WorkloadReport => ({ + id: "w", + args: ["migration", "new"], + base, + branch, +}); + +const times = (n: number, make: (i: number) => RunSample) => + Array.from({ length: n }, (_, i) => make(i)); + +describe("analyzeWorkload and collectFlags", () => { + it("flags a workload whose base makes no API calls and whose branch makes one", () => { + const analysis = analyzeWorkload( + workload( + times(5, () => sample(50, { calls: 0 })), + times(5, () => sample(51, { calls: 1, endpoints: { "GET /v1/projects/abc": 1 } })), + ), + true, + ); + + expect(collectFlags(analysis).map((flag) => flag.kind)).toEqual(["new-api-calls"]); + expect(analysis.endpointChanges).toEqual([ + { endpoint: "GET /v1/projects/abc", base: 0, branch: 1 }, + ]); + }); + + it("lists a count change from one call to two without a zero flag", () => { + const analysis = analyzeWorkload( + workload( + times(5, () => sample(50, { calls: 1, endpoints: { "GET /v1/projects": 1 } })), + times(5, () => sample(50, { calls: 2, endpoints: { "GET /v1/projects": 2 } })), + ), + true, + ); + + expect(collectFlags(analysis)).toEqual([]); + expect(analysis.branch.api?.medianCalls).toBe(2); + expect(analysis.endpointChanges).toEqual([ + { endpoint: "GET /v1/projects", base: 1, branch: 2 }, + ]); + }); + + it("flags external calls, non-determinism, and spans that disagree with the request log", () => { + const noisy = [ + sample(50, { calls: 1, endpoints: { "GET /a": 1 }, external: 1 }), + sample(50, { calls: 2, endpoints: { "GET /a": 2 }, log: 1 }), + sample(50, { calls: 1, endpoints: { "GET /a": 1 } }), + sample(50, { calls: 1, endpoints: { "GET /a": 1 } }), + sample(50, { calls: 1, endpoints: { "GET /a": 1 } }), + ]; + const steady = times(5, () => sample(50, { calls: 1, endpoints: { "GET /a": 1 } })); + + const kinds = collectFlags(analyzeWorkload(workload(steady, noisy), true)).map( + (flag) => flag.kind, + ); + + expect(kinds).toEqual(["external-calls", "non-deterministic", "request-log-mismatch"]); + }); + + it("flags a branch that lost all API calls and treats a missing trace as incomplete", () => { + const analysis = analyzeWorkload( + workload( + times(5, () => sample(50, { calls: 1 })), + [...times(4, () => sample(50, { calls: 0 })), { wallMs: 50, exitCode: 0 }], + ), + true, + ); + + expect(collectFlags(analysis).map((flag) => flag.kind)).toEqual([ + "api-calls-dropped", + "trace-incomplete", + ]); + }); + + it("withholds the verdict and flags failures when a build exits non-zero or gets error responses", () => { + const failing = times(5, () => sample(200, { calls: 1, errors: 1 }, 1)); + const analysis = analyzeWorkload( + workload( + times(5, () => sample(50, { calls: 1 })), + failing, + ), + true, + ); + + expect(analysis.verdict).toBe("not comparable"); + expect(collectFlags(analysis).map((flag) => flag.kind)).toEqual([ + "exit-codes-differ", + "non-zero-exit", + "error-status", + ]); + }); + + it("reports the heaviest phase deltas only when the timing verdict is slower or faster", () => { + const slow = analyzeWorkload( + workload( + times(5, (i) => sample(100 + i)), + times(5, (i) => sample(200 + i)), + ), + true, + ); + const same = analyzeWorkload( + workload( + times(5, (i) => sample(100 + i)), + times(5, (i) => sample(101 + i)), + ), + true, + ); + + expect(slow.verdict).toBe("slower"); + expect(slow.phaseRows.map((row) => row.name)).toEqual(["cli.run"]); + expect(same.verdict).toBe("within noise"); + expect(same.phaseRows).toEqual([]); + }); +}); diff --git a/apps/cli/scripts/bench-summary.ts b/apps/cli/scripts/bench-summary.ts new file mode 100644 index 0000000000..739850dc40 --- /dev/null +++ b/apps/cli/scripts/bench-summary.ts @@ -0,0 +1,181 @@ +/** + * Renders a `bench-cli.ts --out` report as a Markdown job summary: flags, wall time, API calls, + * phase deltas, and binary size. Run as + * `bun scripts/bench-summary.ts --report [--base-build ok|failed]`; writes to + * `GITHUB_STEP_SUMMARY` when set, otherwise to stdout. + */ +import process from "node:process"; +import { BunRuntime, BunServices } from "@effect/platform-bun"; +import { Config, Console, Effect, FileSystem, Option, Stdio } from "effect"; +import { + analyzeWorkload, + collectFlags, + decodeBenchReport, + type BenchReport, + type Flag, + type WorkloadAnalysis, +} from "./bench-report.ts"; + +const USAGE = "usage: bun scripts/bench-summary.ts --report [--base-build ok|failed]"; + +export interface SummaryOptions { + readonly report: string | undefined; + readonly baseBuild: "ok" | "failed"; +} + +/** Parses argv; throws on an unknown `--base-build` or a missing report for a successful base. */ +export function parseSummaryArgs(argv: ReadonlyArray): SummaryOptions { + const valueOf = (flag: string) => { + const index = argv.indexOf(flag); + return index === -1 ? undefined : argv[index + 1]; + }; + const baseBuild = valueOf("--base-build") ?? "ok"; + if (baseBuild !== "ok" && baseBuild !== "failed") { + throw new Error('--base-build must be "ok" or "failed"'); + } + const report = valueOf("--report"); + if (report === undefined && baseBuild === "ok") throw new Error(USAGE); + return { report, baseBuild }; +} + +const BASE_FAILED = "base build failed, no comparison"; + +const sign = (value: number) => (value >= 0 ? "+" : ""); +const ms = (value: number) => `${value} ms`; +const range = (build: WorkloadAnalysis["base"]) => `${build.timing.min}-${build.timing.max} ms`; + +function table(header: ReadonlyArray, rows: ReadonlyArray>): string { + return [ + `| ${header.join(" | ")} |`, + `| ${header.map(() => "---").join(" | ")} |`, + ...rows.map((row) => `| ${row.join(" | ")} |`), + ].join("\n"); +} + +function flagsSection(flags: ReadonlyArray): string { + if (flags.length === 0) return "No flags."; + return flags.map((flag) => `- **${flag.workload}** (${flag.kind}): ${flag.detail}`).join("\n"); +} + +function wallTimeSection(analyses: ReadonlyArray): string { + return table( + ["Workload", "Base", "Branch", "Change", "Base range", "Branch range", "Verdict"], + analyses.map((analysis) => [ + analysis.id, + ms(analysis.base.timing.median), + ms(analysis.branch.timing.median), + `${sign(analysis.deltaMs)}${analysis.deltaMs} ms (${sign(analysis.deltaPct)}${analysis.deltaPct}%)`, + range(analysis.base), + range(analysis.branch), + analysis.verdict, + ]), + ); +} + +function apiSection(analyses: ReadonlyArray): string { + const rows = analyses + .filter((analysis) => analysis.base.api !== undefined || analysis.branch.api !== undefined) + .map((analysis) => [ + analysis.id, + String(analysis.base.api?.medianCalls ?? "n/a"), + String(analysis.branch.api?.medianCalls ?? "n/a"), + analysis.endpointChanges.length === 0 + ? "" + : analysis.endpointChanges + .map((change) => `\`${change.endpoint}\` ${change.base} -> ${change.branch}`) + .join("
"), + ]); + if (rows.length === 0) return "API calls were not counted."; + return table(["Workload", "Base", "Branch", "Endpoint changes"], rows); +} + +function phaseSection(analyses: ReadonlyArray): string { + const rows = analyses.flatMap((analysis) => + analysis.phaseRows.map((row) => { + const only = row.onlyIn === undefined ? undefined : row[row.onlyIn]; + return [ + analysis.id, + row.name, + row.deltaMs === undefined + ? `${row.onlyIn} only, ${only?.medianMs} ms` + : `${sign(row.deltaMs)}${row.deltaMs} ms`, + `${row.base?.medianMs ?? "-"} -> ${row.branch?.medianMs ?? "-"} ms`, + ]; + }), + ); + if (rows.length === 0) return "No workload moved beyond noise, so there are no phase deltas."; + return table(["Workload", "Span", "Change", "Base -> branch"], rows); +} + +function sizeSection(report: BenchReport): string { + const { baseSizeBytes: base, branchSizeBytes: branch } = report; + if (base === undefined || branch === undefined) return "Binary sizes were not recorded."; + const delta = branch - base; + return table( + ["Base", "Branch", "Change"], + [[`${base} bytes`, `${branch} bytes`, `${sign(delta)}${delta} bytes`]], + ); +} + +/** The Markdown summary, or the base-build-failed notice when there is no report to compare. */ +export function renderSummary(report: BenchReport | undefined): string { + const title = "## CLI benchmark (report only)"; + if (report === undefined) return `${title}\n\n${BASE_FAILED}\n`; + const analyses = report.workloads.map((workload) => analyzeWorkload(workload, report.trace)); + const flags = analyses.flatMap(collectFlags); + return ( + [ + title, + `${report.runs} measured runs per build after ${report.warmup} warm-up runs. This job never blocks a pull request.`, + "### Flags", + flagsSection(flags), + "### Wall time", + wallTimeSection(analyses), + "### API calls", + apiSection(analyses), + "### Phase deltas", + phaseSection(analyses), + "### Binary size", + sizeSection(report), + ].join("\n\n") + "\n" + ); +} + +const main = Effect.gen(function* () { + const stdio = yield* Stdio.Stdio; + const argv = yield* stdio.args; + const options = yield* Effect.try({ + try: () => parseSummaryArgs(argv), + catch: (cause) => (cause instanceof Error ? cause.message : String(cause)), + }).pipe( + Effect.catch((message) => + Console.error(message).pipe( + Effect.andThen( + Effect.sync(() => { + process.exitCode = 2; + }), + ), + Effect.as(undefined), + ), + ), + ); + if (options === undefined) return; + + const fs = yield* FileSystem.FileSystem; + const report = + options.baseBuild === "failed" || options.report === undefined + ? undefined + : yield* decodeBenchReport(yield* fs.readFileString(options.report)); + const markdown = renderSummary(report); + + const summaryFile = yield* Config.option(Config.String("GITHUB_STEP_SUMMARY")); + if (Option.isSome(summaryFile) && summaryFile.value !== "") { + yield* fs.writeFileString(summaryFile.value, markdown, { flag: "a" }); + } else { + yield* Console.log(markdown); + } +}); + +if (import.meta.main) { + BunRuntime.runMain(main.pipe(Effect.provide(BunServices.layer))); +} diff --git a/apps/cli/scripts/bench-summary.unit.test.ts b/apps/cli/scripts/bench-summary.unit.test.ts new file mode 100644 index 0000000000..642d8c8660 --- /dev/null +++ b/apps/cli/scripts/bench-summary.unit.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; +import type { BenchReport, RunSample } from "./bench-report.ts"; +import { parseSummaryArgs, renderSummary } from "./bench-summary.ts"; + +const run = (wallMs: number, calls: number, endpoints: Record): RunSample => ({ + wallMs, + exitCode: 0, + trace: { + exitCode: 0, + skippedLines: 0, + spans: { "cli.run": { totalMs: wallMs / 2, count: 1 } }, + api: { apiCalls: calls, externalCalls: 0, errorStatuses: 0, endpoints, requestLogCount: calls }, + }, +}); + +const runs = (n: number, make: (i: number) => RunSample) => + Array.from({ length: n }, (_, i) => make(i)); + +const report = (workloads: BenchReport["workloads"]): BenchReport => ({ + base: "/base/supabase", + branch: "/branch/supabase", + baseSizeBytes: 1000, + branchSizeBytes: 1500, + runs: 5, + warmup: 1, + trace: true, + apiUrl: "http://127.0.0.1:1", + workloads, +}); + +describe("renderSummary", () => { + it("lists a flagged workload first and shows the endpoint that appeared", () => { + const markdown = renderSummary( + report([ + { + id: "migration-new", + args: ["migration", "new", "bench"], + base: runs(5, (i) => run(50 + i, 0, {})), + branch: runs(5, (i) => run(120 + i, 1, { "GET /v1/projects/abc": 1 })), + }, + ]), + ); + + expect(markdown).toContain("report only"); + expect(markdown).toMatch(/\*\*migration-new\*\* \(new-api-calls\)/u); + expect(markdown).not.toContain("No flags."); + expect(markdown).toContain("`GET /v1/projects/abc` 0 -> 1"); + expect(markdown).toContain("slower"); + expect(markdown).toContain("+500 bytes"); + }); + + it("says there are no flags and no phase deltas when nothing moved", () => { + const markdown = renderSummary( + report([ + { + id: "version", + args: ["--version"], + base: runs(5, (i) => run(30 + i, 0, {})), + branch: runs(5, (i) => run(30 + i, 0, {})), + }, + ]), + ); + + expect(markdown).toContain("No flags."); + expect(markdown).toContain("within noise"); + expect(markdown).toContain("no phase deltas"); + }); + + it("reports a failed base build without a comparison", () => { + expect(renderSummary(undefined)).toContain("base build failed, no comparison"); + }); +}); + +describe("parseSummaryArgs", () => { + it("needs a report unless the base build failed", () => { + expect(() => parseSummaryArgs([])).toThrow(); + expect(parseSummaryArgs(["--base-build", "failed"])).toEqual({ + report: undefined, + baseBuild: "failed", + }); + expect(parseSummaryArgs(["--report", "r.json"])).toEqual({ report: "r.json", baseBuild: "ok" }); + }); +}); diff --git a/apps/cli/scripts/bench-workloads.unit.test.ts b/apps/cli/scripts/bench-workloads.unit.test.ts new file mode 100644 index 0000000000..0f48264239 --- /dev/null +++ b/apps/cli/scripts/bench-workloads.unit.test.ts @@ -0,0 +1,39 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem } from "effect"; +import { parseWorkloadManifest, validateWorkloads } from "./bench-report.ts"; + +const manifestPath = new URL("../bench/workloads.pr.json", import.meta.url).pathname; +const scenariosDir = new URL("../../cli-e2e/fixtures/scenarios/", import.meta.url).pathname; + +const loadManifest = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return yield* parseWorkloadManifest(yield* fs.readFileString(manifestPath)); +}); + +describe("PR workload manifest", () => { + it.effect("decodes and passes the structural rules when a replay server is configured", () => + Effect.gen(function* () { + const workloads = yield* loadManifest; + + expect(workloads.length).toBeGreaterThan(0); + expect(validateWorkloads(workloads, { apiUrl: "http://127.0.0.1:1" })).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("names only replay scenarios that exist and serve the project ref it uses", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const workloads = yield* loadManifest; + + for (const workload of workloads) { + if (workload.scenario === undefined) continue; + const fixture = yield* fs.readFileString( + `${scenariosDir}${workload.scenario}/interactions.json`, + ); + const ref = workload.projectRef ?? workload.args.find((arg) => arg.startsWith("replay")); + if (ref !== undefined) expect(fixture).toContain(`/v1/projects/${ref}`); + } + }).pipe(Effect.provide(BunServices.layer)), + ); +}); diff --git a/apps/cli/scripts/replay-control.integration.test.ts b/apps/cli/scripts/replay-control.integration.test.ts new file mode 100644 index 0000000000..2ea262acae --- /dev/null +++ b/apps/cli/scripts/replay-control.integration.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { FetchHttpClient } from "effect/http"; +import * as HttpClient from "effect/http/HttpClient"; +import { startReplayServer } from "../../cli-e2e/src/server/replay-server.ts"; +import { controlPlan, readRequestLogCount, resetReplay } from "./replay-control.ts"; + +const fixturesDir = new URL("../../cli-e2e/fixtures", import.meta.url).pathname; +const SCENARIO = "projects-list-renders-project-list"; + +const replayServer = Effect.acquireRelease( + Effect.promise(() => startReplayServer({ fixturesDir })), + (server) => Effect.promise(() => server.stop()), +); + +const callApi = (url: string, path: string) => + HttpClient.HttpClient.use((client) => client.get(new URL(path, url).toString())); + +describe("controlPlan", () => { + it("always clears the request log and scenario before selecting a scenario", () => { + expect(controlPlan(undefined).map((call) => `${call.method} ${call.path}`)).toEqual([ + "DELETE /_ctrl/requests", + "DELETE /_ctrl/scenario", + ]); + expect(controlPlan("s").at(-1)).toEqual({ + method: "POST", + path: "/_ctrl/scenario", + body: { name: "s" }, + }); + }); +}); + +describe("replay control against the replay server", () => { + it.effect("serves a consumed scenario again and starts each launch with an empty log", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* replayServer; + + yield* resetReplay(server.url, SCENARIO); + const first = yield* callApi(server.url, "/v1/projects"); + const logged = yield* readRequestLogCount(server.url); + + yield* resetReplay(server.url, SCENARIO); + const cleared = yield* readRequestLogCount(server.url); + const second = yield* callApi(server.url, "/v1/projects"); + + expect([first.status, second.status]).toEqual([200, 200]); + expect(logged).toBe(1); + expect(cleared).toBe(0); + }), + ).pipe(Effect.provide(FetchHttpClient.layer)), + ); + + it.effect("fails when the scenario does not exist instead of keeping the previous one", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* replayServer; + + const failure = yield* resetReplay(server.url, "no-such-scenario").pipe(Effect.flip); + + expect(failure._tag).toBe("ReplayControlError"); + expect(failure.step).toBe("POST /_ctrl/scenario"); + }), + ).pipe(Effect.provide(FetchHttpClient.layer)), + ); + + it.effect("fails when the server is unreachable", () => + Effect.gen(function* () { + const failure = yield* resetReplay("http://127.0.0.1:1", undefined).pipe(Effect.flip); + + expect(failure._tag).toBe("ReplayControlError"); + }).pipe(Effect.provide(FetchHttpClient.layer)), + ); +}); diff --git a/apps/cli/scripts/replay-control.ts b/apps/cli/scripts/replay-control.ts new file mode 100644 index 0000000000..b8b72e052b --- /dev/null +++ b/apps/cli/scripts/replay-control.ts @@ -0,0 +1,80 @@ +/** + * Client for the replay server's `/_ctrl/*` HTTP API, used by `bench-cli.ts` to start each launch + * from a known server state. Any failed control call aborts the benchmark, because a launch that + * runs against stale or missing scenario state measures the wrong work. + */ +import { Effect, Schema } from "effect"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientRequest from "effect/http/HttpClientRequest"; +import * as HttpClientResponse from "effect/http/HttpClientResponse"; + +export class ReplayControlError extends Schema.TaggedError()( + "ReplayControlError", + { step: Schema.String, message: Schema.String }, +) {} + +export interface ControlCall { + readonly method: "DELETE" | "POST"; + readonly path: string; + readonly body?: { readonly name: string }; +} + +/** The calls that clear the request log and select `scenario`, or no scenario when it is absent. */ +export function controlPlan(scenario: string | undefined): ReadonlyArray { + return [ + { method: "DELETE", path: "/_ctrl/requests" }, + { method: "DELETE", path: "/_ctrl/scenario" }, + ...(scenario === undefined + ? [] + : [{ method: "POST", path: "/_ctrl/scenario", body: { name: scenario } } as const]), + ]; +} + +const controlUrl = (apiUrl: string, path: string) => new URL(path, apiUrl).toString(); + +const failure = (step: string) => (cause: unknown) => + new ReplayControlError({ + step, + message: `${step} failed: ${cause instanceof Error ? cause.message : String(cause)}`, + }); + +const send = Effect.fn("ReplayControl.send")(function* ( + apiUrl: string, + method: "GET" | "DELETE" | "POST", + path: string, + body?: { readonly name: string }, +) { + const step = `${method} ${path}`; + const client = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk); + const base = + method === "GET" + ? HttpClientRequest.get(controlUrl(apiUrl, path)) + : method === "DELETE" + ? HttpClientRequest.delete(controlUrl(apiUrl, path)) + : HttpClientRequest.post(controlUrl(apiUrl, path)); + const request = body === undefined ? base : HttpClientRequest.bodyJsonUnsafe(base, body); + return yield* client.execute(request).pipe(Effect.mapError(failure(step))); +}); + +/** Resets the request log and scenario so the next launch sees a freshly loaded scenario. */ +export const resetReplay = Effect.fn("ReplayControl.reset")(function* ( + apiUrl: string, + scenario: string | undefined, +) { + for (const call of controlPlan(scenario)) { + yield* send(apiUrl, call.method, call.path, call.body); + } +}); + +const RequestLog = Schema.Array(Schema.Struct({ method: Schema.String, pathname: Schema.String })); + +/** Number of API requests the server has logged since the last reset. */ +export const readRequestLogCount = Effect.fn("ReplayControl.readRequestLogCount")(function* ( + apiUrl: string, +) { + const response = yield* send(apiUrl, "GET", "/_ctrl/requests"); + const json = yield* HttpClientResponse.schemaBodyJson(RequestLog)(response).pipe( + Effect.mapError(failure("GET /_ctrl/requests body")), + ); + return json.length; +}); diff --git a/apps/cli/scripts/trace-report.ts b/apps/cli/scripts/trace-report.ts index a16430839c..41cc789ece 100644 --- a/apps/cli/scripts/trace-report.ts +++ b/apps/cli/scripts/trace-report.ts @@ -8,13 +8,23 @@ import { Console, Effect, FileSystem, Schema, Stdio } from "effect"; const KeyValues = Schema.Array( Schema.Struct({ key: Schema.String, - value: Schema.Struct({ stringValue: Schema.optional(Schema.String) }), + value: Schema.Struct({ + stringValue: Schema.optional(Schema.String), + // OTLP/JSON encodes 64-bit integers as strings; this exporter writes numbers. + intValue: Schema.optional(Schema.Union([Schema.Finite, Schema.String])), + }), }), ); const stringAttribute = (attributes: typeof KeyValues.Type | undefined, key: string) => attributes?.find((attribute) => attribute.key === key)?.value.stringValue; +const intAttribute = (attributes: typeof KeyValues.Type | undefined, key: string) => { + const value = attributes?.find((attribute) => attribute.key === key)?.value.intValue; + const number = value === undefined ? Number.NaN : Number(value); + return Number.isInteger(number) ? number : undefined; +}; + /** Labels container CLI spans with their verb so `pull` and `run` report separately. */ export function reportSpanName(name: string, subcommand: string | undefined): string { return subcommand === undefined ? name : `${name} (${subcommand})`; @@ -47,6 +57,15 @@ const TraceBatch = Schema.fromJsonString( const ReportJson = Schema.fromJsonString(Schema.Unknown, { space: 2 }); +/** Attributes of an HTTP client span; `path` has no query string. */ +export interface HttpSpanAttributes { + readonly method: string; + readonly path: string | undefined; + readonly address: string | undefined; + readonly port: number | undefined; + readonly status: number | undefined; +} + export interface ReportSpan { readonly traceId: string; readonly spanId: string; @@ -56,6 +75,8 @@ export interface ReportSpan { readonly endMs: number; readonly failed: boolean; readonly errorType: string | undefined; + /** Present only on HTTP client spans. */ + readonly http?: HttpSpanAttributes; } export interface TraceReport { @@ -74,6 +95,19 @@ export interface TraceReport { } const STATUS_ERROR = 2; + +function httpAttributes(attributes: typeof KeyValues.Type | undefined) { + const method = stringAttribute(attributes, "http.request.method"); + if (method === undefined) return undefined; + return { + method, + path: stringAttribute(attributes, "url.path")?.split("?")[0], + address: stringAttribute(attributes, "server.address"), + port: intAttribute(attributes, "server.port"), + status: intAttribute(attributes, "http.response.status_code"), + } satisfies HttpSpanAttributes; +} + const nanosToMs = (value: string) => Number(BigInt(value) / 1_000n) / 1_000; const round = (value: number) => Math.round(value * 100) / 100; @@ -192,6 +226,7 @@ export const readSpans = Effect.fnUntraced(function* (file: string) { for (const resource of batch.resourceSpans) { for (const scope of resource.scopeSpans) { for (const span of scope.spans) { + const http = httpAttributes(span.attributes); spans.push({ traceId: span.traceId, spanId: span.spanId, @@ -204,6 +239,7 @@ export const readSpans = Effect.fnUntraced(function* (file: string) { span.events?.find((event) => event.name === "exception")?.attributes, "exception.type", ), + ...(http === undefined ? {} : { http }), }); } } diff --git a/apps/cli/scripts/trace-report.unit.test.ts b/apps/cli/scripts/trace-report.unit.test.ts index c2fff6938e..b9195f141f 100644 --- a/apps/cli/scripts/trace-report.unit.test.ts +++ b/apps/cli/scripts/trace-report.unit.test.ts @@ -149,6 +149,74 @@ describe("readSpans", () => { }); }); +describe("readSpans HTTP attributes", () => { + const fixture = new URL("./__fixtures__/api-call-trace.jsonl", import.meta.url).pathname; + + it.effect("reads method, path, host, port and status from traces of both CLI generations", () => + Effect.gen(function* () { + const { spans, skippedLines } = yield* readSpans(fixture); + const http = spans.flatMap((span) => (span.http === undefined ? [] : [span.http])); + + expect(skippedLines).toBe(0); + expect(http).toHaveLength(2); + for (const call of http) { + expect(call).toEqual({ + method: "GET", + path: "/v1/projects", + address: "127.0.0.1", + port: 62824, + status: 200, + }); + } + expect(spans.find((span) => span.name === "cli.run")?.http).toBeUndefined(); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("accepts OTLP string-encoded integers and drops the query string from the path", () => { + const batch = JSON.stringify({ + resourceSpans: [ + { + scopeSpans: [ + { + spans: [ + { + traceId: "t", + spanId: "s", + name: "POST", + startTimeUnixNano: "0", + endTimeUnixNano: "1000000", + status: { code: 1 }, + attributes: [ + { key: "http.request.method", value: { stringValue: "POST" } }, + { key: "url.path", value: { stringValue: "/v1/x?limit=1" } }, + { key: "server.port", value: { intValue: "8443" } }, + { key: "http.response.status_code", value: { intValue: "503" } }, + ], + }, + ], + }, + ], + }, + ], + }); + + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const file = yield* fs.makeTempFileScoped({ suffix: ".jsonl" }); + yield* fs.writeFileString(file, `${batch}\n`); + const { spans } = yield* readSpans(file); + + expect(spans[0]?.http).toEqual({ + method: "POST", + path: "/v1/x", + address: undefined, + port: 8443, + status: 503, + }); + }).pipe(Effect.provide(BunServices.layer)); + }); +}); + describe("selectTrace", () => { const twoRuns = [ { ...span("a1", undefined, "cli.run", 0, 100), traceId: "first" }, From 5b3417f6fd19472db72fcb3ca0633c3e8c0648f4 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 15:01:54 +0530 Subject: [PATCH 03/15] chore(cli): harden bench-cli replay reset ordering and summary rendering Co-Authored-By: Claude Sonnet 5.5 --- .../cli/scripts/bench-cli.integration.test.ts | 72 +++++++++++++++ apps/cli/scripts/bench-cli.ts | 32 +++---- apps/cli/scripts/bench-cli.unit.test.ts | 6 +- apps/cli/scripts/bench-report.ts | 16 +++- apps/cli/scripts/bench-report.unit.test.ts | 88 ++++++++++++++++++- apps/cli/scripts/bench-summary.ts | 26 +++--- apps/cli/scripts/bench-summary.unit.test.ts | 26 ++++++ .../replay-control.integration.test.ts | 16 +--- 8 files changed, 229 insertions(+), 53 deletions(-) create mode 100644 apps/cli/scripts/bench-cli.integration.test.ts diff --git a/apps/cli/scripts/bench-cli.integration.test.ts b/apps/cli/scripts/bench-cli.integration.test.ts new file mode 100644 index 0000000000..244408070d --- /dev/null +++ b/apps/cli/scripts/bench-cli.integration.test.ts @@ -0,0 +1,72 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Ref } from "effect"; +import { FetchHttpClient } from "effect/http"; +import { startReplayServer } from "../../cli-e2e/src/server/replay-server.ts"; +import { runWorkloadBench, type BenchOptions } from "./bench-cli.ts"; + +const fixturesDir = new URL("../../cli-e2e/fixtures", import.meta.url).pathname; + +// Stands in for a CLI build: every launch makes one request to the profile's API and, when asked +// to trace, records it as an HTTP client span. +const FAKE_CLI = `#!/bin/sh +URL=$(sed -n 's/^api_url: "\\(.*\\)"$/\\1/p' "$SUPABASE_PROFILE") +curl -s -o /dev/null "$URL/v1/projects" +if [ -n "$SUPABASE_TRACE_FILE" ]; then + PORT=\${URL##*:} + printf '{"resourceSpans":[{"scopeSpans":[{"spans":[{"traceId":"t","spanId":"s","name":"GET","startTimeUnixNano":"0","endTimeUnixNano":"1000000","status":{"code":1},"attributes":[{"key":"http.request.method","value":{"stringValue":"GET"}},{"key":"url.path","value":{"stringValue":"/v1/projects"}},{"key":"server.address","value":{"stringValue":"127.0.0.1"}},{"key":"server.port","value":{"intValue":%s}},{"key":"http.response.status_code","value":{"intValue":200}}]}]}]}]}\\n' "$PORT" >> "$SUPABASE_TRACE_FILE" +fi +`; + +describe("runWorkloadBench", () => { + it.effect("counts only the measured launch's API calls, not setup or earlier launches", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const server = yield* Effect.acquireRelease( + Effect.promise(() => startReplayServer({ fixturesDir })), + (handle) => Effect.promise(() => handle.stop()), + ); + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "bench-fake-cli-" }); + const binary = `${dir}/supabase`; + yield* fs.writeFileString(binary, FAKE_CLI); + yield* fs.chmod(binary, 0o755); + const options: BenchOptions = { + base: binary, + branch: binary, + runs: 2, + warmup: 1, + commands: [], + workloadsFile: undefined, + cwdSetupInit: false, + apiUrl: server.url, + trace: true, + json: false, + out: undefined, + cpuProf: false, + updateCheck: false, + }; + + const { report } = yield* runWorkloadBench( + options, + { + id: "w", + args: ["projects", "list"], + cwdSetup: "init", + scenario: "projects-list-renders-project-list", + }, + yield* Ref.make(false), + ); + + const samples = [...report.base, ...report.branch]; + expect(samples).toHaveLength(4); + for (const sample of samples) { + expect(sample.exitCode).toBe(0); + expect(sample.trace?.api?.apiCalls).toBe(1); + expect(sample.trace?.api?.requestLogCount).toBe(1); + expect(sample.trace?.api?.externalCalls).toBe(0); + } + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); +}); diff --git a/apps/cli/scripts/bench-cli.ts b/apps/cli/scripts/bench-cli.ts index 8494fc8e92..1227703f14 100644 --- a/apps/cli/scripts/bench-cli.ts +++ b/apps/cli/scripts/bench-cli.ts @@ -10,24 +10,14 @@ import { tmpdir } from "node:os"; import process from "node:process"; import { parseArgs } from "node:util"; import { BunPath, BunRuntime, BunServices } from "@effect/platform-bun"; -import { - Clock, - Console, - Effect, - FileSystem, - Layer, - Option, - Path, - Ref, - Schema, - Stdio, -} from "effect"; +import { Clock, Console, Effect, FileSystem, Layer, Option, Path, Ref, Stdio } from "effect"; import { FetchHttpClient } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { analyzeWorkload, BenchReport, classifyHttpSpans, + encodeBenchReport, collectFlags, parseWorkloadManifest, summarizeSpanRun, @@ -46,7 +36,7 @@ const USAGE = 'usage: bun scripts/bench-cli.ts --base --branch [--runs N] [--warmup N] [--command ""]... [--workloads ] [--api-url ] [--cwd-setup init] [--trace] [--json] [--out ] [--cpu-prof] [--update-check]\n' + " --command splits on whitespace; it does not support shell quoting. Use --workloads for quoted arguments.\n" + " --workloads reads a manifest of workloads and cannot be combined with --command or --cwd-setup.\n" + - " --api-url points every launch at a replay server through a profile file, and counts Management API calls; it requires --trace.\n" + + " --api-url points every launch at a replay server through a profile file, and counts Management API calls from the traced launches; it requires --trace.\n" + " --update-check keeps the GitHub release check, which otherwise adds network time to every launch."; export interface BenchOptions { @@ -183,13 +173,13 @@ export interface PlannedRun { readonly iteration: number; } -/** Launches base and branch once per iteration, swapping which goes first on odd iterations. */ +/** Launches base and branch once per iteration, swapping which goes first on every other measured iteration. */ export function buildRunPlan(runs: number, warmup: number): ReadonlyArray { const plan: Array = []; for (let iteration = 0; iteration < runs + warmup; iteration++) { const isWarmup = iteration < warmup; const order: ReadonlyArray = - iteration % 2 === 0 ? ["base", "branch"] : ["branch", "base"]; + Math.abs(iteration - warmup) % 2 === 0 ? ["base", "branch"] : ["branch", "base"]; for (const build of order) plan.push({ build, warmup: isWarmup, iteration }); } return plan; @@ -319,7 +309,11 @@ const prepareLaunch = (options: BenchOptions, workload: Workload, binary: string return { projectDir, env }; }); -const runWorkloadBench = (options: BenchOptions, workload: Workload, profiled: Ref.Ref) => +export const runWorkloadBench = ( + options: BenchOptions, + workload: Workload, + profiled: Ref.Ref, +) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -479,8 +473,6 @@ function formatReport(report: BenchReport): string { return lines.join("\n"); } -const encodeReport = Schema.encodeEffect(Schema.fromJsonString(BenchReport, { space: 2 })); - const fileSize = (binary: string) => FileSystem.FileSystem.use((fs) => fs.stat(binary)).pipe( Effect.map((info) => Number(info.size)), @@ -540,10 +532,10 @@ const main = Effect.gen(function* () { if (options.out !== undefined) { const fs = yield* FileSystem.FileSystem; - yield* fs.writeFileString(options.out, yield* encodeReport(report)); + yield* fs.writeFileString(options.out, yield* encodeBenchReport(report)); } - yield* Console.log(options.json ? yield* encodeReport(report) : formatReport(report)); + yield* Console.log(options.json ? yield* encodeBenchReport(report) : formatReport(report)); }); if (import.meta.main) { diff --git a/apps/cli/scripts/bench-cli.unit.test.ts b/apps/cli/scripts/bench-cli.unit.test.ts index 903031d96b..b35c4b7e3b 100644 --- a/apps/cli/scripts/bench-cli.unit.test.ts +++ b/apps/cli/scripts/bench-cli.unit.test.ts @@ -79,12 +79,12 @@ describe("buildRunPlan", () => { const plan = buildRunPlan(2, 1); expect(plan.map((run) => [run.build, run.warmup])).toEqual([ - ["base", true], ["branch", true], - ["branch", false], - ["base", false], + ["base", true], ["base", false], ["branch", false], + ["branch", false], + ["base", false], ]); }); }); diff --git a/apps/cli/scripts/bench-report.ts b/apps/cli/scripts/bench-report.ts index ad99ab2ecd..6ab3135a81 100644 --- a/apps/cli/scripts/bench-report.ts +++ b/apps/cli/scripts/bench-report.ts @@ -197,6 +197,7 @@ export function classifyHttpSpans( apiUrl: string, ): HttpCallSummary { const url = new URL(apiUrl); + const apiHost = url.hostname.replace(/^\[|\]$/gu, ""); const apiPort = url.port === "" ? (url.protocol === "https:" ? 443 : 80) : Number(url.port); let apiCalls = 0; let externalCalls = 0; @@ -205,7 +206,7 @@ export function classifyHttpSpans( for (const span of spans) { const http = span.http; if (http === undefined) continue; - if (http.address !== url.hostname || (http.port !== undefined && http.port !== apiPort)) { + if (http.address !== apiHost || (http.port !== undefined && http.port !== apiPort)) { externalCalls += 1; continue; } @@ -263,7 +264,9 @@ export const BenchReport = Schema.Struct({ }); export type BenchReport = typeof BenchReport.Type; -export const decodeBenchReport = Schema.decodeUnknownEffect(Schema.fromJsonString(BenchReport)); +const BenchReportJson = Schema.fromJsonString(BenchReport, { space: 2 }); +export const decodeBenchReport = Schema.decodeUnknownEffect(BenchReportJson); +export const encodeBenchReport = Schema.encodeEffect(BenchReportJson); /* Analysis */ @@ -440,7 +443,8 @@ export type FlagKind = | "trace-incomplete" | "non-zero-exit" | "exit-codes-differ" - | "traced-exit-differs"; + | "traced-exit-differs" + | "insufficient-samples"; export interface Flag { readonly workload: string; @@ -461,6 +465,12 @@ export function collectFlags(analysis: WorkloadAnalysis): ReadonlyArray { if (baseCalls !== undefined && baseCalls > 0 && branchCalls === 0) { add("api-calls-dropped", `base makes ${baseCalls} API calls, branch makes 0`); } + if (Math.min(analysis.base.samples, analysis.branch.samples) < VERDICT_MIN_SAMPLES) { + add( + "insufficient-samples", + `fewer than ${VERDICT_MIN_SAMPLES} measured runs, so no timing verdict`, + ); + } if (analysis.exitCodesDiffer) { add("exit-codes-differ", "exit codes differ between builds, so timings are not comparable"); } diff --git a/apps/cli/scripts/bench-report.unit.test.ts b/apps/cli/scripts/bench-report.unit.test.ts index 44dcaa5a13..fb77db1a69 100644 --- a/apps/cli/scripts/bench-report.unit.test.ts +++ b/apps/cli/scripts/bench-report.unit.test.ts @@ -5,6 +5,8 @@ import { analyzeWorkload, classifyHttpSpans, collectFlags, + decodeBenchReport, + encodeBenchReport, compareSpans, deltaOf, exitCodesDiffer, @@ -13,6 +15,7 @@ import { summarizeTimings, timingVerdict, validateWorkloads, + type BenchReport, type RunSample, type SpanRunSummary, type WorkloadReport, @@ -117,7 +120,7 @@ describe("compareSpans", () => { }); describe("workload manifest", () => { - const entry = { id: "w", args: ["init"], cwdSetup: "none" }; + const entry = { id: "w", args: ["init"], cwdSetup: "none" } as const; it.effect("decodes entries and rejects an unknown cwdSetup", () => Effect.gen(function* () { @@ -149,7 +152,7 @@ describe("workload manifest", () => { 'duplicate workload id "b"', ]); expect( - validateWorkloads([{ ...entry, scenario: "s" }] as never, { apiUrl: "http://127.0.0.1:1" }), + validateWorkloads([{ ...entry, scenario: "s" }], { apiUrl: "http://127.0.0.1:1" }), ).toEqual([]); }); }); @@ -357,3 +360,84 @@ describe("analyzeWorkload and collectFlags", () => { expect(same.phaseRows).toEqual([]); }); }); + +describe("classifyHttpSpans host matching", () => { + it("matches a bracketed IPv6 API URL against the unbracketed span address", () => { + const summary = classifyHttpSpans( + [httpSpan("GET", "/v1/x", 4010, 200, "::1")], + "http://[::1]:4010", + ); + + expect(summary.apiCalls).toBe(1); + expect(summary.externalCalls).toBe(0); + }); + + it("counts a call to another host as external even when its port is unknown", () => { + const other: ReportSpan = { + ...span("GET", 0, 1), + http: { method: "GET", path: "/", address: "api.example.com", port: undefined, status: 200 }, + }; + + expect(classifyHttpSpans([other], "http://127.0.0.1:4010").externalCalls).toBe(1); + }); +}); + +describe("more flags", () => { + it("explains a missing verdict when there are too few runs", () => { + const few = times(3, () => sample(50, { calls: 1 })); + + const flags = collectFlags(analyzeWorkload(workload(few, few), true)); + + expect(flags.map((flag) => flag.kind)).toEqual(["insufficient-samples"]); + }); + + it("flags a traced launch that exits differently from the timed launches", () => { + const traced = sample(50, { calls: 1 }); + const odd: RunSample = { ...traced, trace: { ...traced.trace!, exitCode: 1 } }; + + const flags = collectFlags( + analyzeWorkload( + workload( + times(5, () => sample(50, { calls: 1 })), + times(5, () => odd), + ), + true, + ), + ); + + expect(flags.map((flag) => flag.kind)).toContain("traced-exit-differs"); + }); +}); + +describe("report JSON", () => { + const minimal: BenchReport = { + base: "/b", + branch: "/c", + runs: 5, + warmup: 1, + trace: false, + workloads: [{ id: "w", args: ["x"], base: [{ wallMs: 1, exitCode: 0 }], branch: [] }], + }; + const full: BenchReport = { + ...minimal, + baseSizeBytes: 10, + branchSizeBytes: 11, + trace: true, + apiUrl: "http://127.0.0.1:1", + cpuProfilePath: "/p", + workloads: [ + { ...minimal.workloads[0]!, base: [sample(1, { calls: 1, endpoints: { "GET /a": 1 } })] }, + ], + }; + + it.effect.each([ + { name: "omitted optional fields", report: minimal }, + { name: "every field", report: full }, + ])("round-trips a report with $name", ({ report }) => + Effect.gen(function* () { + const decoded = yield* decodeBenchReport(yield* encodeBenchReport(report)); + + expect(decoded).toEqual(report); + }), + ); +}); diff --git a/apps/cli/scripts/bench-summary.ts b/apps/cli/scripts/bench-summary.ts index 739850dc40..02a0c51bff 100644 --- a/apps/cli/scripts/bench-summary.ts +++ b/apps/cli/scripts/bench-summary.ts @@ -5,6 +5,7 @@ * `GITHUB_STEP_SUMMARY` when set, otherwise to stdout. */ import process from "node:process"; +import { parseArgs } from "node:util"; import { BunRuntime, BunServices } from "@effect/platform-bun"; import { Config, Console, Effect, FileSystem, Option, Stdio } from "effect"; import { @@ -25,17 +26,16 @@ export interface SummaryOptions { /** Parses argv; throws on an unknown `--base-build` or a missing report for a successful base. */ export function parseSummaryArgs(argv: ReadonlyArray): SummaryOptions { - const valueOf = (flag: string) => { - const index = argv.indexOf(flag); - return index === -1 ? undefined : argv[index + 1]; - }; - const baseBuild = valueOf("--base-build") ?? "ok"; + const { values } = parseArgs({ + args: [...argv], + options: { report: { type: "string" }, "base-build": { type: "string", default: "ok" } }, + }); + const baseBuild = values["base-build"]; if (baseBuild !== "ok" && baseBuild !== "failed") { throw new Error('--base-build must be "ok" or "failed"'); } - const report = valueOf("--report"); - if (report === undefined && baseBuild === "ok") throw new Error(USAGE); - return { report, baseBuild }; + if (values.report === undefined && baseBuild === "ok") throw new Error(USAGE); + return { report: values.report, baseBuild }; } const BASE_FAILED = "base build failed, no comparison"; @@ -44,11 +44,13 @@ const sign = (value: number) => (value >= 0 ? "+" : ""); const ms = (value: number) => `${value} ms`; const range = (build: WorkloadAnalysis["base"]) => `${build.timing.min}-${build.timing.max} ms`; +const cell = (value: string) => value.replaceAll("|", "\\|"); + function table(header: ReadonlyArray, rows: ReadonlyArray>): string { return [ `| ${header.join(" | ")} |`, `| ${header.map(() => "---").join(" | ")} |`, - ...rows.map((row) => `| ${row.join(" | ")} |`), + ...rows.map((row) => `| ${row.map(cell).join(" | ")} |`), ].join("\n"); } @@ -103,7 +105,11 @@ function phaseSection(analyses: ReadonlyArray): string { ]; }), ); - if (rows.length === 0) return "No workload moved beyond noise, so there are no phase deltas."; + if (rows.length === 0) { + return analyses.some((analysis) => analysis.verdict === "not comparable") + ? "No phase deltas: no workload had a timing verdict, and some were not comparable." + : "No workload moved beyond noise, so there are no phase deltas."; + } return table(["Workload", "Span", "Change", "Base -> branch"], rows); } diff --git a/apps/cli/scripts/bench-summary.unit.test.ts b/apps/cli/scripts/bench-summary.unit.test.ts index 642d8c8660..77aa7871dd 100644 --- a/apps/cli/scripts/bench-summary.unit.test.ts +++ b/apps/cli/scripts/bench-summary.unit.test.ts @@ -71,7 +71,33 @@ describe("renderSummary", () => { }); }); +describe("renderSummary without API counts", () => { + it("says API calls were not counted and keeps table cells intact when an id contains a pipe", () => { + const noTrace = (wallMs: number): RunSample => ({ wallMs, exitCode: 0 }); + const markdown = renderSummary({ + ...report([ + { + id: "a|b", + args: ["x"], + base: runs(5, (i) => noTrace(30 + i)), + branch: runs(5, (i) => noTrace(30 + i)), + }, + ]), + trace: false, + }); + + expect(markdown).toContain("API calls were not counted."); + expect(markdown).toContain("a\\|b"); + }); +}); + describe("parseSummaryArgs", () => { + it("rejects an unknown base build state", () => { + expect(() => parseSummaryArgs(["--report", "r.json", "--base-build", "maybe"])).toThrow( + "--base-build", + ); + }); + it("needs a report unless the base build failed", () => { expect(() => parseSummaryArgs([])).toThrow(); expect(parseSummaryArgs(["--base-build", "failed"])).toEqual({ diff --git a/apps/cli/scripts/replay-control.integration.test.ts b/apps/cli/scripts/replay-control.integration.test.ts index 2ea262acae..ee393ead1e 100644 --- a/apps/cli/scripts/replay-control.integration.test.ts +++ b/apps/cli/scripts/replay-control.integration.test.ts @@ -3,7 +3,7 @@ import { Effect } from "effect"; import { FetchHttpClient } from "effect/http"; import * as HttpClient from "effect/http/HttpClient"; import { startReplayServer } from "../../cli-e2e/src/server/replay-server.ts"; -import { controlPlan, readRequestLogCount, resetReplay } from "./replay-control.ts"; +import { readRequestLogCount, resetReplay } from "./replay-control.ts"; const fixturesDir = new URL("../../cli-e2e/fixtures", import.meta.url).pathname; const SCENARIO = "projects-list-renders-project-list"; @@ -16,20 +16,6 @@ const replayServer = Effect.acquireRelease( const callApi = (url: string, path: string) => HttpClient.HttpClient.use((client) => client.get(new URL(path, url).toString())); -describe("controlPlan", () => { - it("always clears the request log and scenario before selecting a scenario", () => { - expect(controlPlan(undefined).map((call) => `${call.method} ${call.path}`)).toEqual([ - "DELETE /_ctrl/requests", - "DELETE /_ctrl/scenario", - ]); - expect(controlPlan("s").at(-1)).toEqual({ - method: "POST", - path: "/_ctrl/scenario", - body: { name: "s" }, - }); - }); -}); - describe("replay control against the replay server", () => { it.effect("serves a consumed scenario again and starts each launch with an empty log", () => Effect.scoped( From 2e6af683671e8ffb99a45e6598274bd0215a2256 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 15:04:02 +0530 Subject: [PATCH 04/15] ci: add a report-only CLI benchmark workflow and docs Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/cli-bench.yml | 126 ++++++++++++++++++++++++++++ apps/cli/docs/tracing-monitoring.md | 117 +++++++++++++++++++++++++- 2 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/cli-bench.yml diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml new file mode 100644 index 0000000000..a8900addb5 --- /dev/null +++ b/.github/workflows/cli-bench.yml @@ -0,0 +1,126 @@ +name: CLI benchmark + +# Report-only comparison of the pull request's CLI build with a build of its merge base. Every +# workload runs against the replay server, so no Docker, database, or network is involved. The +# job never blocks a pull request and is not a required check. +on: + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + branches: + - develop + - next + paths: + - "apps/cli/**" + - "apps/cli-e2e/**" + - "packages/**" + - "pnpm-lock.yaml" + - "turbo.json" + - ".github/workflows/cli-bench.yml" + +permissions: + contents: read + +concurrency: + group: cli-bench-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + bench: + if: github.event.pull_request.draft == false + name: Benchmark CLI against merge base + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + steps: + - name: Prepare benchmark directory + run: | + mkdir -p "$RUNNER_TEMP/bench" + echo "BENCH_DIR=$RUNNER_TEMP/bench" >> "$GITHUB_ENV" + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Build branch binary + id: branch_build + run: | + pnpm --filter supabase run build:binary + mkdir -p "$BENCH_DIR/branch" + cp apps/cli/dist/supabase "$BENCH_DIR/branch/supabase" + + - name: Build merge-base binary + id: base_build + continue-on-error: true + env: + BASE_REF: ${{ github.base_ref }} + run: | + merge_base="$(git merge-base HEAD "origin/$BASE_REF")" + echo "Merge base: $merge_base" + git worktree add "$RUNNER_TEMP/base-src" "$merge_base" + cd "$RUNNER_TEMP/base-src" + pnpm install --frozen-lockfile + pnpm --filter supabase run build:binary + mkdir -p "$BENCH_DIR/base" + cp apps/cli/dist/supabase "$BENCH_DIR/base/supabase" + + - name: Start replay server + id: replay + if: steps.base_build.outcome == 'success' + run: | + bun apps/cli-e2e/scripts/replay-serve.ts > "$BENCH_DIR/replay.log" 2>&1 & + echo "REPLAY_PID=$!" >> "$GITHUB_ENV" + for _ in $(seq 1 60); do + url="$(grep -o 'http://[^"]*' "$BENCH_DIR/replay.log" | head -n 1 || true)" + if [ -n "$url" ]; then + echo "REPLAY_URL=$url" >> "$GITHUB_ENV" + exit 0 + fi + sleep 0.5 + done + echo "replay server did not report a URL" >&2 + cat "$BENCH_DIR/replay.log" >&2 + exit 1 + + - name: Run benchmark + id: bench + if: steps.base_build.outcome == 'success' + shell: bash + run: | + bun apps/cli/scripts/bench-cli.ts \ + --base "$BENCH_DIR/base/supabase" \ + --branch "$BENCH_DIR/branch/supabase" \ + --workloads apps/cli/bench/workloads.pr.json \ + --api-url "$REPLAY_URL" \ + --trace \ + --out "$BENCH_DIR/report.json" | tee "$BENCH_DIR/report.txt" + + - name: Stop replay server + if: always() && env.REPLAY_PID != '' + run: kill "$REPLAY_PID" || true + + - name: Write job summary + if: always() && steps.branch_build.outcome == 'success' + env: + BASE_BUILD: ${{ steps.base_build.outcome == 'success' && 'ok' || 'failed' }} + run: bun apps/cli/scripts/bench-summary.ts --report "$BENCH_DIR/report.json" --base-build "$BASE_BUILD" + + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-report + path: | + ${{ runner.temp }}/bench/report.json + ${{ runner.temp }}/bench/report.txt + retention-days: 14 + if-no-files-found: ignore diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index a9cf43b439..ffb06b5768 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -65,7 +65,7 @@ milliseconds, plus the branch-minus-base delta in ms and percent. With `--trace` launch is followed by an untimed launch with `SUPABASE_TRACE_FILE` set, so tracing never inflates the wall times, and the report adds, per span name, the median total duration and count for each build, sorted by the largest absolute delta; a span name present in only one build is flagged -rather than compared. `--command` splits on whitespace and does not support shell quoting; repeat the +rather than compared. `--command` splits on whitespace and does not support shell quoting (use `--workloads` for quoted arguments); repeat the flag for more than one command, and use `--cwd-setup init` when a command, such as `status`, needs to run inside an initialized project. Pass `--json` for a machine-readable report, `--out ` to also write the raw per-run samples, and `--cpu-prof` to capture a Bun CPU profile of one untimed @@ -84,6 +84,121 @@ rm -f /tmp/cli-base/mise.toml /tmp/cli-base/mise.lock # if mise refuses the untr (cd /tmp/cli-base && pnpm install && pnpm exec turbo run supabase#build) ``` +### Pull request benchmark + +`.github/workflows/cli-bench.yml` runs on every ready pull request into `develop` or `next`. It builds +the pull request and its merge base with `pnpm --filter supabase run build:binary`, runs both +through `bench-cli.ts` against a local replay server, and writes a job summary with `bench-summary.ts`. +The job is report only: it never blocks a pull request, and no benchmark problem fails it. The +workloads use no Docker, database, or network, and no humans are involved. + +To run it locally with two binaries (for example builds from two commits): + +```sh +pnpm --filter @supabase/cli-e2e run replay:serve # prints {"url":"http://127.0.0.1:"} +bun apps/cli/scripts/bench-cli.ts --base --branch \ + --workloads apps/cli/bench/workloads.pr.json --api-url --trace --out /tmp/report.json +bun apps/cli/scripts/bench-summary.ts --report /tmp/report.json +``` + +`--workloads` replaces repeated `--command` flags. `apps/cli/bench/workloads.pr.json` lists entries +with an `id`, an `args` array (so quoting works), a `cwdSetup` of `none`, `init`, or `linked`, and +optionally a replay `scenario` and a `projectRef`. `linked` runs `init`, then writes the files a +linked project has (`supabase/.temp/project-ref` and a stub `pooler-url`), so `--linked` commands +resolve a project without a database. Setup runs outside the timed region. + +`--api-url` (which requires `--trace`) points every launch at the replay server by writing a profile +file into the launch's fresh `SUPABASE_HOME` and setting `SUPABASE_PROFILE`, plus a dummy access +token. The profile is the CLI's only API redirect: `SUPABASE_API_URL` does not change where the +Management API client sends requests, so it is never set. Every launch gets the profile, including +`--version` and `init`, so an unexpected new API call reaches the server and is counted. Before each +launch the harness clears the server's request log and scenario, then selects the workload's +scenario, so scenarios that are used up as they are served start fresh every time. A failing control +call aborts the benchmark. + +API calls are counted from the traced launch's HTTP client spans: spans to the `--api-url` host and +port are API calls, and spans to any other host are `externalCalls`. The count is cross-checked +against the replay server's request log for the same launch. The `--out` file stores raw per-run +data; the summary derives medians, flags, and verdicts from it, so thresholds can change without +re-running. + +The summary reports: + +- Flags: an API call count that goes from 0 to more than 0 (for example `migration new` starting to + call the API) or from more than 0 to 0, calls to hosts other than the replay server, counts that + differ between runs of one build, spans that disagree with the request log, 4xx/5xx replay + responses (the build did not follow the recorded scenario), non-zero or differing exit codes, and + missing traces. Other count changes, such as 1 to 2 calls, appear in the API calls table with the + changed endpoints but are not flagged. +- Wall time: base and branch medians and ranges. A verdict of slower or faster needs the branch + median outside the base min-max range, a change of at least 5 percent and 5 ms, and at least 5 + measured runs; otherwise it reads "within noise", or "not comparable" when a build failed. + Thresholds are named constants in `bench-report.ts`. +- API calls per workload, base and branch. +- Phase deltas: the three largest span deltas for each workload with a slower or faster verdict. +- Binary size of `dist/supabase`. + +The baseline API calls per workload on both `develop` and `next` builds (a project-scoped command +also refreshes the linked-project cache, so it makes two calls): + +| Workload | API calls | +| ------------------------------------------ | ---------------------------------------------------------------------- | +| `version`, `help`, `init`, `migration-new` | 0 | +| `projects-list` | 1: `GET /v1/projects` | +| `db-query` | 2: `POST /v1/projects/{ref}/database/query`, `GET /v1/projects/{ref}` | +| `gen-types` | 2: `GET /v1/projects/{ref}/types/typescript`, `GET /v1/projects/{ref}` | +| `functions-list` | 2: `GET /v1/projects/{ref}/functions`, `GET /v1/projects/{ref}` | + +`SUPABASE_NO_KEYRING=1` disables the keyring (`CommandCredentials.load` reports +`keyring.enabled=false`); the value `true` is not recognized. Counts come from the untimed traced +launch, so a launch that behaves differently when timed shows up only as an exit-code difference. + +#### How the workloads were chosen + +The list comes from `cli_command_executed` usage in PostHog over the last 30 days, for stable CLI +versions only (prereleases excluded). Commands are ranked by distinct `device_id`, then by run +count; the p50 and p90 duration of successful runs, the CI share, and the agent share are context. +Only commands that run with no Docker, no database, no real network, and no prompts are kept, and +every Go-proxied command is dropped. One command is taken from each class (startup, local files, +API read without a ref, API calls with a ref, API read of a large response) rather than the top N +overall, capped at 8 workloads to fit the time budget. Refresh the list each quarter, or after the +v3 release with the version regex updated: + +```sql +SELECT properties.command cmd, uniq(properties.device_id) users, count() runs, + round(100*countIf(properties.exit_code != 0)/count(),1) err_pct, + round(quantileIf(0.5)(properties.duration_ms, properties.exit_code = 0)) p50_ms, + round(quantileIf(0.9)(properties.duration_ms, properties.exit_code = 0)) p90_ms, + round(100*countIf(properties.is_ci = true)/count(),1) ci_pct, + round(100*countIf(properties.is_agent = true)/count(),1) agent_pct +FROM events WHERE event='cli_command_executed' AND timestamp >= now() - INTERVAL 30 DAY + AND match(properties.cli_version,'^2\\.(119|120|121|122)\\.[0-9]+$') +GROUP BY cmd ORDER BY users DESC LIMIT 40 +``` + +Caveats: + +- Flags differ between branches. The manifest uses the `develop` form (`--linked`, `--project-ref`, + `--project-id`). `next` accepts these as deprecated aliases until v4 and prints a warning in both + builds, but `next` requires `db query --remote --read-only` and sends `read_only: true`, which + the `develop` replay scenario does not match, so the `db-query` workload exits non-zero on `next` + with the `develop` manifest. After the v3 cut, switch the manifest to `--remote[=]` and the + `db-query-remote-*` scenario. +- Telemetry records the command path but no flags or arguments, so `SELECT 1`, `bench`, and the + TypeScript target only approximate real use. +- `device_id` counts machines, not people, short-lived CI runners inflate it, and a large share of + runs come from agents. Telemetry durations include network, Docker, and wait time, so they guided + the choice but are not what the benchmark measures. +- Excluded high-usage commands: `start`, `status`, `stop`, `db start`, `db reset`, and `test db` + need Docker; `db push`, `db lint`, `db dump`, `db diff`, `migration list`, `migration up`, + `migration repair`, and `db schema declarative sync` need a database (a nightly tier); + `link` probes `https://.`, which never reaches the replay server; + `functions deploy` uses Docker by default and its API path has no success scenario; `login` is + interactive unless a token is passed; `functions serve` needs Docker. +- The first launch of a process is slower (credential loading); the warm-up runs absorb it. +- Launches write a profile file, so numbers from runs with `--api-url` are not comparable with runs + without it. + ## Grafana, Tempo, or Jaeger Run a local OpenTelemetry stack, for example Grafana's all-in-one image: From 1f86adb892cbe80dfd990d280dece92d06f45c8c Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 15:06:47 +0530 Subject: [PATCH 05/15] ci: keep the CLI benchmark report-only on infrastructure failures Binds the replay server to loopback, routes the merge-base install through the dependency firewall, and renders a notice when no report was produced. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/cli-bench.yml | 6 ++++++ apps/cli-e2e/scripts/replay-serve.ts | 1 + apps/cli-e2e/src/server/replay-server.ts | 3 +++ apps/cli/docs/tracing-monitoring.md | 13 +++++++------ apps/cli/scripts/bench-summary.ts | 18 +++++++++++++----- apps/cli/scripts/bench-summary.unit.test.ts | 4 ++++ 6 files changed, 34 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml index a8900addb5..1ca6439e63 100644 --- a/.github/workflows/cli-bench.yml +++ b/.github/workflows/cli-bench.yml @@ -68,6 +68,10 @@ jobs: echo "Merge base: $merge_base" git worktree add "$RUNNER_TEMP/base-src" "$merge_base" cd "$RUNNER_TEMP/base-src" + # Setup writes this file only when the Dependency Firewall token is available. + if [ -f "$RUNNER_TEMP/dependency-firewall.npmrc" ]; then + export NPM_CONFIG_USERCONFIG="$RUNNER_TEMP/dependency-firewall.npmrc" + fi pnpm install --frozen-lockfile pnpm --filter supabase run build:binary mkdir -p "$BENCH_DIR/base" @@ -75,6 +79,7 @@ jobs: - name: Start replay server id: replay + continue-on-error: true if: steps.base_build.outcome == 'success' run: | bun apps/cli-e2e/scripts/replay-serve.ts > "$BENCH_DIR/replay.log" 2>&1 & @@ -93,6 +98,7 @@ jobs: - name: Run benchmark id: bench + continue-on-error: true if: steps.base_build.outcome == 'success' shell: bash run: | diff --git a/apps/cli-e2e/scripts/replay-serve.ts b/apps/cli-e2e/scripts/replay-serve.ts index 3d65892662..f1ea4a6e88 100644 --- a/apps/cli-e2e/scripts/replay-serve.ts +++ b/apps/cli-e2e/scripts/replay-serve.ts @@ -19,6 +19,7 @@ if (process.env["RECORD"] === "true") { const server = await startReplayServer({ fixturesDir: FIXTURES_DIR, + hostname: "127.0.0.1", port: parsePort(process.argv.slice(2)), }); diff --git a/apps/cli-e2e/src/server/replay-server.ts b/apps/cli-e2e/src/server/replay-server.ts index 8706bc8736..4d85198cac 100644 --- a/apps/cli-e2e/src/server/replay-server.ts +++ b/apps/cli-e2e/src/server/replay-server.ts @@ -153,6 +153,8 @@ interface ReplayServerOptions { fixturesDir: string; /** Port to listen on (0 = random). */ port?: number; + /** Interface to bind; Bun binds all interfaces when omitted. */ + hostname?: string; /** Optional Postgres mock server to control via /_ctrl/pg-* endpoints. */ pgMock?: PgMockHandle; } @@ -193,6 +195,7 @@ export async function startReplayServer(options: ReplayServerOptions): Promise): SummaryOptions { } const BASE_FAILED = "base build failed, no comparison"; +const NO_REPORT = "the benchmark did not produce a report; see the Run benchmark step log"; const sign = (value: number) => (value >= 0 ? "+" : ""); const ms = (value: number) => `${value} ms`; @@ -124,9 +125,12 @@ function sizeSection(report: BenchReport): string { } /** The Markdown summary, or the base-build-failed notice when there is no report to compare. */ -export function renderSummary(report: BenchReport | undefined): string { +export function renderSummary( + report: BenchReport | undefined, + notice: string = BASE_FAILED, +): string { const title = "## CLI benchmark (report only)"; - if (report === undefined) return `${title}\n\n${BASE_FAILED}\n`; + if (report === undefined) return `${title}\n\n${notice}\n`; const analyses = report.workloads.map((workload) => analyzeWorkload(workload, report.trace)); const flags = analyses.flatMap(collectFlags); return ( @@ -168,11 +172,15 @@ const main = Effect.gen(function* () { if (options === undefined) return; const fs = yield* FileSystem.FileSystem; + const reportFile = options.baseBuild === "failed" ? undefined : options.report; const report = - options.baseBuild === "failed" || options.report === undefined + reportFile === undefined ? undefined - : yield* decodeBenchReport(yield* fs.readFileString(options.report)); - const markdown = renderSummary(report); + : yield* fs.readFileString(reportFile).pipe( + Effect.flatMap(decodeBenchReport), + Effect.orElseSucceed(() => undefined), + ); + const markdown = renderSummary(report, reportFile === undefined ? BASE_FAILED : NO_REPORT); const summaryFile = yield* Config.option(Config.String("GITHUB_STEP_SUMMARY")); if (Option.isSome(summaryFile) && summaryFile.value !== "") { diff --git a/apps/cli/scripts/bench-summary.unit.test.ts b/apps/cli/scripts/bench-summary.unit.test.ts index 77aa7871dd..3894d0ee61 100644 --- a/apps/cli/scripts/bench-summary.unit.test.ts +++ b/apps/cli/scripts/bench-summary.unit.test.ts @@ -66,6 +66,10 @@ describe("renderSummary", () => { expect(markdown).toContain("no phase deltas"); }); + it("reports a missing report with the notice it is given", () => { + expect(renderSummary(undefined, "no report here")).toContain("no report here"); + }); + it("reports a failed base build without a comparison", () => { expect(renderSummary(undefined)).toContain("base build failed, no comparison"); }); From a93a84699fbb8981e700d3d46177e04b3b868e2b Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 15:29:30 +0530 Subject: [PATCH 06/15] chore(cli): drop the db query workload from the PR benchmark until the v3 cut Co-Authored-By: Claude Opus 5.5 --- apps/cli/bench/workloads.pr.json | 7 ------ apps/cli/docs/tracing-monitoring.md | 23 +++++++++---------- apps/cli/scripts/bench-cli.ts | 11 --------- apps/cli/scripts/bench-report.ts | 6 +---- apps/cli/scripts/bench-report.unit.test.ts | 4 +--- apps/cli/scripts/bench-workloads.unit.test.ts | 2 +- 6 files changed, 14 insertions(+), 39 deletions(-) diff --git a/apps/cli/bench/workloads.pr.json b/apps/cli/bench/workloads.pr.json index 68d9c733ec..d599457f3d 100644 --- a/apps/cli/bench/workloads.pr.json +++ b/apps/cli/bench/workloads.pr.json @@ -4,13 +4,6 @@ { "id": "help", "args": ["--help"], "cwdSetup": "none" }, { "id": "init", "args": ["init"], "cwdSetup": "none" }, { "id": "migration-new", "args": ["migration", "new", "bench"], "cwdSetup": "init" }, - { - "id": "db-query", - "args": ["db", "query", "--linked", "SELECT 1"], - "cwdSetup": "linked", - "projectRef": "replayfixtureprojref", - "scenario": "db-query-linked-returns-select-1-result-in-table-format" - }, { "id": "gen-types", "args": ["gen", "types", "--lang", "typescript", "--project-id", "replayfixtureprojref"], diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index 1ad06a4de3..a38866d637 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -103,10 +103,9 @@ bun apps/cli/scripts/bench-summary.ts --report /tmp/report.json ``` `--workloads` replaces repeated `--command` flags. `apps/cli/bench/workloads.pr.json` lists entries -with an `id`, an `args` array (so quoting works), a `cwdSetup` of `none`, `init`, or `linked`, and -optionally a replay `scenario` and a `projectRef`. `linked` runs `init`, then writes the files a -linked project has (`supabase/.temp/project-ref` and a stub `pooler-url`), so `--linked` commands -resolve a project without a database. Setup runs outside the timed region. +with an `id`, an `args` array (so quoting works), a `cwdSetup` of `none` or `init`, and optionally +a replay `scenario`. `init` runs `supabase init` in the launch's project directory first. Setup runs +outside the timed region. `--api-url` (which requires `--trace`) points every launch at the replay server by writing a profile file into the launch's fresh `SUPABASE_HOME` and setting `SUPABASE_PROFILE`, plus a dummy access @@ -146,7 +145,6 @@ also refreshes the linked-project cache, so it makes two calls): | ------------------------------------------ | ---------------------------------------------------------------------- | | `version`, `help`, `init`, `migration-new` | 0 | | `projects-list` | 1: `GET /v1/projects` | -| `db-query` | 2: `POST /v1/projects/{ref}/database/query`, `GET /v1/projects/{ref}` | | `gen-types` | 2: `GET /v1/projects/{ref}/types/typescript`, `GET /v1/projects/{ref}` | | `functions-list` | 2: `GET /v1/projects/{ref}/functions`, `GET /v1/projects/{ref}` | @@ -162,7 +160,7 @@ count; the p50 and p90 duration of successful runs, the CI share, and the agent Only commands that run with no Docker, no database, no real network, and no prompts are kept, and every Go-proxied command is dropped. One command is taken from each class (startup, local files, API read without a ref, API calls with a ref, API read of a large response) rather than the top N -overall, capped at 8 workloads to fit the time budget. Refresh the list each quarter, or after the +overall, capped to fit the time budget. Refresh the list each quarter, or after the v3 release with the version regex updated: ```sql @@ -179,13 +177,14 @@ GROUP BY cmd ORDER BY users DESC LIMIT 40 Caveats: -- Flags differ between branches. The manifest uses the `develop` form (`--linked`, `--project-ref`, +- Flags differ between branches. The manifest uses the `develop` form (`--project-ref`, `--project-id`). `next` accepts these as deprecated aliases until v4 and prints a warning in both - builds, but `next` requires `db query --remote --read-only` and sends `read_only: true`, which - the `develop` replay scenario does not match, so the `db-query` workload exits non-zero on `next` - with the `develop` manifest. After the v3 cut, switch the manifest to `--remote[=]` and the - `db-query-remote-*` scenario. -- Telemetry records the command path but no flags or arguments, so `SELECT 1`, `bench`, and the + builds. After the v3 cut, switch the manifest to `--remote[=]`. +- `db query` is not benchmarked yet, although it is one of the most used commands. `next` requires + `db query --remote --read-only`, whose request the `develop` replay scenario does not match, so one + manifest cannot run it on both branches. Add it after the v3 cut with the `db-query-remote-*` + scenario. +- Telemetry records the command path but no flags or arguments, so `bench` and the TypeScript target only approximate real use. - `device_id` counts machines, not people, short-lived CI runners inflate it, and a large share of runs come from agents. Telemetry durations include network, Docker, and wait time, so they guided diff --git a/apps/cli/scripts/bench-cli.ts b/apps/cli/scripts/bench-cli.ts index 1227703f14..9c779da0bb 100644 --- a/apps/cli/scripts/bench-cli.ts +++ b/apps/cli/scripts/bench-cli.ts @@ -264,8 +264,6 @@ const baseLaunchEnv = ( SUPABASE_PROFILE: profilePath, SUPABASE_ACCESS_TOKEN: BENCH_ACCESS_TOKEN, SUPABASE_TELEMETRY_DISABLED: "1", - // With a pooler-url file present, a set password skips the interactive login-role flow. - SUPABASE_DB_PASSWORD: "bench-placeholder-password", }), }); @@ -297,15 +295,6 @@ const prepareLaunch = (options: BenchOptions, workload: Workload, binary: string ); } } - if (workload.cwdSetup === "linked" && workload.projectRef !== undefined) { - const tempDir = path.join(projectDir, "supabase", ".temp"); - yield* fs.makeDirectory(tempDir, { recursive: true }); - yield* fs.writeFileString(path.join(tempDir, "project-ref"), workload.projectRef); - yield* fs.writeFileString( - path.join(tempDir, "pooler-url"), - `postgresql://postgres.${workload.projectRef}:[YOUR-PASSWORD]@aws-0-eu-central-1.pooler.supabase.com:6543/postgres`, - ); - } return { projectDir, env }; }); diff --git a/apps/cli/scripts/bench-report.ts b/apps/cli/scripts/bench-report.ts index 6ab3135a81..4a6932ba18 100644 --- a/apps/cli/scripts/bench-report.ts +++ b/apps/cli/scripts/bench-report.ts @@ -145,9 +145,8 @@ export function exitCodesDiffer( const Workload = Schema.Struct({ id: Schema.NonEmptyString, args: Schema.Array(Schema.String), - cwdSetup: Schema.Literals(["none", "init", "linked"]), + cwdSetup: Schema.Literals(["none", "init"]), scenario: Schema.optionalKey(Schema.String), - projectRef: Schema.optionalKey(Schema.String), }); export type Workload = typeof Workload.Type; @@ -165,9 +164,6 @@ export function validateWorkloads( for (const workload of workloads) { if (seen.has(workload.id)) problems.push(`duplicate workload id "${workload.id}"`); seen.add(workload.id); - if (workload.cwdSetup === "linked" && workload.projectRef === undefined) { - problems.push(`workload "${workload.id}" uses cwdSetup "linked" and needs a projectRef`); - } if (workload.scenario !== undefined && context.apiUrl === undefined) { problems.push(`workload "${workload.id}" names a scenario and needs --api-url`); } diff --git a/apps/cli/scripts/bench-report.unit.test.ts b/apps/cli/scripts/bench-report.unit.test.ts index fb77db1a69..e220b57de5 100644 --- a/apps/cli/scripts/bench-report.unit.test.ts +++ b/apps/cli/scripts/bench-report.unit.test.ts @@ -136,10 +136,9 @@ describe("workload manifest", () => { }), ); - it("requires a projectRef for linked workloads, --api-url for scenarios, and unique ids", () => { + it("requires --api-url for scenarios and unique ids", () => { const problems = validateWorkloads( [ - { id: "a", args: [], cwdSetup: "linked" }, { id: "b", args: [], cwdSetup: "none", scenario: "s" }, { id: "b", args: [], cwdSetup: "none" }, ], @@ -147,7 +146,6 @@ describe("workload manifest", () => { ); expect(problems).toEqual([ - 'workload "a" uses cwdSetup "linked" and needs a projectRef', 'workload "b" names a scenario and needs --api-url', 'duplicate workload id "b"', ]); diff --git a/apps/cli/scripts/bench-workloads.unit.test.ts b/apps/cli/scripts/bench-workloads.unit.test.ts index 0f48264239..403ce9664f 100644 --- a/apps/cli/scripts/bench-workloads.unit.test.ts +++ b/apps/cli/scripts/bench-workloads.unit.test.ts @@ -31,7 +31,7 @@ describe("PR workload manifest", () => { const fixture = yield* fs.readFileString( `${scenariosDir}${workload.scenario}/interactions.json`, ); - const ref = workload.projectRef ?? workload.args.find((arg) => arg.startsWith("replay")); + const ref = workload.args.find((arg) => arg.startsWith("replay")); if (ref !== undefined) expect(fixture).toContain(`/v1/projects/${ref}`); } }).pipe(Effect.provide(BunServices.layer)), From f396769d540c3f6fba2571a8600b97ac461aa060 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 15:47:43 +0530 Subject: [PATCH 07/15] ci: report branch build and replay failures in the CLI benchmark summary Keep the job green when the branch build or the replay server fails, skip the dependent steps, and name the failed step in the summary. Set HUSKY=0 for the merge-base install. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/cli-bench.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml index 1ca6439e63..3f665cda1f 100644 --- a/.github/workflows/cli-bench.yml +++ b/.github/workflows/cli-bench.yml @@ -53,6 +53,7 @@ jobs: - name: Build branch binary id: branch_build + continue-on-error: true run: | pnpm --filter supabase run build:binary mkdir -p "$BENCH_DIR/branch" @@ -61,7 +62,9 @@ jobs: - name: Build merge-base binary id: base_build continue-on-error: true + if: steps.branch_build.outcome == 'success' env: + HUSKY: "0" BASE_REF: ${{ github.base_ref }} run: | merge_base="$(git merge-base HEAD "origin/$BASE_REF")" @@ -80,7 +83,7 @@ jobs: - name: Start replay server id: replay continue-on-error: true - if: steps.base_build.outcome == 'success' + if: steps.branch_build.outcome == 'success' && steps.base_build.outcome == 'success' run: | bun apps/cli-e2e/scripts/replay-serve.ts > "$BENCH_DIR/replay.log" 2>&1 & echo "REPLAY_PID=$!" >> "$GITHUB_ENV" @@ -99,7 +102,7 @@ jobs: - name: Run benchmark id: bench continue-on-error: true - if: steps.base_build.outcome == 'success' + if: steps.replay.outcome == 'success' shell: bash run: | bun apps/cli/scripts/bench-cli.ts \ @@ -115,10 +118,17 @@ jobs: run: kill "$REPLAY_PID" || true - name: Write job summary - if: always() && steps.branch_build.outcome == 'success' + if: always() env: + BRANCH_BUILD: ${{ steps.branch_build.outcome == 'success' && 'ok' || 'failed' }} BASE_BUILD: ${{ steps.base_build.outcome == 'success' && 'ok' || 'failed' }} - run: bun apps/cli/scripts/bench-summary.ts --report "$BENCH_DIR/report.json" --base-build "$BASE_BUILD" + REPLAY: ${{ steps.replay.outcome == 'success' && 'ok' || 'failed' }} + run: | + bun apps/cli/scripts/bench-summary.ts \ + --report "$BENCH_DIR/report.json" \ + --branch-build "$BRANCH_BUILD" \ + --base-build "$BASE_BUILD" \ + --replay "$REPLAY" - name: Upload report if: always() From cb9a04206809091e349375e5c72c9de410cbb5c7 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 15:47:43 +0530 Subject: [PATCH 08/15] chore(cli): fix benchmark report edge cases and summary outcomes Record the traced launch exit code independently of span decoding, skip endpoint changes when a build has no API measurement, separate absent from unreadable reports, fix the phase-summary wording, and raise a tagged workload validation error. Co-Authored-By: Claude Sonnet 5.5 --- apps/cli/docs/tracing-monitoring.md | 3 +- apps/cli/scripts/bench-cli.ts | 28 +++++- apps/cli/scripts/bench-cli.unit.test.ts | 19 ++++ apps/cli/scripts/bench-report.ts | 19 ++-- apps/cli/scripts/bench-report.unit.test.ts | 37 ++++++- .../scripts/bench-summary.integration.test.ts | 25 +++++ apps/cli/scripts/bench-summary.ts | 97 ++++++++++++++----- apps/cli/scripts/bench-summary.unit.test.ts | 94 +++++++++++++----- 8 files changed, 259 insertions(+), 63 deletions(-) create mode 100644 apps/cli/scripts/bench-summary.integration.test.ts diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index a38866d637..18e16fd9e8 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -90,7 +90,8 @@ rm -f /tmp/cli-base/mise.toml /tmp/cli-base/mise.lock # if mise refuses the untr the e2e replay fixtures. It builds the pull request and the base tip it merges into with `pnpm --filter supabase run build:binary`, runs both through `bench-cli.ts` against a local replay server, and writes a job summary with `bench-summary.ts`. The job is report only: it never blocks a pull request, and a benchmark problem is reported in the -summary instead of failing the job. The workloads use no Docker, database, or network, and no +summary instead of failing the job: a failed branch or base build or a replay server that does not +start replaces the comparison with a notice naming the step. The workloads use no Docker, database, or network, and no prompts. To run it locally with two binaries (for example builds from two commits): diff --git a/apps/cli/scripts/bench-cli.ts b/apps/cli/scripts/bench-cli.ts index 9c779da0bb..8ec2d862d1 100644 --- a/apps/cli/scripts/bench-cli.ts +++ b/apps/cli/scripts/bench-cli.ts @@ -10,7 +10,18 @@ import { tmpdir } from "node:os"; import process from "node:process"; import { parseArgs } from "node:util"; import { BunPath, BunRuntime, BunServices } from "@effect/platform-bun"; -import { Clock, Console, Effect, FileSystem, Layer, Option, Path, Ref, Stdio } from "effect"; +import { + Clock, + Console, + Effect, + FileSystem, + Layer, + Option, + Path, + Ref, + Schema, + Stdio, +} from "effect"; import { FetchHttpClient } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { @@ -185,6 +196,15 @@ export function buildRunPlan(runs: number, warmup: number): ReadonlyArray()( + "WorkloadValidationError", + { problems: Schema.Array(Schema.String) }, +) { + override get message(): string { + return this.problems.join("\n"); + } +} + /** The workloads for a run: the manifest entries, or one per `--command` with `--cwd-setup`. */ export const resolveWorkloads = Effect.fn("BenchCli.resolveWorkloads")(function* ( options: BenchOptions, @@ -201,7 +221,7 @@ export const resolveWorkloads = Effect.fn("BenchCli.resolveWorkloads")(function* return yield* parseWorkloadManifest(yield* fs.readFileString(options.workloadsFile!)); }); const problems = validateWorkloads(workloads, { apiUrl: options.apiUrl }); - if (problems.length > 0) return yield* Effect.fail(problems.join("\n")); + if (problems.length > 0) return yield* new WorkloadValidationError({ problems }); return workloads; }); @@ -350,6 +370,7 @@ export const runWorkloadBench = ( } let trace: RunSample["trace"]; + let tracedExitCode: number | undefined; if (traceFile !== undefined || bunOptions !== undefined) { const observed = yield* prepareLaunch(options, workload, binary); yield* resetServer; @@ -363,6 +384,7 @@ export const runWorkloadBench = ( ...(bunOptions === undefined ? {} : { BUN_OPTIONS: bunOptions }), }), }); + tracedExitCode = traced.exitCode; if (traceFile !== undefined) { const requestLogCount = options.apiUrl === undefined @@ -377,7 +399,6 @@ export const runWorkloadBench = ( ? undefined : classifyHttpSpans(spans, options.apiUrl); trace = { - exitCode: traced.exitCode, skippedLines, spans: summarizeSpanRun(spans), ...(http === undefined || requestLogCount === undefined @@ -391,6 +412,7 @@ export const runWorkloadBench = ( samples[planned.build].push({ wallMs: result.wallMs, exitCode: result.exitCode, + ...(tracedExitCode === undefined ? {} : { tracedExitCode }), ...(trace === undefined ? {} : { trace }), }); }), diff --git a/apps/cli/scripts/bench-cli.unit.test.ts b/apps/cli/scripts/bench-cli.unit.test.ts index b35c4b7e3b..8906f3a380 100644 --- a/apps/cli/scripts/bench-cli.unit.test.ts +++ b/apps/cli/scripts/bench-cli.unit.test.ts @@ -67,6 +67,25 @@ describe("resolveWorkloads", () => { ); }); +describe("resolveWorkloads validation", () => { + it.effect("fails with a WorkloadValidationError carrying the problems", () => + Effect.gen(function* () { + const options = parseBenchArgs([ + ...required, + "--command", + "migration new bench", + "--command", + "migration new bench", + ]); + + const error = yield* resolveWorkloads(options).pipe(Effect.flip); + + const problems = error._tag === "WorkloadValidationError" ? error.problems : undefined; + expect(problems).toEqual(['duplicate workload id "migration new bench"']); + }).pipe(Effect.provide(BunServices.layer)), + ); +}); + describe("wantsHelp", () => { it("treats --help as a flag only when it is not a --command value", () => { expect(wantsHelp(["--help"])).toBe(true); diff --git a/apps/cli/scripts/bench-report.ts b/apps/cli/scripts/bench-report.ts index 4a6932ba18..bd787c6537 100644 --- a/apps/cli/scripts/bench-report.ts +++ b/apps/cli/scripts/bench-report.ts @@ -225,7 +225,6 @@ const ApiSample = Schema.Struct({ }); const TraceSample = Schema.Struct({ - exitCode: Schema.Finite, skippedLines: Schema.Finite, spans: Schema.Record(Schema.String, SpanRunSummary), api: Schema.optionalKey(ApiSample), @@ -234,6 +233,8 @@ const TraceSample = Schema.Struct({ const RunSample = Schema.Struct({ wallMs: Schema.Finite, exitCode: Schema.Finite, + /** Exit code of the traced launch, recorded even when no spans could be decoded. */ + tracedExitCode: Schema.optionalKey(Schema.Finite), trace: Schema.optionalKey(TraceSample), }); export type RunSample = typeof RunSample.Type; @@ -344,7 +345,9 @@ function summarizeBuild(samples: ReadonlyArray, traced: boolean): Bui timing: summarizeTimings(samples.map((sample) => sample.wallMs)), samples: samples.length, exitCodes: samples.map((sample) => sample.exitCode), - tracedExitCodes: withTrace.map((trace) => trace.exitCode), + tracedExitCodes: samples.flatMap((sample) => + sample.tracedExitCode === undefined ? [] : [sample.tracedExitCode], + ), spans: traced ? aggregateSpanRuns(withTrace.map((trace) => trace.spans)) : undefined, api: summarizeApi(samples), traceMissingRuns: traced ? samples.length - withTrace.length : 0, @@ -385,16 +388,16 @@ function endpointChanges( base: BuildAnalysis, branch: BuildAnalysis, ): ReadonlyArray { - const keys = new Set([ - ...Object.keys(base.api?.endpoints ?? {}), - ...Object.keys(branch.api?.endpoints ?? {}), - ]); + if (base.api === undefined || branch.api === undefined) return []; + const baseEndpoints = base.api.endpoints; + const branchEndpoints = branch.api.endpoints; + const keys = new Set([...Object.keys(baseEndpoints), ...Object.keys(branchEndpoints)]); return [...keys] .sort() .map((endpoint) => ({ endpoint, - base: base.api?.endpoints[endpoint] ?? 0, - branch: branch.api?.endpoints[endpoint] ?? 0, + base: baseEndpoints[endpoint] ?? 0, + branch: branchEndpoints[endpoint] ?? 0, })) .filter((change) => change.base !== change.branch); } diff --git a/apps/cli/scripts/bench-report.unit.test.ts b/apps/cli/scripts/bench-report.unit.test.ts index e220b57de5..9f41a41d98 100644 --- a/apps/cli/scripts/bench-report.unit.test.ts +++ b/apps/cli/scripts/bench-report.unit.test.ts @@ -223,8 +223,8 @@ const sample = ( ): RunSample => ({ wallMs, exitCode, + tracedExitCode: exitCode, trace: { - exitCode, skippedLines: 0, spans: { "cli.run": { totalMs: wallMs / 2, count: 1 } }, ...(api === undefined @@ -391,7 +391,7 @@ describe("more flags", () => { it("flags a traced launch that exits differently from the timed launches", () => { const traced = sample(50, { calls: 1 }); - const odd: RunSample = { ...traced, trace: { ...traced.trace!, exitCode: 1 } }; + const odd: RunSample = { ...traced, tracedExitCode: 1 }; const flags = collectFlags( analyzeWorkload( @@ -407,6 +407,39 @@ describe("more flags", () => { }); }); +describe("traced launch failures", () => { + it("keeps a failed traced launch visible when no spans were decoded", () => { + const failedWithoutSpans: RunSample = { wallMs: 50, exitCode: 0, tracedExitCode: 1 }; + + const analysis = analyzeWorkload( + workload( + times(5, () => sample(50)), + times(5, () => failedWithoutSpans), + ), + true, + ); + + expect(analysis.verdict).toBe("not comparable"); + expect(collectFlags(analysis).map((flag) => flag.kind)).toEqual( + expect.arrayContaining(["traced-exit-differs", "trace-incomplete"]), + ); + }); +}); + +describe("endpoint changes without an API measurement", () => { + it("reports none when one build has no API measurement", () => { + const analysis = analyzeWorkload( + workload( + times(5, () => ({ wallMs: 50, exitCode: 0 })), + times(5, () => sample(50, { calls: 1, endpoints: { "GET /v1/projects": 1 } })), + ), + true, + ); + + expect(analysis.endpointChanges).toEqual([]); + }); +}); + describe("report JSON", () => { const minimal: BenchReport = { base: "/b", diff --git a/apps/cli/scripts/bench-summary.integration.test.ts b/apps/cli/scripts/bench-summary.integration.test.ts new file mode 100644 index 0000000000..3db5d19afe --- /dev/null +++ b/apps/cli/scripts/bench-summary.integration.test.ts @@ -0,0 +1,25 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem } from "effect"; +import { loadReport } from "./bench-summary.ts"; + +describe("loadReport", () => { + it.effect("tells an absent report from an unreadable one", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "bench-summary-" }); + const corrupt = `${dir}/report.json`; + yield* fs.writeFileString(corrupt, "{ not a report"); + + const absent = yield* loadReport(fs, `${dir}/missing.json`); + const unreadable = yield* loadReport(fs, corrupt); + + expect(absent.report).toBeUndefined(); + expect(absent.notice).toContain("did not produce a report"); + expect(unreadable.report).toBeUndefined(); + expect(unreadable.notice).toContain("could not be read or decoded"); + }), + ).pipe(Effect.provide(BunServices.layer)), + ); +}); diff --git a/apps/cli/scripts/bench-summary.ts b/apps/cli/scripts/bench-summary.ts index f2e74c04d7..55b3fca145 100644 --- a/apps/cli/scripts/bench-summary.ts +++ b/apps/cli/scripts/bench-summary.ts @@ -1,7 +1,7 @@ /** * Renders a `bench-cli.ts --out` report as a Markdown job summary: flags, wall time, API calls, * phase deltas, and binary size. Run as - * `bun scripts/bench-summary.ts --report [--base-build ok|failed]`; writes to + * `bun scripts/bench-summary.ts --report [--branch-build|--base-build|--replay ok|failed]`; writes to * `GITHUB_STEP_SUMMARY` when set, otherwise to stdout. */ import process from "node:process"; @@ -17,30 +17,60 @@ import { type WorkloadAnalysis, } from "./bench-report.ts"; -const USAGE = "usage: bun scripts/bench-summary.ts --report [--base-build ok|failed]"; +const USAGE = + "usage: bun scripts/bench-summary.ts --report [--branch-build ok|failed] [--base-build ok|failed] [--replay ok|failed]"; + +type Outcome = "ok" | "failed"; export interface SummaryOptions { readonly report: string | undefined; - readonly baseBuild: "ok" | "failed"; + readonly branchBuild: Outcome; + readonly baseBuild: Outcome; + readonly replay: Outcome; } -/** Parses argv; throws on an unknown `--base-build` or a missing report for a successful base. */ +const OUTCOME_FLAGS = ["branch-build", "base-build", "replay"] as const; + +/** Parses argv; throws on an unknown outcome or a missing report when every step succeeded. */ export function parseSummaryArgs(argv: ReadonlyArray): SummaryOptions { const { values } = parseArgs({ args: [...argv], - options: { report: { type: "string" }, "base-build": { type: "string", default: "ok" } }, + options: { + report: { type: "string" }, + "branch-build": { type: "string", default: "ok" }, + "base-build": { type: "string", default: "ok" }, + replay: { type: "string", default: "ok" }, + }, }); - const baseBuild = values["base-build"]; - if (baseBuild !== "ok" && baseBuild !== "failed") { - throw new Error('--base-build must be "ok" or "failed"'); + for (const flag of OUTCOME_FLAGS) { + if (values[flag] !== "ok" && values[flag] !== "failed") { + throw new Error(`--${flag} must be "ok" or "failed"`); + } } - if (values.report === undefined && baseBuild === "ok") throw new Error(USAGE); - return { report: values.report, baseBuild }; + const options: SummaryOptions = { + report: values.report, + branchBuild: values["branch-build"] === "failed" ? "failed" : "ok", + baseBuild: values["base-build"] === "failed" ? "failed" : "ok", + replay: values.replay === "failed" ? "failed" : "ok", + }; + if (values.report === undefined && summaryNotice(options) === undefined) throw new Error(USAGE); + return options; } -const BASE_FAILED = "base build failed, no comparison"; const NO_REPORT = "the benchmark did not produce a report; see the Run benchmark step log"; +/** The reason there is nothing to compare, in the order the steps run, or undefined when all ran. */ +export function summaryNotice( + options: Pick, +): string | undefined { + if (options.branchBuild === "failed") return "branch build failed, no comparison"; + if (options.baseBuild === "failed") return "base build failed, no comparison"; + if (options.replay === "failed") { + return "replay server failed to start, no comparison; see the Start replay server step log"; + } + return undefined; +} + const sign = (value: number) => (value >= 0 ? "+" : ""); const ms = (value: number) => `${value} ms`; const range = (build: WorkloadAnalysis["base"]) => `${build.timing.min}-${build.timing.max} ms`; @@ -108,7 +138,7 @@ function phaseSection(analyses: ReadonlyArray): string { ); if (rows.length === 0) { return analyses.some((analysis) => analysis.verdict === "not comparable") - ? "No phase deltas: no workload had a timing verdict, and some were not comparable." + ? "No phase deltas are available; some workloads were not comparable." : "No workload moved beyond noise, so there are no phase deltas."; } return table(["Workload", "Span", "Change", "Base -> branch"], rows); @@ -124,11 +154,8 @@ function sizeSection(report: BenchReport): string { ); } -/** The Markdown summary, or the base-build-failed notice when there is no report to compare. */ -export function renderSummary( - report: BenchReport | undefined, - notice: string = BASE_FAILED, -): string { +/** The Markdown summary, or `notice` when there is no report to compare. */ +export function renderSummary(report: BenchReport | undefined, notice: string): string { const title = "## CLI benchmark (report only)"; if (report === undefined) return `${title}\n\n${notice}\n`; const analyses = report.workloads.map((workload) => analyzeWorkload(workload, report.trace)); @@ -151,6 +178,27 @@ export function renderSummary( ); } +export const loadReport = Effect.fn("BenchSummary.loadReport")(function* ( + fs: FileSystem.FileSystem, + file: string, +) { + if (!(yield* fs.exists(file).pipe(Effect.orElseSucceed(() => false)))) { + return { report: undefined, notice: NO_REPORT }; + } + return yield* fs.readFileString(file).pipe( + Effect.flatMap(decodeBenchReport), + Effect.map((report) => ({ report, notice: NO_REPORT })), + Effect.catch((cause) => + Console.error(`could not read benchmark report ${file}: ${String(cause)}`).pipe( + Effect.as({ + report: undefined, + notice: "the benchmark report could not be read or decoded; see the job log", + }), + ), + ), + ); +}); + const main = Effect.gen(function* () { const stdio = yield* Stdio.Stdio; const argv = yield* stdio.args; @@ -172,15 +220,12 @@ const main = Effect.gen(function* () { if (options === undefined) return; const fs = yield* FileSystem.FileSystem; - const reportFile = options.baseBuild === "failed" ? undefined : options.report; - const report = - reportFile === undefined - ? undefined - : yield* fs.readFileString(reportFile).pipe( - Effect.flatMap(decodeBenchReport), - Effect.orElseSucceed(() => undefined), - ); - const markdown = renderSummary(report, reportFile === undefined ? BASE_FAILED : NO_REPORT); + const outcomeNotice = summaryNotice(options); + const loaded = + outcomeNotice === undefined && options.report !== undefined + ? yield* loadReport(fs, options.report) + : { report: undefined, notice: outcomeNotice ?? NO_REPORT }; + const markdown = renderSummary(loaded.report, loaded.notice); const summaryFile = yield* Config.option(Config.String("GITHUB_STEP_SUMMARY")); if (Option.isSome(summaryFile) && summaryFile.value !== "") { diff --git a/apps/cli/scripts/bench-summary.unit.test.ts b/apps/cli/scripts/bench-summary.unit.test.ts index 3894d0ee61..bbca043947 100644 --- a/apps/cli/scripts/bench-summary.unit.test.ts +++ b/apps/cli/scripts/bench-summary.unit.test.ts @@ -1,12 +1,12 @@ import { describe, expect, it } from "vitest"; import type { BenchReport, RunSample } from "./bench-report.ts"; -import { parseSummaryArgs, renderSummary } from "./bench-summary.ts"; +import { parseSummaryArgs, renderSummary, summaryNotice } from "./bench-summary.ts"; const run = (wallMs: number, calls: number, endpoints: Record): RunSample => ({ wallMs, exitCode: 0, + tracedExitCode: 0, trace: { - exitCode: 0, skippedLines: 0, spans: { "cli.run": { totalMs: wallMs / 2, count: 1 } }, api: { apiCalls: calls, externalCalls: 0, errorStatuses: 0, endpoints, requestLogCount: calls }, @@ -39,6 +39,7 @@ describe("renderSummary", () => { branch: runs(5, (i) => run(120 + i, 1, { "GET /v1/projects/abc": 1 })), }, ]), + "unused", ); expect(markdown).toContain("report only"); @@ -59,6 +60,7 @@ describe("renderSummary", () => { branch: runs(5, (i) => run(30 + i, 0, {})), }, ]), + "unused", ); expect(markdown).toContain("No flags."); @@ -70,25 +72,66 @@ describe("renderSummary", () => { expect(renderSummary(undefined, "no report here")).toContain("no report here"); }); - it("reports a failed base build without a comparison", () => { - expect(renderSummary(undefined)).toContain("base build failed, no comparison"); + it("does not claim a missing timing verdict when workloads were not comparable", () => { + const failing = (wallMs: number): RunSample => ({ ...run(wallMs, 0, {}), exitCode: 1 }); + + const markdown = renderSummary( + report([ + { + id: "broken", + args: ["x"], + base: runs(5, (i) => run(30 + i, 0, {})), + branch: runs(5, (i) => failing(30 + i)), + }, + ]), + "unused", + ); + + expect(markdown).toContain( + "No phase deltas are available; some workloads were not comparable.", + ); + }); +}); + +describe("summaryNotice", () => { + const ok = { branchBuild: "ok", baseBuild: "ok", replay: "ok" } as const; + + it("is undefined when every step succeeded", () => { + expect(summaryNotice(ok)).toBeUndefined(); + }); + + it.each([ + { failed: { branchBuild: "failed" }, expected: "branch build failed" }, + { failed: { baseBuild: "failed" }, expected: "base build failed" }, + { failed: { replay: "failed" }, expected: "replay server failed to start" }, + ] as const)("names the failed step: $expected", ({ failed, expected }) => { + expect(summaryNotice({ ...ok, ...failed })).toContain(expected); + }); + + it("reports the earliest failed step when later ones were skipped", () => { + expect( + summaryNotice({ branchBuild: "failed", baseBuild: "failed", replay: "failed" }), + ).toContain("branch build failed"); }); }); describe("renderSummary without API counts", () => { it("says API calls were not counted and keeps table cells intact when an id contains a pipe", () => { const noTrace = (wallMs: number): RunSample => ({ wallMs, exitCode: 0 }); - const markdown = renderSummary({ - ...report([ - { - id: "a|b", - args: ["x"], - base: runs(5, (i) => noTrace(30 + i)), - branch: runs(5, (i) => noTrace(30 + i)), - }, - ]), - trace: false, - }); + const markdown = renderSummary( + { + ...report([ + { + id: "a|b", + args: ["x"], + base: runs(5, (i) => noTrace(30 + i)), + branch: runs(5, (i) => noTrace(30 + i)), + }, + ]), + trace: false, + }, + "unused", + ); expect(markdown).toContain("API calls were not counted."); expect(markdown).toContain("a\\|b"); @@ -96,18 +139,23 @@ describe("renderSummary without API counts", () => { }); describe("parseSummaryArgs", () => { - it("rejects an unknown base build state", () => { - expect(() => parseSummaryArgs(["--report", "r.json", "--base-build", "maybe"])).toThrow( - "--base-build", - ); + it("rejects an unknown step outcome", () => { + expect(() => parseSummaryArgs(["--report", "r.json", "--replay", "maybe"])).toThrow("--replay"); }); - it("needs a report unless the base build failed", () => { + it("needs a report unless a step failed", () => { expect(() => parseSummaryArgs([])).toThrow(); - expect(parseSummaryArgs(["--base-build", "failed"])).toEqual({ + expect(parseSummaryArgs(["--replay", "failed"])).toEqual({ report: undefined, - baseBuild: "failed", + branchBuild: "ok", + baseBuild: "ok", + replay: "failed", + }); + expect(parseSummaryArgs(["--report", "r.json"])).toEqual({ + report: "r.json", + branchBuild: "ok", + baseBuild: "ok", + replay: "ok", }); - expect(parseSummaryArgs(["--report", "r.json"])).toEqual({ report: "r.json", baseBuild: "ok" }); }); }); From e22b2596ebfde98c39efd4563f3fcf4c12b47ffe Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 16:10:31 +0530 Subject: [PATCH 09/15] chore(cli): render the benchmark report as a plain-words summary Lead with the answer, show commands and rounded timings, and describe each flag in a sentence. The comment variant adds the marker and a re-run hint. Co-Authored-By: Claude Sonnet 5.5 --- apps/cli/docs/tracing-monitoring.md | 26 +- apps/cli/scripts/bench-report.ts | 18 +- .../scripts/bench-summary.integration.test.ts | 2 +- apps/cli/scripts/bench-summary.ts | 344 ++++++++++++++---- apps/cli/scripts/bench-summary.unit.test.ts | 205 +++++++---- 5 files changed, 449 insertions(+), 146 deletions(-) diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index 18e16fd9e8..ce0c8c2098 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -94,6 +94,25 @@ summary instead of failing the job: a failed branch or base build or a replay se start replaces the comparison with a notice naming the step. The workloads use no Docker, database, or network, and no prompts. +The first run on a pull request from this repository also posts one comment, tagged with the hidden +marker ``. Later pushes leave that comment alone. Pull requests from forks get +the job summary only, because their token is read-only. + +A maintainer can comment exactly `/bench` on a pull request to run the benchmark again on its +current head commit against the merge base with its base branch (`develop` or `next`). The comment +is updated in place, or created if it is missing. Only the repository owner and people with write +access can run it; the command must be the whole comment, so `/bench full` (reserved for a future +heavier tier) and anything else is ignored. The commenter's request is acknowledged with an eyes +reaction. Workflows triggered by comments run from the default branch, so `/bench` works only once +the workflow is on `develop`. Against a fork's head the run gets no secrets and does not use +dependency caches; it still installs and builds the fork's code, so it needs a maintainer's +judgment about the change. + +Three jobs keep that safe. `resolve` and `comment` run scripts from `.github/scripts/cli-bench` +on the default branch. `bench` builds and runs the pull request code with read-only permissions, +and the one job that can write comments only downloads its rendered markdown and posts it, so +it never runs pull request code. + To run it locally with two binaries (for example builds from two commits): ```sh @@ -123,7 +142,10 @@ against the replay server's request log for the same launch. The `--out` file st data; the summary derives medians, flags, and verdicts from it, so thresholds can change without re-running. -The summary reports: +The report (the comment and the job summary share one renderer) leads with the answer, "no changes +found" or "N things to check", then one plain-words line for each flag, a table of the median time +per command before and after the change, API calls, and the binary size. A collapsed section holds +the timing ranges, API calls per command, and the slowest phases. It reports: - Flags: an API call count that goes from 0 to more than 0 (for example `migration new` starting to call the API) or from more than 0 to 0, calls to hosts other than the replay server, counts that @@ -131,7 +153,7 @@ The summary reports: responses (the build did not follow the recorded scenario), non-zero or differing exit codes, missing traces, and too few runs for a timing verdict. Other count changes, such as 1 to 2 calls, appear in the API calls table with the changed endpoints but are not flagged. -- Wall time: base and branch medians and ranges. A verdict of slower or faster needs the branch +- Wall time: base ("before") and branch ("after") medians and ranges. A verdict of slower or faster needs the branch median outside the base min-max range, a change of at least 5 percent and 5 ms, and at least 5 measured runs; otherwise it reads "within noise", or "not comparable" when a build failed. Thresholds are named constants in `bench-report.ts`. diff --git a/apps/cli/scripts/bench-report.ts b/apps/cli/scripts/bench-report.ts index bd787c6537..8e48662b76 100644 --- a/apps/cli/scripts/bench-report.ts +++ b/apps/cli/scripts/bench-report.ts @@ -449,13 +449,15 @@ export interface Flag { readonly workload: string; readonly kind: FlagKind; readonly detail: string; + /** The build the flag is about, absent when it concerns the comparison. */ + readonly build?: Build; } /** Everything about a workload that makes its comparison suspect or notable beyond timing. */ export function collectFlags(analysis: WorkloadAnalysis): ReadonlyArray { const flags: Array = []; - const add = (kind: FlagKind, detail: string) => - flags.push({ workload: analysis.id, kind, detail }); + const add = (kind: FlagKind, detail: string, build?: Build) => + flags.push({ workload: analysis.id, kind, detail, ...(build === undefined ? {} : { build }) }); const baseCalls = analysis.base.api?.medianCalls; const branchCalls = analysis.branch.api?.medianCalls; if (baseCalls === 0 && branchCalls !== undefined && branchCalls > 0) { @@ -477,37 +479,43 @@ export function collectFlags(analysis: WorkloadAnalysis): ReadonlyArray { const summary = analysis[build]; const failed = summary.exitCodes.filter((code) => code !== 0).length; if (failed > 0) { - add("non-zero-exit", `${build} exited non-zero in ${failed}/${summary.samples} runs`); + add("non-zero-exit", `${build} exited non-zero in ${failed}/${summary.samples} runs`, build); } const timed = new Set(summary.exitCodes); if (summary.tracedExitCodes.some((code) => !timed.has(code))) { add( "traced-exit-differs", `${build} traced launch exited differently from its timed launches`, + build, ); } if (summary.traceMissingRuns > 0 || summary.skippedTraceLines > 0) { add( "trace-incomplete", `${build} has ${summary.traceMissingRuns} runs without a trace and ${summary.skippedTraceLines} undecodable trace lines`, + build, ); } const api = summary.api; if (api === undefined) continue; if (api.externalCalls > 0) { - add("external-calls", `${build} made calls to hosts other than the replay server`); + add("external-calls", `${build} made calls to hosts other than the replay server`, build); + } + if (api.nonDeterministic) { + add("non-deterministic", `${build} API calls differ between runs`, build); } - if (api.nonDeterministic) add("non-deterministic", `${build} API calls differ between runs`); if (api.logMismatchRuns > 0) { add( "request-log-mismatch", `${build} spans disagree with the replay request log in ${api.logMismatchRuns} runs`, + build, ); } if (api.errorStatuses > 0) { add( "error-status", `${build} got 4xx/5xx replay responses, so it did not follow the recorded scenario`, + build, ); } } diff --git a/apps/cli/scripts/bench-summary.integration.test.ts b/apps/cli/scripts/bench-summary.integration.test.ts index 3db5d19afe..98d45acc1a 100644 --- a/apps/cli/scripts/bench-summary.integration.test.ts +++ b/apps/cli/scripts/bench-summary.integration.test.ts @@ -18,7 +18,7 @@ describe("loadReport", () => { expect(absent.report).toBeUndefined(); expect(absent.notice).toContain("did not produce a report"); expect(unreadable.report).toBeUndefined(); - expect(unreadable.notice).toContain("could not be read or decoded"); + expect(unreadable.notice).toContain("could not be read"); }), ).pipe(Effect.provide(BunServices.layer)), ); diff --git a/apps/cli/scripts/bench-summary.ts b/apps/cli/scripts/bench-summary.ts index 55b3fca145..21b94399ee 100644 --- a/apps/cli/scripts/bench-summary.ts +++ b/apps/cli/scripts/bench-summary.ts @@ -1,8 +1,9 @@ /** - * Renders a `bench-cli.ts --out` report as a Markdown job summary: flags, wall time, API calls, - * phase deltas, and binary size. Run as - * `bun scripts/bench-summary.ts --report [--branch-build|--base-build|--replay ok|failed]`; writes to - * `GITHUB_STEP_SUMMARY` when set, otherwise to stdout. + * Renders a `bench-cli.ts --out` report as Markdown: the answer first, then a timing table, API + * calls, binary size, and collapsed details. Run as `bun scripts/bench-summary.ts --report ` + * with optional `--branch-build|--base-build|--replay ok|failed` step outcomes, `--head-sha`, + * `--base-ref`, `--merge-base`, and `--comment-out ` for the pull request + * comment variant; the job summary goes to `GITHUB_STEP_SUMMARY` when set, otherwise to stdout. */ import process from "node:process"; import { parseArgs } from "node:util"; @@ -12,13 +13,14 @@ import { analyzeWorkload, collectFlags, decodeBenchReport, + VERDICT_MIN_PCT, type BenchReport, type Flag, type WorkloadAnalysis, } from "./bench-report.ts"; const USAGE = - "usage: bun scripts/bench-summary.ts --report [--branch-build ok|failed] [--base-build ok|failed] [--replay ok|failed]"; + "usage: bun scripts/bench-summary.ts --report [--branch-build ok|failed] [--base-build ok|failed] [--replay ok|failed] [--head-sha ] [--base-ref ] [--merge-base ] [--comment-out ]"; type Outcome = "ok" | "failed"; @@ -27,6 +29,10 @@ export interface SummaryOptions { readonly branchBuild: Outcome; readonly baseBuild: Outcome; readonly replay: Outcome; + readonly headSha: string | undefined; + readonly baseRef: string | undefined; + readonly mergeBase: string | undefined; + readonly commentOut: string | undefined; } const OUTCOME_FLAGS = ["branch-build", "base-build", "replay"] as const; @@ -40,6 +46,10 @@ export function parseSummaryArgs(argv: ReadonlyArray): SummaryOptions { "branch-build": { type: "string", default: "ok" }, "base-build": { type: "string", default: "ok" }, replay: { type: "string", default: "ok" }, + "head-sha": { type: "string" }, + "base-ref": { type: "string" }, + "merge-base": { type: "string" }, + "comment-out": { type: "string" }, }, }); for (const flag of OUTCOME_FLAGS) { @@ -52,30 +62,53 @@ export function parseSummaryArgs(argv: ReadonlyArray): SummaryOptions { branchBuild: values["branch-build"] === "failed" ? "failed" : "ok", baseBuild: values["base-build"] === "failed" ? "failed" : "ok", replay: values.replay === "failed" ? "failed" : "ok", + headSha: values["head-sha"], + baseRef: values["base-ref"], + mergeBase: values["merge-base"], + commentOut: values["comment-out"], }; if (values.report === undefined && summaryNotice(options) === undefined) throw new Error(USAGE); return options; } -const NO_REPORT = "the benchmark did not produce a report; see the Run benchmark step log"; +const NO_REPORT = + "The benchmark did not produce a report. See the Run benchmark step in the workflow run."; +const UNREADABLE_REPORT = + "The benchmark report could not be read. See the Write job summary step in the workflow run."; /** The reason there is nothing to compare, in the order the steps run, or undefined when all ran. */ export function summaryNotice( options: Pick, ): string | undefined { - if (options.branchBuild === "failed") return "branch build failed, no comparison"; - if (options.baseBuild === "failed") return "base build failed, no comparison"; + if (options.branchBuild === "failed") { + return "The build of this commit failed, so there is nothing to compare."; + } + if (options.baseBuild === "failed") { + return "The build of the base branch failed, so there is nothing to compare."; + } if (options.replay === "failed") { - return "replay server failed to start, no comparison; see the Start replay server step log"; + return "The local test server failed to start, so the benchmark did not run. See the Start replay server step in the workflow run."; } return undefined; } const sign = (value: number) => (value >= 0 ? "+" : ""); -const ms = (value: number) => `${value} ms`; -const range = (build: WorkloadAnalysis["base"]) => `${build.timing.min}-${build.timing.max} ms`; +const ms = (value: number) => `${Math.round(value)} ms`; +const range = (build: WorkloadAnalysis["base"]) => + `${Math.round(build.timing.min)}-${Math.round(build.timing.max)} ms`; +const signedMs = (value: number) => `${sign(value)}${Math.round(value)} ms`; +const signedPct = (value: number) => `${sign(value)}${Math.round(value)}%`; const cell = (value: string) => value.replaceAll("|", "\\|"); +/** Inline code for text that comes from the benchmarked build or its workload file. */ +const code = (value: string) => `\`${value.replaceAll("`", "'").replaceAll(/\s+/gu, " ")}\``; +const command = (analysis: Pick) => + code(["supabase", ...analysis.args].join(" ")); +const plural = (count: number, one: string, many: string) => (count === 1 ? one : many); +const callCount = (count: number | undefined) => + `${count ?? "some"} ${count === 1 ? "call" : "calls"}`; +const MIB = 1024 * 1024; +const mb = (bytes: number) => `${(bytes / MIB).toFixed(1)} MB`; function table(header: ReadonlyArray, rows: ReadonlyArray>): string { return [ @@ -85,99 +118,265 @@ function table(header: ReadonlyArray, rows: ReadonlyArray): string { - if (flags.length === 0) return "No flags."; - return flags.map((flag) => `- **${flag.workload}** (${flag.kind}): ${flag.detail}`).join("\n"); +const when = (build: Flag["build"]) => (build === "base" ? "before" : "after"); + +/** One plain-words sentence for a flag. */ +function describeFlag(flag: Flag, analysis: WorkloadAnalysis): string { + const name = command(analysis); + const at = when(flag.build); + switch (flag.kind) { + case "new-api-calls": { + const added = analysis.endpointChanges + .filter((change) => change.branch > change.base) + .slice(0, 3) + .map((change) => code(change.endpoint)); + return ( + `${name} now calls the Management API. It made no calls before and now makes ` + + `${analysis.branch.api?.medianCalls ?? "some"}${added.length === 0 ? "" : ` (${added.join(", ")})`}.` + ); + } + case "api-calls-dropped": + return `${name} no longer calls the Management API. It made ${callCount(analysis.base.api?.medianCalls)} before and now makes none.`; + case "external-calls": + return `${name} made calls to hosts other than the local test server ${at}, so its timing may be skewed.`; + case "non-deterministic": + return `${name} makes a different number of API calls from run to run ${at}, so its results are less reliable.`; + case "request-log-mismatch": + return `The API calls counted for ${name} ${at} do not match what the local test server saw, so the counts may be wrong.`; + case "error-status": + return `${name} got error responses from the local test server ${at}, so it did not follow the recorded scenario.`; + case "trace-incomplete": + return `Some runs of ${name} ${at} produced no usable trace, so call counts and phase timings may be incomplete.`; + case "non-zero-exit": + return `${name} exited with an error ${at}; its timing is not compared.`; + case "exit-codes-differ": + return `${name} exits with different codes before and after, so its timing is not compared.`; + case "traced-exit-differs": + return `${name} exited differently when traced ${at}, so its timing is not compared.`; + case "insufficient-samples": + return `${name} had too few timed runs for a timing verdict.`; + } +} + +function changeText(analysis: WorkloadAnalysis): string { + switch (analysis.verdict) { + case "not comparable": + return "not compared"; + case "slower": + case "faster": { + const delta = Math.abs(Math.round(analysis.deltaMs)); + return `${analysis.verdict} by ${delta} ms (${signedPct(analysis.deltaPct)})`; + } + case "within noise": + return Math.abs(analysis.deltaPct) >= VERDICT_MIN_PCT + ? `no real change (${signedPct(analysis.deltaPct)}, within normal variation)` + : "no real change"; + } } -function wallTimeSection(analyses: ReadonlyArray): string { +function timingTable(analyses: ReadonlyArray): string { return table( - ["Workload", "Base", "Branch", "Change", "Base range", "Branch range", "Verdict"], + ["Command", "Before", "After", "Change"], analyses.map((analysis) => [ - analysis.id, + command(analysis), ms(analysis.base.timing.median), ms(analysis.branch.timing.median), - `${sign(analysis.deltaMs)}${analysis.deltaMs} ms (${sign(analysis.deltaPct)}${analysis.deltaPct}%)`, - range(analysis.base), - range(analysis.branch), - analysis.verdict, + changeText(analysis), ]), ); } -function apiSection(analyses: ReadonlyArray): string { +interface ApiOverview { + readonly line: string; + /** Ids of the commands whose API calls changed. */ + readonly changedIds: ReadonlyArray; +} + +function apiOverview(analyses: ReadonlyArray): ApiOverview { + const measured = analyses.filter( + (analysis) => analysis.base.api !== undefined && analysis.branch.api !== undefined, + ); + if (measured.length === 0) return { line: "API calls were not counted.", changedIds: [] }; + const changed = measured.filter( + (analysis) => + analysis.base.api?.medianCalls !== analysis.branch.api?.medianCalls || + analysis.endpointChanges.length > 0, + ); + const missing = analyses.length - measured.length; + const suffix = missing === 0 ? "" : ` (not measured for ${missing})`; + if (changed.length === 0) { + return { + line: `API calls: unchanged for ${measured.length === 1 ? "the command" : `all ${measured.length} commands`}${suffix}.`, + changedIds: [], + }; + } + const lines = changed.map((analysis) => { + const endpoints = analysis.endpointChanges + .map((change) => `${code(change.endpoint)} ${change.base} -> ${change.branch}`) + .join(", "); + return `- ${command(analysis)}: ${callCount(analysis.base.api?.medianCalls)} before, ${analysis.branch.api?.medianCalls} now${endpoints === "" ? "" : ` (${endpoints})`}`; + }); + return { + line: [ + `API calls changed for ${changed.length} ${plural(changed.length, "command", "commands")}${suffix}:`, + ...lines, + ].join("\n"), + changedIds: changed.map((analysis) => analysis.id), + }; +} + +function sizeLine(report: BenchReport): string { + const { baseSizeBytes: base, branchSizeBytes: branch } = report; + if (base === undefined || branch === undefined) return "Binary size: not recorded."; + const delta = branch - base; + if (delta === 0) return `Binary size: ${mb(branch)}, unchanged.`; + const abs = Math.abs(delta); + const size = abs < 1024 ? `${abs} bytes` : abs < MIB ? `${(abs / 1024).toFixed(1)} KB` : mb(abs); + return `Binary size: ${mb(branch)}, ${delta > 0 ? "up" : "down"} ${size} from ${mb(base)}.`; +} + +function apiDetailTable(analyses: ReadonlyArray): string { const rows = analyses .filter((analysis) => analysis.base.api !== undefined || analysis.branch.api !== undefined) .map((analysis) => [ - analysis.id, + command(analysis), String(analysis.base.api?.medianCalls ?? "n/a"), String(analysis.branch.api?.medianCalls ?? "n/a"), - analysis.endpointChanges.length === 0 - ? "" - : analysis.endpointChanges - .map((change) => `\`${change.endpoint}\` ${change.base} -> ${change.branch}`) - .join("
"), + analysis.endpointChanges + .map((change) => `${code(change.endpoint)} ${change.base} -> ${change.branch}`) + .join("
"), ]); if (rows.length === 0) return "API calls were not counted."; - return table(["Workload", "Base", "Branch", "Endpoint changes"], rows); + return table(["Command", "Before", "After", "Endpoint changes"], rows); +} + +function rangeTable(analyses: ReadonlyArray): string { + return table( + ["Command", "Before range", "After range", "Change", "Verdict"], + analyses.map((analysis) => [ + command(analysis), + range(analysis.base), + range(analysis.branch), + `${signedMs(analysis.deltaMs)} (${signedPct(analysis.deltaPct)})`, + analysis.verdict, + ]), + ); } -function phaseSection(analyses: ReadonlyArray): string { +function phaseDetail(analyses: ReadonlyArray): string { const rows = analyses.flatMap((analysis) => analysis.phaseRows.map((row) => { const only = row.onlyIn === undefined ? undefined : row[row.onlyIn]; return [ - analysis.id, - row.name, + command(analysis), + code(row.name), row.deltaMs === undefined - ? `${row.onlyIn} only, ${only?.medianMs} ms` - : `${sign(row.deltaMs)}${row.deltaMs} ms`, - `${row.base?.medianMs ?? "-"} -> ${row.branch?.medianMs ?? "-"} ms`, + ? `${row.onlyIn === "base" ? "before" : "after"} only, ${Math.round(only?.medianMs ?? 0)} ms` + : signedMs(row.deltaMs), + `${row.base === undefined ? "-" : Math.round(row.base.medianMs)} -> ${row.branch === undefined ? "-" : Math.round(row.branch.medianMs)} ms`, ]; }), ); if (rows.length === 0) { return analyses.some((analysis) => analysis.verdict === "not comparable") - ? "No phase deltas are available; some workloads were not comparable." - : "No workload moved beyond noise, so there are no phase deltas."; + ? "No phase deltas are available; some commands were not comparable." + : "No command moved beyond noise, so there are no phase deltas."; } - return table(["Workload", "Span", "Change", "Base -> branch"], rows); + return table(["Command", "Span", "Change", "Before -> after"], rows); } -function sizeSection(report: BenchReport): string { - const { baseSizeBytes: base, branchSizeBytes: branch } = report; - if (base === undefined || branch === undefined) return "Binary sizes were not recorded."; - const delta = branch - base; - return table( - ["Base", "Branch", "Change"], - [[`${base} bytes`, `${branch} bytes`, `${sign(delta)}${delta} bytes`]], - ); +export const BENCH_MARKER = ""; + +export interface RenderContext { + /** `comment` adds the marker and the re-run footer; `summary` is the job summary. */ + readonly target: "summary" | "comment"; + readonly headSha?: string | undefined; + readonly baseRef?: string | undefined; + readonly mergeBase?: string | undefined; } -/** The Markdown summary, or `notice` when there is no report to compare. */ -export function renderSummary(report: BenchReport | undefined, notice: string): string { - const title = "## CLI benchmark (report only)"; - if (report === undefined) return `${title}\n\n${notice}\n`; - const analyses = report.workloads.map((workload) => analyzeWorkload(workload, report.trace)); - const flags = analyses.flatMap(collectFlags); +const short = (sha: string) => sha.slice(0, 7); + +function contextLine(report: BenchReport | undefined, context: RenderContext): string { + const parts: Array = []; + if (context.headSha !== undefined && context.headSha !== "") { + const against = + context.baseRef === undefined || context.baseRef === "" + ? "" + : ` against ${code(context.baseRef)}${ + context.mergeBase === undefined || context.mergeBase === "" + ? "" + : ` at ${code(short(context.mergeBase))}` + }`; + parts.push(`Commit ${code(short(context.headSha))}${against}`); + } + if (report !== undefined) parts.push(`${report.runs} ${plural(report.runs, "run", "runs")} each`); + parts.push("report only, never blocks merging"); + return parts.join(" · "); +} + +function frame( + title: string, + body: ReadonlyArray, + context: RenderContext, + report?: BenchReport, +): string { + const footer = + context.target === "comment" ? ["Comment `/bench` to run it again on the latest commit."] : []; return ( [ - title, - `${report.runs} measured runs per build after ${report.warmup} warm-up runs. This job never blocks a pull request.`, - "### Flags", - flagsSection(flags), - "### Wall time", - wallTimeSection(analyses), - "### API calls", - apiSection(analyses), - "### Phase deltas", - phaseSection(analyses), - "### Binary size", - sizeSection(report), + ...(context.target === "comment" ? [BENCH_MARKER] : []), + `## CLI benchmark: ${title}`, + contextLine(report, context), + ...body, + ...footer, ].join("\n\n") + "\n" ); } +/** The Markdown report, or `notice` in plain words when there is no report to compare. */ +export function renderSummary( + report: BenchReport | undefined, + notice: string, + context: RenderContext = { target: "summary" }, +): string { + if (report === undefined) return frame("could not run", [notice], context); + const analyses = report.workloads.map((workload) => analyzeWorkload(workload, report.trace)); + const flags = analyses.flatMap((analysis) => + collectFlags(analysis).map((flag) => ({ flag, analysis })), + ); + if (analyses.length === 0) return frame("no commands were benchmarked", [], context, report); + const api = apiOverview(analyses); + const moved = analyses + .filter((analysis) => analysis.verdict === "slower" || analysis.verdict === "faster") + .map((analysis) => analysis.id); + const things = new Set([...flags.map(({ analysis }) => analysis.id), ...moved, ...api.changedIds]) + .size; + const title = + things === 0 ? "no changes found" : `${things} ${plural(things, "thing", "things")} to check`; + return frame( + title, + [ + ...flags.map(({ flag, analysis }) => `⚠️ ${describeFlag(flag, analysis)}`), + timingTable(analyses), + api.line, + sizeLine(report), + [ + "
Details: timing ranges, API calls per command, slowest phases", + "#### Timing ranges", + rangeTable(analyses), + "#### API calls per command", + apiDetailTable(analyses), + "#### Slowest phases", + phaseDetail(analyses), + "
", + ].join("\n\n"), + ], + context, + report, + ); +} + export const loadReport = Effect.fn("BenchSummary.loadReport")(function* ( fs: FileSystem.FileSystem, file: string, @@ -192,7 +391,7 @@ export const loadReport = Effect.fn("BenchSummary.loadReport")(function* ( Console.error(`could not read benchmark report ${file}: ${String(cause)}`).pipe( Effect.as({ report: undefined, - notice: "the benchmark report could not be read or decoded; see the job log", + notice: UNREADABLE_REPORT, }), ), ), @@ -225,14 +424,25 @@ const main = Effect.gen(function* () { outcomeNotice === undefined && options.report !== undefined ? yield* loadReport(fs, options.report) : { report: undefined, notice: outcomeNotice ?? NO_REPORT }; - const markdown = renderSummary(loaded.report, loaded.notice); - + const context = { + headSha: options.headSha, + baseRef: options.baseRef, + mergeBase: options.mergeBase, + }; + const markdown = renderSummary(loaded.report, loaded.notice, { target: "summary", ...context }); const summaryFile = yield* Config.option(Config.String("GITHUB_STEP_SUMMARY")); if (Option.isSome(summaryFile) && summaryFile.value !== "") { yield* fs.writeFileString(summaryFile.value, markdown, { flag: "a" }); } else { yield* Console.log(markdown); } + + if (options.commentOut !== undefined) { + yield* fs.writeFileString( + options.commentOut, + renderSummary(loaded.report, loaded.notice, { target: "comment", ...context }), + ); + } }); if (import.meta.main) { diff --git a/apps/cli/scripts/bench-summary.unit.test.ts b/apps/cli/scripts/bench-summary.unit.test.ts index bbca043947..3d5e369c6a 100644 --- a/apps/cli/scripts/bench-summary.unit.test.ts +++ b/apps/cli/scripts/bench-summary.unit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import type { BenchReport, RunSample } from "./bench-report.ts"; -import { parseSummaryArgs, renderSummary, summaryNotice } from "./bench-summary.ts"; +import { BENCH_MARKER, parseSummaryArgs, renderSummary, summaryNotice } from "./bench-summary.ts"; const run = (wallMs: number, calls: number, endpoints: Record): RunSample => ({ wallMs, @@ -28,51 +28,121 @@ const report = (workloads: BenchReport["workloads"]): BenchReport => ({ workloads, }); +const steady = (id: string, args: ReadonlyArray) => ({ + id, + args, + base: runs(5, (i) => run(30 + i, 0, {})), + branch: runs(5, (i) => run(30 + i, 0, {})), +}); + +const migrationNew = { + id: "migration-new", + args: ["migration", "new", "bench"], + base: runs(5, (i) => run(50 + i, 0, {})), + branch: runs(5, (i) => run(120 + i, 1, { "GET /v1/projects/abc": 1 })), +}; + describe("renderSummary", () => { - it("lists a flagged workload first and shows the endpoint that appeared", () => { - const markdown = renderSummary( - report([ - { - id: "migration-new", - args: ["migration", "new", "bench"], - base: runs(5, (i) => run(50 + i, 0, {})), - branch: runs(5, (i) => run(120 + i, 1, { "GET /v1/projects/abc": 1 })), - }, - ]), - "unused", + it("leads with no changes found and shows commands, not workload ids", () => { + const sized = { + ...report([steady("version", ["--version"])]), + baseSizeBytes: 140269744, + branchSizeBytes: 140269744, + }; + const markdown = renderSummary(sized, "unused", { + target: "summary", + headSha: "abcdef1234567890", + baseRef: "develop", + mergeBase: "1234567890abcdef", + }); + + expect(markdown).toContain("## CLI benchmark: no changes found"); + expect(markdown).toContain( + "Commit `abcdef1` against `develop` at `1234567` · 5 runs each · report only, never blocks merging", ); + expect(markdown).toContain("| `supabase --version` | 32 ms | 32 ms | no real change |"); + expect(markdown).toContain("API calls: unchanged for the command."); + expect(markdown).toContain("Binary size: 133.8 MB, unchanged."); + expect(markdown).not.toContain("⚠️"); + }); + + it("shows a binary size change in MB, or KB when it is small", () => { + const grown = { + ...report([steady("version", ["--version"])]), + baseSizeBytes: 140269744, + branchSizeBytes: 140269744 + 3 * 1048576, + }; + const nudged = { ...grown, branchSizeBytes: 140269744 + 2048 }; - expect(markdown).toContain("report only"); - expect(markdown).toMatch(/\*\*migration-new\*\* \(new-api-calls\)/u); - expect(markdown).not.toContain("No flags."); - expect(markdown).toContain("`GET /v1/projects/abc` 0 -> 1"); - expect(markdown).toContain("slower"); - expect(markdown).toContain("+500 bytes"); + expect(renderSummary(grown, "unused")).toContain( + "Binary size: 136.8 MB, up 3.0 MB from 133.8 MB.", + ); + expect(renderSummary(nudged, "unused")).toContain("up 2.0 KB from 133.8 MB"); }); - it("says there are no flags and no phase deltas when nothing moved", () => { - const markdown = renderSummary( - report([ - { - id: "version", - args: ["--version"], - base: runs(5, (i) => run(30 + i, 0, {})), - branch: runs(5, (i) => run(30 + i, 0, {})), - }, - ]), - "unused", + it("explains a new API call in plain words and counts each command once", () => { + const markdown = renderSummary(report([migrationNew]), "unused"); + + expect(markdown).toContain("## CLI benchmark: 1 thing to check"); + expect(markdown).toContain( + "⚠️ `supabase migration new bench` now calls the Management API. It made no calls before and now makes 1 (`GET /v1/projects/abc`).", + ); + expect(markdown).toContain("slower by 70 ms (+135%)"); + }); + + it("keeps command text from the benchmarked build inside code spans", () => { + const markdown = renderSummary(report([steady("x", ["a`@someone`", "b|c"])]), "unused"); + + expect(markdown).toContain("`supabase a'@someone' b\\|c`"); + }); + + it("says nothing was benchmarked instead of a clean pass when there are no commands", () => { + const markdown = renderSummary(report([]), "unused"); + + expect(markdown).toContain("## CLI benchmark: no commands were benchmarked"); + expect(markdown).not.toContain("no changes found"); + }); + + it("describes a dropped API call with a singular count", () => { + const dropped = { + id: "list", + args: ["projects", "list"], + base: runs(5, (i) => run(30 + i, 1, { "GET /v1/projects": 1 })), + branch: runs(5, (i) => run(30 + i, 0, {})), + }; + + const markdown = renderSummary(report([dropped]), "unused"); + + expect(markdown).toContain( + "⚠️ `supabase projects list` no longer calls the Management API. It made 1 call before and now makes none.", ); + }); + + it("names the failed step and stays a notice when there is no report", () => { + const markdown = renderSummary(undefined, "The build of this commit failed.", { + target: "summary", + }); - expect(markdown).toContain("No flags."); - expect(markdown).toContain("within noise"); - expect(markdown).toContain("no phase deltas"); + expect(markdown).toContain("## CLI benchmark: could not run"); + expect(markdown).toContain("The build of this commit failed."); + expect(markdown).not.toContain("/bench"); }); - it("reports a missing report with the notice it is given", () => { - expect(renderSummary(undefined, "no report here")).toContain("no report here"); + it("adds the marker, re-run hint only to the comment", () => { + const input = report([steady("version", ["--version"])]); + const context = { headSha: "abcdef1234" } as const; + + const comment = renderSummary(input, "unused", { target: "comment", ...context }); + const summary = renderSummary(input, "unused", { target: "summary", ...context }); + + expect(BENCH_MARKER).toBe(""); + expect(comment.startsWith(`${BENCH_MARKER}\n`)).toBe(true); + expect(comment).toContain("Comment `/bench` to run it again on the latest commit."); + expect(summary).not.toContain(BENCH_MARKER); + expect(summary).not.toContain("/bench"); }); - it("does not claim a missing timing verdict when workloads were not comparable", () => { + it("does not claim a missing timing verdict when commands were not comparable", () => { const failing = (wallMs: number): RunSample => ({ ...run(wallMs, 0, {}), exitCode: 1 }); const markdown = renderSummary( @@ -87,9 +157,28 @@ describe("renderSummary", () => { "unused", ); - expect(markdown).toContain( - "No phase deltas are available; some workloads were not comparable.", + expect(markdown).toContain("No phase deltas are available; some commands were not comparable."); + expect(markdown).toContain("not compared"); + }); + + it("says API calls were not counted when no call data exists", () => { + const noTrace = (wallMs: number): RunSample => ({ wallMs, exitCode: 0 }); + const markdown = renderSummary( + { + ...report([ + { + id: "a", + args: ["x"], + base: runs(5, (i) => noTrace(30 + i)), + branch: runs(5, (i) => noTrace(30 + i)), + }, + ]), + trace: false, + }, + "unused", ); + + expect(markdown).toContain("API calls were not counted."); }); }); @@ -101,9 +190,9 @@ describe("summaryNotice", () => { }); it.each([ - { failed: { branchBuild: "failed" }, expected: "branch build failed" }, - { failed: { baseBuild: "failed" }, expected: "base build failed" }, - { failed: { replay: "failed" }, expected: "replay server failed to start" }, + { failed: { branchBuild: "failed" }, expected: "build of this commit failed" }, + { failed: { baseBuild: "failed" }, expected: "build of the base branch failed" }, + { failed: { replay: "failed" }, expected: "test server failed to start" }, ] as const)("names the failed step: $expected", ({ failed, expected }) => { expect(summaryNotice({ ...ok, ...failed })).toContain(expected); }); @@ -111,30 +200,7 @@ describe("summaryNotice", () => { it("reports the earliest failed step when later ones were skipped", () => { expect( summaryNotice({ branchBuild: "failed", baseBuild: "failed", replay: "failed" }), - ).toContain("branch build failed"); - }); -}); - -describe("renderSummary without API counts", () => { - it("says API calls were not counted and keeps table cells intact when an id contains a pipe", () => { - const noTrace = (wallMs: number): RunSample => ({ wallMs, exitCode: 0 }); - const markdown = renderSummary( - { - ...report([ - { - id: "a|b", - args: ["x"], - base: runs(5, (i) => noTrace(30 + i)), - branch: runs(5, (i) => noTrace(30 + i)), - }, - ]), - trace: false, - }, - "unused", - ); - - expect(markdown).toContain("API calls were not counted."); - expect(markdown).toContain("a\\|b"); + ).toContain("build of this commit failed"); }); }); @@ -145,17 +211,14 @@ describe("parseSummaryArgs", () => { it("needs a report unless a step failed", () => { expect(() => parseSummaryArgs([])).toThrow(); - expect(parseSummaryArgs(["--replay", "failed"])).toEqual({ + expect(parseSummaryArgs(["--replay", "failed"])).toMatchObject({ report: undefined, branchBuild: "ok", baseBuild: "ok", replay: "failed", }); - expect(parseSummaryArgs(["--report", "r.json"])).toEqual({ - report: "r.json", - branchBuild: "ok", - baseBuild: "ok", - replay: "ok", - }); + expect( + parseSummaryArgs(["--report", "r.json", "--head-sha", "abc", "--comment-out", "c.md"]), + ).toMatchObject({ report: "r.json", replay: "ok", headSha: "abc", commentOut: "c.md" }); }); }); From 1a2799d2eb916c4328e408f5ced15712e25b00cc Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 16:10:32 +0530 Subject: [PATCH 10/15] ci: post the CLI benchmark as a PR comment and add a /bench command The first run on a pull request from this repository posts one marker comment; a maintainer's exact /bench comment re-runs the head commit and updates it. Pull request code runs in a read-only job with no secrets, and a separate job posts the sanitized markdown. Co-Authored-By: Claude Sonnet 5.5 --- .github/actions/setup/action.yml | 1 + .github/scripts/cli-bench/github.ts | 27 +++ .../scripts/cli-bench/post-comment.test.ts | 95 +++++++++ .github/scripts/cli-bench/post-comment.ts | 134 ++++++++++++ .github/scripts/cli-bench/resolve.test.ts | 115 ++++++++++ .github/scripts/cli-bench/resolve.ts | 198 ++++++++++++++++++ .github/workflows/cli-bench.yml | 175 +++++++++++++++- 7 files changed, 735 insertions(+), 10 deletions(-) create mode 100644 .github/scripts/cli-bench/github.ts create mode 100644 .github/scripts/cli-bench/post-comment.test.ts create mode 100644 .github/scripts/cli-bench/post-comment.ts create mode 100644 .github/scripts/cli-bench/resolve.test.ts create mode 100644 .github/scripts/cli-bench/resolve.ts diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index eebbb5d160..aa7dc275f0 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -130,6 +130,7 @@ runs: with: version: 2026.10.17 install_args: ${{ inputs.tools }} + cache: ${{ inputs.dependency-cache }} - name: Resolve pnpm store path if: inputs.dependency-cache == 'true' diff --git a/.github/scripts/cli-bench/github.ts b/.github/scripts/cli-bench/github.ts new file mode 100644 index 0000000000..c13b100406 --- /dev/null +++ b/.github/scripts/cli-bench/github.ts @@ -0,0 +1,27 @@ +/** Small GitHub REST helpers shared by the CLI benchmark scripts. */ + +export function requireEnv(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`Missing required environment variable: ${name}`); + return value; +} + +export async function githubFetch( + url: string, + token: string, + init: RequestInit = {}, +): Promise { + const response = await fetch(url, { + ...init, + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "Content-Type": "application/json", + }, + }); + if (!response.ok) { + throw new Error(`GitHub request failed (${response.status}) for ${url}`); + } + return response; +} diff --git a/.github/scripts/cli-bench/post-comment.test.ts b/.github/scripts/cli-bench/post-comment.test.ts new file mode 100644 index 0000000000..cd9b74823a --- /dev/null +++ b/.github/scripts/cli-bench/post-comment.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, test } from "bun:test"; +import { + BENCH_MARKER, + FAILURE_TITLE, + MAX_BODY_CHARS, + planComment, + prepareBody, + sanitize, +} from "./post-comment.ts"; + +const BOT = "github-actions[bot]"; + +describe("planComment", () => { + test("creates the comment when none carries the marker", () => { + expect(planComment("create-if-missing", [{ id: 1, body: "hi", authorLogin: BOT }])).toEqual({ + kind: "create", + }); + }); + + test("leaves an existing marker comment alone on a later push", () => { + const existing = { id: 5, body: `${BENCH_MARKER}\nold`, authorLogin: BOT }; + + expect(planComment("create-if-missing", [existing])).toEqual({ kind: "skip" }); + }); + + test("replaces a first-run failure notice on a later push", () => { + const failed = { + id: 5, + body: `${BENCH_MARKER}\n${FAILURE_TITLE}\nbuild failed`, + authorLogin: BOT, + }; + + expect(planComment("create-if-missing", [failed])).toEqual({ kind: "update", id: 5 }); + }); + + test("updates the marker comment in place for /bench", () => { + const existing = { id: 5, body: `${BENCH_MARKER}\nold`, authorLogin: BOT }; + + expect(planComment("upsert", [existing])).toEqual({ kind: "update", id: 5 }); + }); + + test("ignores a marker pasted by someone else", () => { + const forged = { id: 6, body: BENCH_MARKER, authorLogin: "mallory" }; + + expect(planComment("create-if-missing", [forged])).toEqual({ kind: "create" }); + }); +}); + +describe("prepareBody", () => { + test("puts the marker first exactly once", () => { + const body = prepareBody(`text ${BENCH_MARKER} more`); + + expect(body.startsWith(`${BENCH_MARKER}\n`)).toBe(true); + expect(body.split(BENCH_MARKER)).toHaveLength(2); + }); + + test("cuts an oversized report below GitHub's limit and says so", () => { + const body = prepareBody("x".repeat(200_000)); + + expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); + expect(body).toContain("truncated"); + }); +}); + +describe("sanitize", () => { + test("neutralizes mentions, links, images and unknown HTML but keeps the report tags", () => { + const hostile = + "@org/team [x](https://evil.test) ![i](https://evil.test/a.png) @u [a][r] www.evil.test
"; + + const clean = sanitize(hostile); + + expect(clean).not.toMatch(/@(?!\u200b)/u); + expect(clean).not.toContain("@"); + expect(clean).not.toMatch(/(?"); + }); + + test("pins the failure title and marker literals the renderer emits", () => { + expect(FAILURE_TITLE).toBe("## CLI benchmark: could not run"); + expect(BENCH_MARKER).toBe(""); + }); +}); + +describe("prepareBody run link", () => { + test("appends the trusted run link after the report", () => { + const body = prepareBody("report", "https://github.com/o/r/actions/runs/1"); + + expect(body.trimEnd().endsWith("[Workflow run](https://github.com/o/r/actions/runs/1)")).toBe( + true, + ); + }); +}); diff --git a/.github/scripts/cli-bench/post-comment.ts b/.github/scripts/cli-bench/post-comment.ts new file mode 100644 index 0000000000..78d27079ec --- /dev/null +++ b/.github/scripts/cli-bench/post-comment.ts @@ -0,0 +1,134 @@ +/** + * Posts the CLI benchmark summary as the pull request's single marker comment. + * + * Runs in the one job that can write comments and treats the rendered markdown as untrusted data + * (pull request code produced it): it reads a bounded regular file, neutralizes mentions, links and + * HTML, adds the marker and the run link itself, truncates to GitHub's body limit, and creates or + * updates the comment. Run in CI as: `bun .github/scripts/cli-bench/post-comment.ts`. + * + * Modes: `create-if-missing` leaves an existing marker comment alone unless it reports a failed + * run (first run on a pull request); `upsert` updates it in place or creates it (`/bench`). + */ +import { lstatSync, readFileSync } from "node:fs"; + +import { githubFetch, requireEnv } from "./github.ts"; + +/** Hidden tag identifying the benchmark comment; the renderer in `bench-summary.ts` emits the same text. */ +export const BENCH_MARKER = ""; +/** Login of comments posted with the workflow's `GITHUB_TOKEN`. */ +const WORKFLOW_BOT_LOGIN = "github-actions[bot]"; +/** GitHub rejects comment bodies longer than 65536 characters. */ +export const MAX_BODY_CHARS = 65_000; +const TRUNCATION_NOTICE = + "\n\n
\n\n_The report was truncated; see the workflow run for the rest._\n"; + +export type Mode = "create-if-missing" | "upsert"; + +export interface ExistingComment { + id: number; + body: string; + authorLogin: string; +} + +export type CommentPlan = { kind: "skip" } | { kind: "create" } | { kind: "update"; id: number }; + +const ALLOWED_TAGS = new Set(["
", "
", "", "", "
"]); +const ZERO_WIDTH_SPACE = "\u200b"; +/** Title the renderer gives a run that produced no comparison. */ +export const FAILURE_TITLE = "## CLI benchmark: could not run"; +export const MAX_FILE_BYTES = 1_000_000; + +/** Keeps mentions, links, images and raw HTML in untrusted markdown from rendering. */ +export function sanitize(markdown: string): string { + return markdown + .replaceAll("&", "&") + .replaceAll(/<[^>\n]*>?/gu, (tag) => + ALLOWED_TAGS.has(tag) ? tag : tag.replaceAll("<", "<"), + ) + .replaceAll("@", `@${ZERO_WIDTH_SPACE}`) + .replaceAll("://", `:${ZERO_WIDTH_SPACE}//`) + .replaceAll(/\b(www)\./giu, `$1${ZERO_WIDTH_SPACE}.`) + .replaceAll("[", "\\[") + .replaceAll("]", "\\]"); +} + +/** The comment body: the marker first, the sanitized report, then the trusted run link. */ +export function prepareBody(markdown: string, runUrl?: string): string { + const link = runUrl === undefined || runUrl === "" ? "" : `\n\n[Workflow run](${runUrl})\n`; + const content = sanitize(markdown.replaceAll(BENCH_MARKER, "")).trim(); + const body = `${BENCH_MARKER}\n${content}\n`; + const limit = MAX_BODY_CHARS - link.length; + const fitted = + body.length <= limit + ? body + : body.slice(0, limit - TRUNCATION_NOTICE.length) + TRUNCATION_NOTICE; + return fitted + link; +} + +/** What to do with the pull request's benchmark comment. Only the workflow bot's marker counts. */ +export function planComment(mode: Mode, comments: ReadonlyArray): CommentPlan { + const existing = comments.find( + (comment) => comment.authorLogin === WORKFLOW_BOT_LOGIN && comment.body.includes(BENCH_MARKER), + ); + if (existing === undefined) return { kind: "create" }; + // A first run that could not compare anything must not block the next push's real result. + const replaceable = mode === "upsert" || existing.body.includes(FAILURE_TITLE); + return replaceable ? { kind: "update", id: existing.id } : { kind: "skip" }; +} + +async function listComments(token: string, base: string, pr: number) { + const comments: Array = []; + for (let page = 1; ; page++) { + const batch = (await ( + await githubFetch(`${base}/issues/${pr}/comments?per_page=100&page=${page}`, token) + ).json()) as Array<{ id: number; body: string | null; user: { login: string } | null }>; + comments.push( + ...batch.map((entry) => ({ + id: entry.id, + body: entry.body ?? "", + authorLogin: entry.user?.login ?? "", + })), + ); + if (batch.length < 100) return comments; + } +} + +async function main(): Promise { + const token = requireEnv("GITHUB_TOKEN"); + const base = `https://api.github.com/repos/${requireEnv("GITHUB_REPOSITORY")}`; + const pr = Number(requireEnv("PR_NUMBER")); + const mode = requireEnv("MODE"); + if (mode !== "create-if-missing" && mode !== "upsert") { + throw new Error(`Unsupported MODE "${mode}".`); + } + const file = requireEnv("BODY_FILE"); + const stat = lstatSync(file); + if (!stat.isFile() || stat.size > MAX_FILE_BYTES) { + throw new Error("The benchmark comment file is not a regular file under the size limit."); + } + const body = prepareBody(readFileSync(file, "utf8"), process.env.RUN_URL); + + const plan = planComment(mode, await listComments(token, base, pr)); + if (plan.kind === "skip") { + console.log(`PR #${pr} already has a benchmark comment; leaving it untouched.`); + } else if (plan.kind === "update") { + await githubFetch(`${base}/issues/comments/${plan.id}`, token, { + method: "PATCH", + body: JSON.stringify({ body }), + }); + console.log(`Updated benchmark comment ${plan.id} on PR #${pr}.`); + } else { + await githubFetch(`${base}/issues/${pr}/comments`, token, { + method: "POST", + body: JSON.stringify({ body }), + }); + console.log(`Created benchmark comment on PR #${pr}.`); + } +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/.github/scripts/cli-bench/resolve.test.ts b/.github/scripts/cli-bench/resolve.test.ts new file mode 100644 index 0000000000..5ea69f960a --- /dev/null +++ b/.github/scripts/cli-bench/resolve.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, test } from "bun:test"; +import { type BenchPr, resolveBench, type ResolveIo } from "./resolve.ts"; + +const REPO = "supabase/cli"; +const pr = (overrides: Partial = {}): BenchPr => ({ + state: "open", + headSha: "a".repeat(40), + headRepoFullName: REPO, + baseRepoFullName: REPO, + baseRef: "develop", + ...overrides, +}); + +function io(permissions: Record, head: BenchPr = pr()) { + const reactions: Array = []; + const impl: ResolveIo = { + fetchPr: async () => head, + fetchPermission: async (login) => permissions[login], + reactToComment: async (id) => { + reactions.push(id); + }, + }; + return { impl, reactions }; +} + +const command = (body: string, association = "COLLABORATOR", login = "maintainer") => + ({ + eventName: "issue_comment", + prNumber: 7, + comment: { id: 99, authorLogin: login, authorAssociation: association, body }, + }) as const; + +describe("resolveBench", () => { + test("runs a pull_request event for its own head commit", async () => { + const { impl } = io({}); + + const result = await resolveBench( + { eventName: "pull_request", prNumber: 7, pr: pr({ baseRef: "next" }) }, + impl, + ); + + expect(result).toMatchObject({ + shouldRun: true, + trigger: "auto", + baseRef: "next", + isFork: false, + }); + }); + + test("marks a pull request from another repository as a fork", async () => { + const { impl } = io({}); + + const result = await resolveBench( + { eventName: "pull_request", prNumber: 7, pr: pr({ headRepoFullName: "someone/cli" }) }, + impl, + ); + + expect(result.isFork).toBe(true); + }); + + test("accepts /bench from a writer, runs the current head, and reacts", async () => { + const { impl, reactions } = io({ maintainer: "write" }, pr({ headSha: "b".repeat(40) })); + + const result = await resolveBench(command("/bench\n"), impl); + + expect(result).toMatchObject({ shouldRun: true, trigger: "command", headSha: "b".repeat(40) }); + expect(reactions).toEqual([99]); + }); + + test.each(["/bench full", "/benchmark", "please /bench", "/Bench"])( + "ignores %p", + async (body) => { + const { impl, reactions } = io({ maintainer: "write" }); + + const result = await resolveBench(command(body), impl); + + expect(result.shouldRun).toBe(false); + expect(reactions).toEqual([]); + }, + ); + + test("rejects a commenter without write permission even as a MEMBER", async () => { + const { impl, reactions } = io({ maintainer: "read" }); + + const result = await resolveBench(command("/bench", "MEMBER"), impl); + + expect(result.shouldRun).toBe(false); + expect(result.skipReason).toContain("not authorized"); + expect(reactions).toEqual([]); + }); + + test("lets the repository owner run it without a permission lookup result", async () => { + const { impl } = io({}); + + const result = await resolveBench(command("/bench", "OWNER"), impl); + + expect(result.shouldRun).toBe(true); + }); + + test("does not run on a closed pull request", async () => { + const { impl } = io({ maintainer: "admin" }, pr({ state: "closed" })); + + const result = await resolveBench(command("/bench"), impl); + + expect(result.shouldRun).toBe(false); + }); + + test("flags a fork head so the workflow can withhold secrets", async () => { + const { impl } = io({ maintainer: "write" }, pr({ headRepoFullName: "someone/cli" })); + + const result = await resolveBench(command("/bench"), impl); + + expect(result).toMatchObject({ shouldRun: true, isFork: true }); + }); +}); diff --git a/.github/scripts/cli-bench/resolve.ts b/.github/scripts/cli-bench/resolve.ts new file mode 100644 index 0000000000..055bc74509 --- /dev/null +++ b/.github/scripts/cli-bench/resolve.ts @@ -0,0 +1,198 @@ +/** + * CLI benchmark resolver: decides whether the benchmark runs and for which commit. + * + * Two triggers: + * - `pull_request`: runs for the event's head commit; the event payload is the only input. + * - a maintainer's `/bench` comment on a pull request: runs for the pull request's head commit + * when this workflow accepts the command. The comment must be exactly `/bench`, and the + * commenter must be the repository owner or hold write access. + * + * `resolveBench` is pure orchestration with injected I/O; `main()` wires the real GitHub calls and + * writes the outputs the workflow reads. Run in CI as: `bun .github/scripts/cli-bench/resolve.ts`. + */ +import { appendFileSync } from "node:fs"; + +import { fetchAuthorPermission, WRITE_PERMISSIONS } from "../contribution-gate.ts"; +import { githubFetch, requireEnv } from "./github.ts"; + +export const BENCH_COMMAND = "/bench"; + +export type Trigger = "auto" | "command"; + +export interface BenchPr { + state: "open" | "closed"; + headSha: string; + /** `owner/name` of the head repository, empty when it was deleted. */ + headRepoFullName: string; + baseRepoFullName: string; + baseRef: string; +} + +export type ResolveInput = + | { eventName: "pull_request"; prNumber: number; pr: BenchPr } + | { + eventName: "issue_comment"; + prNumber: number; + comment: { id: number; authorLogin: string; authorAssociation: string; body: string }; + }; + +export interface ResolveIo { + fetchPr: (prNumber: number) => Promise; + fetchPermission: (login: string) => Promise; + reactToComment: (commentId: number) => Promise; +} + +export interface ResolveResult { + shouldRun: boolean; + skipReason?: string; + trigger: Trigger; + headSha: string; + baseRef: string; + /** Whether the head commit lives in a repository other than the base repository. */ + isFork: boolean; +} + +const skipped = (trigger: Trigger, skipReason: string): ResolveResult => ({ + shouldRun: false, + skipReason, + trigger, + headSha: "", + baseRef: "", + isFork: false, +}); + +const accepted = (trigger: Trigger, pr: BenchPr): ResolveResult => ({ + shouldRun: true, + trigger, + headSha: pr.headSha, + baseRef: pr.baseRef, + isFork: pr.headRepoFullName !== pr.baseRepoFullName, +}); + +/** Decides whether the benchmark runs; never throws for an unauthorized or malformed command. */ +export async function resolveBench(input: ResolveInput, io: ResolveIo): Promise { + if (input.eventName === "pull_request") return accepted("auto", input.pr); + + const { comment } = input; + // The workflow's `if:` only pre-filters on `startsWith`, so `/bench full` must be rejected here. + if (comment.body.trim() !== BENCH_COMMAND) { + return skipped("command", `Comment is not exactly ${BENCH_COMMAND}.`); + } + // Only the repository OWNER skips the lookup: MEMBER and COLLABORATOR do not imply push access. + const permission = await io.fetchPermission(comment.authorLogin); + const authorized = + comment.authorAssociation === "OWNER" || + (permission !== undefined && WRITE_PERMISSIONS.has(permission)); + if (!authorized) { + return skipped( + "command", + `Commenter @${comment.authorLogin} is not authorized to run ${BENCH_COMMAND} ` + + `(author_association=${comment.authorAssociation}, permission=${permission ?? "n/a"}).`, + ); + } + const pr = await io.fetchPr(input.prNumber); + if (pr.state === "closed") return skipped("command", `PR #${input.prNumber} is closed.`); + + // Cosmetic: a failed reaction must never fail an authorized run. + try { + await io.reactToComment(comment.id); + } catch (error) { + console.warn(`Could not react to comment ${comment.id}: ${String(error)}`); + } + return accepted("command", pr); +} + +async function fetchPr(token: string, base: string, prNumber: number): Promise { + const value = (await (await githubFetch(`${base}/pulls/${prNumber}`, token)).json()) as { + state: "open" | "closed"; + head: { sha: string; repo: { full_name: string } | null }; + base: { ref: string; repo: { full_name: string } }; + }; + return { + state: value.state, + headSha: value.head.sha, + headRepoFullName: value.head.repo?.full_name ?? "", + baseRepoFullName: value.base.repo.full_name, + baseRef: value.base.ref, + }; +} + +/** Writes each output with a random delimiter so a value can never inject extra output lines. */ +function writeOutputs(result: ResolveResult, prNumber: number): void { + const entries: Record = { + should_run: String(result.shouldRun), + pr_number: String(prNumber), + head_sha: result.headSha, + base_ref: result.baseRef, + is_fork: String(result.isFork), + trigger: result.trigger, + }; + const lines = Object.entries(entries).map(([name, value]) => { + const delimiter = `ghadelim_${crypto.randomUUID()}`; + return `${name}<<${delimiter}\n${value}\n${delimiter}`; + }); + appendFileSync(requireEnv("GITHUB_OUTPUT"), `${lines.join("\n")}\n`); +} + +async function main(): Promise { + const token = requireEnv("GITHUB_TOKEN"); + const [owner, repo] = requireEnv("GITHUB_REPOSITORY").split("/"); + const base = `https://api.github.com/repos/${owner}/${repo}`; + const prNumber = Number(requireEnv("PR_NUMBER")); + const eventName = requireEnv("EVENT_NAME"); + + const io: ResolveIo = { + fetchPr: (n) => fetchPr(token, base, n), + fetchPermission: (login) => fetchAuthorPermission(token, owner!, repo!, login), + reactToComment: async (commentId) => { + await githubFetch(`${base}/issues/comments/${commentId}/reactions`, token, { + method: "POST", + body: JSON.stringify({ content: "eyes" }), + }); + }, + }; + + let input: ResolveInput; + if (eventName === "pull_request") { + input = { + eventName, + prNumber, + pr: { + state: "open", + headSha: requireEnv("HEAD_SHA"), + headRepoFullName: process.env.HEAD_REPO ?? "", + baseRepoFullName: requireEnv("GITHUB_REPOSITORY"), + baseRef: requireEnv("BASE_REF"), + }, + }; + } else if (eventName === "issue_comment") { + input = { + eventName, + prNumber, + comment: { + id: Number(requireEnv("COMMENT_ID")), + authorLogin: requireEnv("COMMENT_AUTHOR_LOGIN"), + authorAssociation: requireEnv("COMMENT_AUTHOR_ASSOCIATION"), + body: requireEnv("COMMENT_BODY"), + }, + }; + } else { + throw new Error(`Unsupported EVENT_NAME "${eventName}".`); + } + + const result = await resolveBench(input, io); + console.log( + `CLI benchmark resolve for PR #${prNumber}: should_run=${result.shouldRun} ` + + `trigger=${result.trigger}${result.skipReason ? ` (${result.skipReason})` : ""}`, + ); + writeOutputs(result, prNumber); + const summaryFile = process.env.GITHUB_STEP_SUMMARY; + if (result.skipReason && summaryFile) appendFileSync(summaryFile, `${result.skipReason}\n`); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml index 3f665cda1f..3239e96a45 100644 --- a/.github/workflows/cli-bench.yml +++ b/.github/workflows/cli-bench.yml @@ -1,8 +1,14 @@ name: CLI benchmark +run-name: "CLI benchmark #${{ github.event.pull_request.number || github.event.issue.number }}" -# Report-only comparison of the pull request's CLI build with a build of its merge base. Every +# Report-only comparison of a pull request's CLI build with a build of its merge base. Every # workload runs against the replay server, so no Docker, database, or network is involved. The # job never blocks a pull request and is not a required check. +# +# Each run writes a job summary. The first run on a pull request from this repository also posts +# one comment tagged ``; later pushes leave it alone. A maintainer commenting +# exactly `/bench` re-runs the benchmark on the pull request's current head and updates that +# comment. See apps/cli/docs/tracing-monitoring.md. on: pull_request: types: @@ -20,36 +26,123 @@ on: - "pnpm-lock.yaml" - "turbo.json" - ".github/workflows/cli-bench.yml" + - ".github/scripts/cli-bench/**" + # A `/bench` comment on a pull request starts a run. The resolve job checks who commented and + # which commit it applies to; the jobs that run pull request code hold no write permission, and + # the one job that writes comments never checks out or runs pull request code. + # jactionlint ignore=dangerous-triggers + issue_comment: + types: + - created -permissions: - contents: read +permissions: {} +# Comments that are not a `/bench` from a maintainer get a per-run group, so they cannot cancel an +# in-flight benchmark; `/bench full` is one of them. concurrency: - group: cli-bench-${{ github.event.pull_request.number }} + group: >- + cli-bench-${{ github.event.pull_request.number || github.event.issue.number }}-${{ + (github.event_name == 'issue_comment' && (github.event.comment.body != '/bench' || + !contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association))) + && github.run_id || 'run' }} cancel-in-progress: true jobs: + resolve: + name: Resolve + # Cheap, non-authoritative pre-filter; resolve.ts does the exact command match and the + # effective-permission lookup. + if: > + (github.event_name == 'pull_request' && github.event.pull_request.draft == false) || + (github.event_name == 'issue_comment' && + github.event.issue.pull_request != null && + startsWith(github.event.comment.body, '/bench') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association)) + runs-on: ubuntu-latest + timeout-minutes: 5 + # Reacts to the accepted command; runs only code from the default branch. + permissions: + contents: read + pull-requests: write + outputs: + should_run: ${{ steps.resolve.outputs.should_run }} + pr_number: ${{ steps.resolve.outputs.pr_number }} + # Every job benchmarks this commit, so a push during the run is never reported as measured. + head_sha: ${{ steps.resolve.outputs.head_sha }} + base_ref: ${{ steps.resolve.outputs.base_ref }} + is_fork: ${{ steps.resolve.outputs.is_fork }} + trigger: ${{ steps.resolve.outputs.trigger }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: ".bun-version" + no-cache: true + - name: Resolve + id: resolve + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + EVENT_NAME: ${{ github.event_name }} + PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + COMMENT_ID: ${{ github.event.comment.id }} + COMMENT_AUTHOR_LOGIN: ${{ github.event.comment.user.login }} + COMMENT_AUTHOR_ASSOCIATION: ${{ github.event.comment.author_association }} + COMMENT_BODY: ${{ github.event.comment.body }} + run: bun .github/scripts/cli-bench/resolve.ts + bench: - if: github.event.pull_request.draft == false name: Benchmark CLI against merge base + needs: resolve + if: needs.resolve.outputs.should_run == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 + # This job builds and runs pull request code, so it holds nothing beyond read access. + permissions: + contents: read + env: + HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} + BASE_REF: ${{ needs.resolve.outputs.base_ref }} steps: - name: Prepare benchmark directory run: | mkdir -p "$RUNNER_TEMP/bench" echo "BENCH_DIR=$RUNNER_TEMP/bench" >> "$GITHUB_ENV" - - name: Checkout + - name: Checkout pull request head uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ needs.resolve.outputs.head_sha }} fetch-depth: 0 persist-credentials: false + - name: Require a free trusted path + run: | + if [ -e trusted ] || [ -L trusted ]; then + echo "The pull request contains a trusted path." >&2 + exit 1 + fi + + # The setup action comes from the default branch, so a pull request cannot change it. + - name: Checkout default branch (setup action only) + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + path: trusted + persist-credentials: false + + # This job never references a secret, so a fork's code cannot read one; installs use the + # public registry. A command run uses the default branch's cache scope, so it skips caches. - name: Setup - uses: ./.github/actions/setup + uses: ./trusted/.github/actions/setup with: - dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + dependency-cache: ${{ needs.resolve.outputs.trigger == 'auto' && 'true' || 'false' }} - name: Build branch binary id: branch_build @@ -65,10 +158,10 @@ jobs: if: steps.branch_build.outcome == 'success' env: HUSKY: "0" - BASE_REF: ${{ github.base_ref }} run: | merge_base="$(git merge-base HEAD "origin/$BASE_REF")" echo "Merge base: $merge_base" + echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" git worktree add "$RUNNER_TEMP/base-src" "$merge_base" cd "$RUNNER_TEMP/base-src" # Setup writes this file only when the Dependency Firewall token is available. @@ -123,12 +216,17 @@ jobs: BRANCH_BUILD: ${{ steps.branch_build.outcome == 'success' && 'ok' || 'failed' }} BASE_BUILD: ${{ steps.base_build.outcome == 'success' && 'ok' || 'failed' }} REPLAY: ${{ steps.replay.outcome == 'success' && 'ok' || 'failed' }} + MERGE_BASE: ${{ steps.base_build.outputs.merge_base }} run: | bun apps/cli/scripts/bench-summary.ts \ --report "$BENCH_DIR/report.json" \ --branch-build "$BRANCH_BUILD" \ --base-build "$BASE_BUILD" \ - --replay "$REPLAY" + --replay "$REPLAY" \ + --head-sha "$HEAD_SHA" \ + --base-ref "$BASE_REF" \ + --merge-base "$MERGE_BASE" \ + --comment-out "$BENCH_DIR/comment.md" - name: Upload report if: always() @@ -140,3 +238,60 @@ jobs: ${{ runner.temp }}/bench/report.txt retention-days: 14 if-no-files-found: ignore + + - name: Upload comment + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-comment + path: ${{ runner.temp }}/bench/comment.md + retention-days: 1 + if-no-files-found: ignore + + comment: + name: Post pull request comment + needs: + - resolve + - bench + # A pull_request run from a fork has a read-only token, so only `/bench` comments a fork's + # result. Failing to post never fails the workflow. + if: >- + ${{ !cancelled() && needs.resolve.outputs.should_run == 'true' && needs.bench.result != 'skipped' && + (needs.resolve.outputs.trigger == 'command' || needs.resolve.outputs.is_fork == 'false') }} + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + # The only job that can write to the pull request. It runs scripts from the default branch and + # reads the rendered markdown as data; it never checks out or runs pull request code. + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: ".bun-version" + no-cache: true + - name: Download comment + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-bench-comment + path: ${{ runner.temp }}/bench-comment + - name: Post comment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} + MODE: ${{ needs.resolve.outputs.trigger == 'command' && 'upsert' || 'create-if-missing' }} + BODY_FILE: ${{ runner.temp }}/bench-comment/comment.md + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + if [ ! -f "$BODY_FILE" ]; then + echo "The benchmark job produced no comment." + exit 0 + fi + bun .github/scripts/cli-bench/post-comment.ts From b246e7c82c1d71b1d6eda2b732b58d070d83c881 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 16:31:25 +0530 Subject: [PATCH 11/15] ci: split the CLI benchmark so /bench runs reference no secrets bench-pr keeps the shared setup action and firewall token; bench-command installs the toolchain with its cache off and dependencies from the public registry. Both call a new local action for the shared steps. Co-Authored-By: Claude Sonnet 5.5 --- .github/actions/cli-bench-run/action.yml | 131 +++++++++++++++++ .github/actions/setup/action.yml | 1 - .github/workflows/cli-bench.yml | 171 ++++++----------------- apps/cli/docs/tracing-monitoring.md | 15 +- 4 files changed, 184 insertions(+), 134 deletions(-) create mode 100644 .github/actions/cli-bench-run/action.yml diff --git a/.github/actions/cli-bench-run/action.yml b/.github/actions/cli-bench-run/action.yml new file mode 100644 index 0000000000..bb98880de9 --- /dev/null +++ b/.github/actions/cli-bench-run/action.yml @@ -0,0 +1,131 @@ +name: CLI benchmark run +description: Builds the pull request head and its merge base, benchmarks both against the replay server, and renders the summary and comment artifacts. The caller checks out the head and installs dependencies first. + +inputs: + head-sha: + description: Commit being measured, shown in the report + required: true + base-ref: + description: Base branch the merge base is computed against + required: true + +runs: + using: composite + steps: + - name: Prepare benchmark directory + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/bench" + echo "BENCH_DIR=$RUNNER_TEMP/bench" >> "$GITHUB_ENV" + + - name: Build branch binary + id: branch_build + continue-on-error: true + shell: bash + run: | + pnpm --filter supabase run build:binary + mkdir -p "$BENCH_DIR/branch" + cp apps/cli/dist/supabase "$BENCH_DIR/branch/supabase" + + - name: Build merge-base binary + id: base_build + continue-on-error: true + if: steps.branch_build.outcome == 'success' + env: + HUSKY: "0" + BASE_REF: ${{ inputs.base-ref }} + shell: bash + run: | + merge_base="$(git merge-base HEAD "origin/$BASE_REF")" + echo "Merge base: $merge_base" + echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" + git worktree add "$RUNNER_TEMP/base-src" "$merge_base" + cd "$RUNNER_TEMP/base-src" + # The setup action writes this file only when the Dependency Firewall token is available. + if [ -f "$RUNNER_TEMP/dependency-firewall.npmrc" ]; then + export NPM_CONFIG_USERCONFIG="$RUNNER_TEMP/dependency-firewall.npmrc" + fi + pnpm install --frozen-lockfile + pnpm --filter supabase run build:binary + mkdir -p "$BENCH_DIR/base" + cp apps/cli/dist/supabase "$BENCH_DIR/base/supabase" + + - name: Start replay server + id: replay + continue-on-error: true + if: steps.branch_build.outcome == 'success' && steps.base_build.outcome == 'success' + shell: bash + run: | + bun apps/cli-e2e/scripts/replay-serve.ts > "$BENCH_DIR/replay.log" 2>&1 & + echo "REPLAY_PID=$!" >> "$GITHUB_ENV" + for _ in $(seq 1 60); do + url="$(grep -o 'http://[^"]*' "$BENCH_DIR/replay.log" | head -n 1 || true)" + if [ -n "$url" ]; then + echo "REPLAY_URL=$url" >> "$GITHUB_ENV" + exit 0 + fi + sleep 0.5 + done + echo "replay server did not report a URL" >&2 + cat "$BENCH_DIR/replay.log" >&2 + exit 1 + + - name: Run benchmark + id: bench + continue-on-error: true + if: steps.replay.outcome == 'success' + shell: bash + run: | + bun apps/cli/scripts/bench-cli.ts \ + --base "$BENCH_DIR/base/supabase" \ + --branch "$BENCH_DIR/branch/supabase" \ + --workloads apps/cli/bench/workloads.pr.json \ + --api-url "$REPLAY_URL" \ + --trace \ + --out "$BENCH_DIR/report.json" | tee "$BENCH_DIR/report.txt" + + - name: Stop replay server + if: always() && env.REPLAY_PID != '' + shell: bash + run: kill "$REPLAY_PID" || true + + - name: Write job summary + if: always() + env: + BRANCH_BUILD: ${{ steps.branch_build.outcome == 'success' && 'ok' || 'failed' }} + BASE_BUILD: ${{ steps.base_build.outcome == 'success' && 'ok' || 'failed' }} + REPLAY: ${{ steps.replay.outcome == 'success' && 'ok' || 'failed' }} + MERGE_BASE: ${{ steps.base_build.outputs.merge_base }} + HEAD_SHA: ${{ inputs.head-sha }} + BASE_REF: ${{ inputs.base-ref }} + shell: bash + run: | + bun apps/cli/scripts/bench-summary.ts \ + --report "$BENCH_DIR/report.json" \ + --branch-build "$BRANCH_BUILD" \ + --base-build "$BASE_BUILD" \ + --replay "$REPLAY" \ + --head-sha "$HEAD_SHA" \ + --base-ref "$BASE_REF" \ + --merge-base "$MERGE_BASE" \ + --comment-out "$BENCH_DIR/comment.md" + + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-report + path: | + ${{ runner.temp }}/bench/report.json + ${{ runner.temp }}/bench/report.txt + retention-days: 14 + if-no-files-found: ignore + + - name: Upload comment + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-comment + path: ${{ runner.temp }}/bench/comment.md + retention-days: 1 + if-no-files-found: ignore diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index aa7dc275f0..eebbb5d160 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -130,7 +130,6 @@ runs: with: version: 2026.10.17 install_args: ${{ inputs.tools }} - cache: ${{ inputs.dependency-cache }} - name: Resolve pnpm store path if: inputs.dependency-cache == 'true' diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml index 3239e96a45..0b7c04fa52 100644 --- a/.github/workflows/cli-bench.yml +++ b/.github/workflows/cli-bench.yml @@ -26,6 +26,7 @@ on: - "pnpm-lock.yaml" - "turbo.json" - ".github/workflows/cli-bench.yml" + - ".github/actions/cli-bench-run/**" - ".github/scripts/cli-bench/**" # A `/bench` comment on a pull request starts a run. The resolve job checks who commented and # which commit it applies to; the jobs that run pull request code hold no write permission, and @@ -97,24 +98,17 @@ jobs: COMMENT_BODY: ${{ github.event.comment.body }} run: bun .github/scripts/cli-bench/resolve.ts - bench: + # Two jobs, chosen by trigger, so the job that runs a `/bench` head never references a secret: + # secrets referenced anywhere in a job are sent to its runner. + bench-pr: name: Benchmark CLI against merge base needs: resolve - if: needs.resolve.outputs.should_run == 'true' + if: needs.resolve.outputs.should_run == 'true' && needs.resolve.outputs.trigger == 'auto' runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 - # This job builds and runs pull request code, so it holds nothing beyond read access. permissions: contents: read - env: - HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} - BASE_REF: ${{ needs.resolve.outputs.base_ref }} steps: - - name: Prepare benchmark directory - run: | - mkdir -p "$RUNNER_TEMP/bench" - echo "BENCH_DIR=$RUNNER_TEMP/bench" >> "$GITHUB_ENV" - - name: Checkout pull request head uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -122,141 +116,64 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Require a free trusted path - run: | - if [ -e trusted ] || [ -L trusted ]; then - echo "The pull request contains a trusted path." >&2 - exit 1 - fi + # GitHub withholds secrets from fork pull requests, so the token is empty for them. + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} - # The setup action comes from the default branch, so a pull request cannot change it. - - name: Checkout default branch (setup action only) + - name: Benchmark + uses: ./.github/actions/cli-bench-run + with: + head-sha: ${{ needs.resolve.outputs.head_sha }} + base-ref: ${{ needs.resolve.outputs.base_ref }} + + # Runs in the default branch's context, so it takes no secrets and writes no cache: the toolchain + # comes from `mise-action` with its cache off, and dependencies install from the public registry + # with no store cache. + bench-command: + name: Benchmark CLI against merge base (command) + needs: resolve + if: needs.resolve.outputs.should_run == 'true' && needs.resolve.outputs.trigger == 'command' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Checkout pull request head uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.repository.default_branch }} - path: trusted + ref: ${{ needs.resolve.outputs.head_sha }} + fetch-depth: 0 persist-credentials: false - # This job never references a secret, so a fork's code cannot read one; installs use the - # public registry. A command run uses the default branch's cache scope, so it skips caches. - - name: Setup - uses: ./trusted/.github/actions/setup + - name: Install toolchains + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - dependency-cache: ${{ needs.resolve.outputs.trigger == 'auto' && 'true' || 'false' }} + version: 2026.10.17 + cache: false - - name: Build branch binary - id: branch_build - continue-on-error: true - run: | - pnpm --filter supabase run build:binary - mkdir -p "$BENCH_DIR/branch" - cp apps/cli/dist/supabase "$BENCH_DIR/branch/supabase" - - - name: Build merge-base binary - id: base_build - continue-on-error: true - if: steps.branch_build.outcome == 'success' + - name: Install dependencies env: HUSKY: "0" - run: | - merge_base="$(git merge-base HEAD "origin/$BASE_REF")" - echo "Merge base: $merge_base" - echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" - git worktree add "$RUNNER_TEMP/base-src" "$merge_base" - cd "$RUNNER_TEMP/base-src" - # Setup writes this file only when the Dependency Firewall token is available. - if [ -f "$RUNNER_TEMP/dependency-firewall.npmrc" ]; then - export NPM_CONFIG_USERCONFIG="$RUNNER_TEMP/dependency-firewall.npmrc" - fi - pnpm install --frozen-lockfile - pnpm --filter supabase run build:binary - mkdir -p "$BENCH_DIR/base" - cp apps/cli/dist/supabase "$BENCH_DIR/base/supabase" - - - name: Start replay server - id: replay - continue-on-error: true - if: steps.branch_build.outcome == 'success' && steps.base_build.outcome == 'success' - run: | - bun apps/cli-e2e/scripts/replay-serve.ts > "$BENCH_DIR/replay.log" 2>&1 & - echo "REPLAY_PID=$!" >> "$GITHUB_ENV" - for _ in $(seq 1 60); do - url="$(grep -o 'http://[^"]*' "$BENCH_DIR/replay.log" | head -n 1 || true)" - if [ -n "$url" ]; then - echo "REPLAY_URL=$url" >> "$GITHUB_ENV" - exit 0 - fi - sleep 0.5 - done - echo "replay server did not report a URL" >&2 - cat "$BENCH_DIR/replay.log" >&2 - exit 1 - - - name: Run benchmark - id: bench - continue-on-error: true - if: steps.replay.outcome == 'success' - shell: bash - run: | - bun apps/cli/scripts/bench-cli.ts \ - --base "$BENCH_DIR/base/supabase" \ - --branch "$BENCH_DIR/branch/supabase" \ - --workloads apps/cli/bench/workloads.pr.json \ - --api-url "$REPLAY_URL" \ - --trace \ - --out "$BENCH_DIR/report.json" | tee "$BENCH_DIR/report.txt" - - - name: Stop replay server - if: always() && env.REPLAY_PID != '' - run: kill "$REPLAY_PID" || true + run: pnpm install --frozen-lockfile - - name: Write job summary - if: always() - env: - BRANCH_BUILD: ${{ steps.branch_build.outcome == 'success' && 'ok' || 'failed' }} - BASE_BUILD: ${{ steps.base_build.outcome == 'success' && 'ok' || 'failed' }} - REPLAY: ${{ steps.replay.outcome == 'success' && 'ok' || 'failed' }} - MERGE_BASE: ${{ steps.base_build.outputs.merge_base }} - run: | - bun apps/cli/scripts/bench-summary.ts \ - --report "$BENCH_DIR/report.json" \ - --branch-build "$BRANCH_BUILD" \ - --base-build "$BASE_BUILD" \ - --replay "$REPLAY" \ - --head-sha "$HEAD_SHA" \ - --base-ref "$BASE_REF" \ - --merge-base "$MERGE_BASE" \ - --comment-out "$BENCH_DIR/comment.md" - - - name: Upload report - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: cli-bench-report - path: | - ${{ runner.temp }}/bench/report.json - ${{ runner.temp }}/bench/report.txt - retention-days: 14 - if-no-files-found: ignore - - - name: Upload comment - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - name: Benchmark + uses: ./.github/actions/cli-bench-run with: - name: cli-bench-comment - path: ${{ runner.temp }}/bench/comment.md - retention-days: 1 - if-no-files-found: ignore + head-sha: ${{ needs.resolve.outputs.head_sha }} + base-ref: ${{ needs.resolve.outputs.base_ref }} comment: name: Post pull request comment needs: - resolve - - bench + - bench-pr + - bench-command # A pull_request run from a fork has a read-only token, so only `/bench` comments a fork's # result. Failing to post never fails the workflow. if: >- - ${{ !cancelled() && needs.resolve.outputs.should_run == 'true' && needs.bench.result != 'skipped' && + ${{ !cancelled() && needs.resolve.outputs.should_run == 'true' && (needs.bench-pr.result == 'success' || needs.bench-command.result == 'success') && (needs.resolve.outputs.trigger == 'command' || needs.resolve.outputs.is_fork == 'false') }} runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index ce0c8c2098..d9df1acfab 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -104,14 +104,17 @@ is updated in place, or created if it is missing. Only the repository owner and access can run it; the command must be the whole comment, so `/bench full` (reserved for a future heavier tier) and anything else is ignored. The commenter's request is acknowledged with an eyes reaction. Workflows triggered by comments run from the default branch, so `/bench` works only once -the workflow is on `develop`. Against a fork's head the run gets no secrets and does not use -dependency caches; it still installs and builds the fork's code, so it needs a maintainer's +the workflow is on `develop`. Against a fork's head the run still installs and builds the fork's code, so it needs a maintainer's judgment about the change. -Three jobs keep that safe. `resolve` and `comment` run scripts from `.github/scripts/cli-bench` -on the default branch. `bench` builds and runs the pull request code with read-only permissions, -and the one job that can write comments only downloads its rendered markdown and posts it, so -it never runs pull request code. +The work is split by trigger so a `/bench` run is safe. `resolve` and `comment` run scripts from the +default branch. `bench-pr` (pull requests) uses the shared setup action with the dependency firewall +token and normal caches; GitHub withholds secrets from fork pull requests. `bench-command` (`/bench`) +references no secrets, installs the toolchain with its cache off, installs dependencies from the +public registry, and writes no caches, because it checks out pull request code, possibly from a +fork, in a context where repository secrets exist. Both call `.github/actions/cli-bench-run`. Only +`comment` can write to the pull request; it downloads the rendered markdown and never runs pull +request code. To run it locally with two binaries (for example builds from two commits): From e574649a32699817a9a2b94fc6b6c347f5c9ec37 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 16:50:47 +0530 Subject: [PATCH 12/15] ci: run /bench only on same-repository pull requests in one bench job Reject /bench on forks before reacting, and collapse the two benchmark jobs back into one that uses the shared setup action and firewall token for both triggers, with the dependency cache off for /bench. Co-Authored-By: Claude Sonnet 5.5 --- .github/actions/cli-bench-run/action.yml | 131 ----------------- .github/scripts/cli-bench/resolve.test.ts | 11 +- .github/scripts/cli-bench/resolve.ts | 6 +- .github/workflows/cli-bench.yml | 166 ++++++++++++++++------ apps/cli/docs/tracing-monitoring.md | 23 ++- 5 files changed, 144 insertions(+), 193 deletions(-) delete mode 100644 .github/actions/cli-bench-run/action.yml diff --git a/.github/actions/cli-bench-run/action.yml b/.github/actions/cli-bench-run/action.yml deleted file mode 100644 index bb98880de9..0000000000 --- a/.github/actions/cli-bench-run/action.yml +++ /dev/null @@ -1,131 +0,0 @@ -name: CLI benchmark run -description: Builds the pull request head and its merge base, benchmarks both against the replay server, and renders the summary and comment artifacts. The caller checks out the head and installs dependencies first. - -inputs: - head-sha: - description: Commit being measured, shown in the report - required: true - base-ref: - description: Base branch the merge base is computed against - required: true - -runs: - using: composite - steps: - - name: Prepare benchmark directory - shell: bash - run: | - mkdir -p "$RUNNER_TEMP/bench" - echo "BENCH_DIR=$RUNNER_TEMP/bench" >> "$GITHUB_ENV" - - - name: Build branch binary - id: branch_build - continue-on-error: true - shell: bash - run: | - pnpm --filter supabase run build:binary - mkdir -p "$BENCH_DIR/branch" - cp apps/cli/dist/supabase "$BENCH_DIR/branch/supabase" - - - name: Build merge-base binary - id: base_build - continue-on-error: true - if: steps.branch_build.outcome == 'success' - env: - HUSKY: "0" - BASE_REF: ${{ inputs.base-ref }} - shell: bash - run: | - merge_base="$(git merge-base HEAD "origin/$BASE_REF")" - echo "Merge base: $merge_base" - echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" - git worktree add "$RUNNER_TEMP/base-src" "$merge_base" - cd "$RUNNER_TEMP/base-src" - # The setup action writes this file only when the Dependency Firewall token is available. - if [ -f "$RUNNER_TEMP/dependency-firewall.npmrc" ]; then - export NPM_CONFIG_USERCONFIG="$RUNNER_TEMP/dependency-firewall.npmrc" - fi - pnpm install --frozen-lockfile - pnpm --filter supabase run build:binary - mkdir -p "$BENCH_DIR/base" - cp apps/cli/dist/supabase "$BENCH_DIR/base/supabase" - - - name: Start replay server - id: replay - continue-on-error: true - if: steps.branch_build.outcome == 'success' && steps.base_build.outcome == 'success' - shell: bash - run: | - bun apps/cli-e2e/scripts/replay-serve.ts > "$BENCH_DIR/replay.log" 2>&1 & - echo "REPLAY_PID=$!" >> "$GITHUB_ENV" - for _ in $(seq 1 60); do - url="$(grep -o 'http://[^"]*' "$BENCH_DIR/replay.log" | head -n 1 || true)" - if [ -n "$url" ]; then - echo "REPLAY_URL=$url" >> "$GITHUB_ENV" - exit 0 - fi - sleep 0.5 - done - echo "replay server did not report a URL" >&2 - cat "$BENCH_DIR/replay.log" >&2 - exit 1 - - - name: Run benchmark - id: bench - continue-on-error: true - if: steps.replay.outcome == 'success' - shell: bash - run: | - bun apps/cli/scripts/bench-cli.ts \ - --base "$BENCH_DIR/base/supabase" \ - --branch "$BENCH_DIR/branch/supabase" \ - --workloads apps/cli/bench/workloads.pr.json \ - --api-url "$REPLAY_URL" \ - --trace \ - --out "$BENCH_DIR/report.json" | tee "$BENCH_DIR/report.txt" - - - name: Stop replay server - if: always() && env.REPLAY_PID != '' - shell: bash - run: kill "$REPLAY_PID" || true - - - name: Write job summary - if: always() - env: - BRANCH_BUILD: ${{ steps.branch_build.outcome == 'success' && 'ok' || 'failed' }} - BASE_BUILD: ${{ steps.base_build.outcome == 'success' && 'ok' || 'failed' }} - REPLAY: ${{ steps.replay.outcome == 'success' && 'ok' || 'failed' }} - MERGE_BASE: ${{ steps.base_build.outputs.merge_base }} - HEAD_SHA: ${{ inputs.head-sha }} - BASE_REF: ${{ inputs.base-ref }} - shell: bash - run: | - bun apps/cli/scripts/bench-summary.ts \ - --report "$BENCH_DIR/report.json" \ - --branch-build "$BRANCH_BUILD" \ - --base-build "$BASE_BUILD" \ - --replay "$REPLAY" \ - --head-sha "$HEAD_SHA" \ - --base-ref "$BASE_REF" \ - --merge-base "$MERGE_BASE" \ - --comment-out "$BENCH_DIR/comment.md" - - - name: Upload report - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: cli-bench-report - path: | - ${{ runner.temp }}/bench/report.json - ${{ runner.temp }}/bench/report.txt - retention-days: 14 - if-no-files-found: ignore - - - name: Upload comment - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: cli-bench-comment - path: ${{ runner.temp }}/bench/comment.md - retention-days: 1 - if-no-files-found: ignore diff --git a/.github/scripts/cli-bench/resolve.test.ts b/.github/scripts/cli-bench/resolve.test.ts index 5ea69f960a..87ffec9220 100644 --- a/.github/scripts/cli-bench/resolve.test.ts +++ b/.github/scripts/cli-bench/resolve.test.ts @@ -105,11 +105,16 @@ describe("resolveBench", () => { expect(result.shouldRun).toBe(false); }); - test("flags a fork head so the workflow can withhold secrets", async () => { - const { impl } = io({ maintainer: "write" }, pr({ headRepoFullName: "someone/cli" })); + test.each([ + ["a fork", "someone/cli"], + ["a deleted head repository", ""], + ])("does not run /bench on %s and does not react", async (_name, headRepo) => { + const { impl, reactions } = io({ maintainer: "write" }, pr({ headRepoFullName: headRepo })); const result = await resolveBench(command("/bench"), impl); - expect(result).toMatchObject({ shouldRun: true, isFork: true }); + expect(result.shouldRun).toBe(false); + expect(result.skipReason).toContain("from this repository"); + expect(reactions).toEqual([]); }); }); diff --git a/.github/scripts/cli-bench/resolve.ts b/.github/scripts/cli-bench/resolve.ts index 055bc74509..ad27ce9223 100644 --- a/.github/scripts/cli-bench/resolve.ts +++ b/.github/scripts/cli-bench/resolve.ts @@ -3,7 +3,7 @@ * * Two triggers: * - `pull_request`: runs for the event's head commit; the event payload is the only input. - * - a maintainer's `/bench` comment on a pull request: runs for the pull request's head commit + * - a maintainer's `/bench` comment on a pull request from this repository: runs for the pull request's head commit * when this workflow accepts the command. The comment must be exactly `/bench`, and the * commenter must be the repository owner or hold write access. * @@ -92,6 +92,10 @@ export async function resolveBench(input: ResolveInput, io: ResolveIo): Promise< } const pr = await io.fetchPr(input.prNumber); if (pr.state === "closed") return skipped("command", `PR #${input.prNumber} is closed.`); + // Only same-repository branches run: their authors already get repository secrets on a pull request. + if (pr.headRepoFullName !== pr.baseRepoFullName) { + return skipped("command", `${BENCH_COMMAND} runs only on pull requests from this repository.`); + } // Cosmetic: a failed reaction must never fail an authorized run. try { diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml index 0b7c04fa52..dbfde3552b 100644 --- a/.github/workflows/cli-bench.yml +++ b/.github/workflows/cli-bench.yml @@ -26,7 +26,6 @@ on: - "pnpm-lock.yaml" - "turbo.json" - ".github/workflows/cli-bench.yml" - - ".github/actions/cli-bench-run/**" - ".github/scripts/cli-bench/**" # A `/bench` comment on a pull request starts a run. The resolve job checks who commented and # which commit it applies to; the jobs that run pull request code hold no write permission, and @@ -98,12 +97,10 @@ jobs: COMMENT_BODY: ${{ github.event.comment.body }} run: bun .github/scripts/cli-bench/resolve.ts - # Two jobs, chosen by trigger, so the job that runs a `/bench` head never references a secret: - # secrets referenced anywhere in a job are sent to its runner. - bench-pr: + bench: name: Benchmark CLI against merge base needs: resolve - if: needs.resolve.outputs.should_run == 'true' && needs.resolve.outputs.trigger == 'auto' + if: needs.resolve.outputs.should_run == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 permissions: @@ -116,64 +113,141 @@ jobs: fetch-depth: 0 persist-credentials: false - # GitHub withholds secrets from fork pull requests, so the token is empty for them. + # `/bench` runs only for same-repository branches. It skips the dependency cache so it saves + # no package cache under the default branch's scope. - name: Setup uses: ./.github/actions/setup with: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + dependency-cache: ${{ needs.resolve.outputs.trigger == 'auto' && 'true' || 'false' }} - - name: Benchmark - uses: ./.github/actions/cli-bench-run - with: - head-sha: ${{ needs.resolve.outputs.head_sha }} - base-ref: ${{ needs.resolve.outputs.base_ref }} - - # Runs in the default branch's context, so it takes no secrets and writes no cache: the toolchain - # comes from `mise-action` with its cache off, and dependencies install from the public registry - # with no store cache. - bench-command: - name: Benchmark CLI against merge base (command) - needs: resolve - if: needs.resolve.outputs.should_run == 'true' && needs.resolve.outputs.trigger == 'command' - runs-on: blacksmith-4vcpu-ubuntu-2404 - timeout-minutes: 30 - permissions: - contents: read - steps: - - name: Checkout pull request head - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ needs.resolve.outputs.head_sha }} - fetch-depth: 0 - persist-credentials: false + - name: Prepare benchmark directory + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/bench" + echo "BENCH_DIR=$RUNNER_TEMP/bench" >> "$GITHUB_ENV" - - name: Install toolchains - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 - with: - version: 2026.10.17 - cache: false + - name: Build branch binary + id: branch_build + continue-on-error: true + shell: bash + run: | + pnpm --filter supabase run build:binary + mkdir -p "$BENCH_DIR/branch" + cp apps/cli/dist/supabase "$BENCH_DIR/branch/supabase" - - name: Install dependencies + - name: Build merge-base binary + id: base_build + continue-on-error: true + if: steps.branch_build.outcome == 'success' env: HUSKY: "0" - run: pnpm install --frozen-lockfile + BASE_REF: ${{ needs.resolve.outputs.base_ref }} + shell: bash + run: | + merge_base="$(git merge-base HEAD "origin/$BASE_REF")" + echo "Merge base: $merge_base" + echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" + git worktree add "$RUNNER_TEMP/base-src" "$merge_base" + cd "$RUNNER_TEMP/base-src" + # The setup action writes this file only when the Dependency Firewall token is available. + if [ -f "$RUNNER_TEMP/dependency-firewall.npmrc" ]; then + export NPM_CONFIG_USERCONFIG="$RUNNER_TEMP/dependency-firewall.npmrc" + fi + pnpm install --frozen-lockfile + pnpm --filter supabase run build:binary + mkdir -p "$BENCH_DIR/base" + cp apps/cli/dist/supabase "$BENCH_DIR/base/supabase" + + - name: Start replay server + id: replay + continue-on-error: true + if: steps.branch_build.outcome == 'success' && steps.base_build.outcome == 'success' + shell: bash + run: | + bun apps/cli-e2e/scripts/replay-serve.ts > "$BENCH_DIR/replay.log" 2>&1 & + echo "REPLAY_PID=$!" >> "$GITHUB_ENV" + for _ in $(seq 1 60); do + url="$(grep -o 'http://[^"]*' "$BENCH_DIR/replay.log" | head -n 1 || true)" + if [ -n "$url" ]; then + echo "REPLAY_URL=$url" >> "$GITHUB_ENV" + exit 0 + fi + sleep 0.5 + done + echo "replay server did not report a URL" >&2 + cat "$BENCH_DIR/replay.log" >&2 + exit 1 + + - name: Run benchmark + id: bench + continue-on-error: true + if: steps.replay.outcome == 'success' + shell: bash + run: | + bun apps/cli/scripts/bench-cli.ts \ + --base "$BENCH_DIR/base/supabase" \ + --branch "$BENCH_DIR/branch/supabase" \ + --workloads apps/cli/bench/workloads.pr.json \ + --api-url "$REPLAY_URL" \ + --trace \ + --out "$BENCH_DIR/report.json" | tee "$BENCH_DIR/report.txt" + + - name: Stop replay server + if: always() && env.REPLAY_PID != '' + shell: bash + run: kill "$REPLAY_PID" || true + + - name: Write job summary + if: always() + env: + BRANCH_BUILD: ${{ steps.branch_build.outcome == 'success' && 'ok' || 'failed' }} + BASE_BUILD: ${{ steps.base_build.outcome == 'success' && 'ok' || 'failed' }} + REPLAY: ${{ steps.replay.outcome == 'success' && 'ok' || 'failed' }} + MERGE_BASE: ${{ steps.base_build.outputs.merge_base }} + HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} + BASE_REF: ${{ needs.resolve.outputs.base_ref }} + shell: bash + run: | + bun apps/cli/scripts/bench-summary.ts \ + --report "$BENCH_DIR/report.json" \ + --branch-build "$BRANCH_BUILD" \ + --base-build "$BASE_BUILD" \ + --replay "$REPLAY" \ + --head-sha "$HEAD_SHA" \ + --base-ref "$BASE_REF" \ + --merge-base "$MERGE_BASE" \ + --comment-out "$BENCH_DIR/comment.md" + + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-report + path: | + ${{ runner.temp }}/bench/report.json + ${{ runner.temp }}/bench/report.txt + retention-days: 14 + if-no-files-found: ignore - - name: Benchmark - uses: ./.github/actions/cli-bench-run + - name: Upload comment + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - head-sha: ${{ needs.resolve.outputs.head_sha }} - base-ref: ${{ needs.resolve.outputs.base_ref }} + name: cli-bench-comment + path: ${{ runner.temp }}/bench/comment.md + retention-days: 1 + if-no-files-found: ignore comment: name: Post pull request comment needs: - resolve - - bench-pr - - bench-command - # A pull_request run from a fork has a read-only token, so only `/bench` comments a fork's - # result. Failing to post never fails the workflow. + - bench + # A pull_request run from a fork has a read-only token, so it posts no comment. Failing to + # post never fails the workflow. if: >- - ${{ !cancelled() && needs.resolve.outputs.should_run == 'true' && (needs.bench-pr.result == 'success' || needs.bench-command.result == 'success') && + ${{ !cancelled() && needs.resolve.outputs.should_run == 'true' && needs.bench.result == 'success' && (needs.resolve.outputs.trigger == 'command' || needs.resolve.outputs.is_fork == 'false') }} runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index d9df1acfab..0092be6d56 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -101,20 +101,19 @@ the job summary only, because their token is read-only. A maintainer can comment exactly `/bench` on a pull request to run the benchmark again on its current head commit against the merge base with its base branch (`develop` or `next`). The comment is updated in place, or created if it is missing. Only the repository owner and people with write -access can run it; the command must be the whole comment, so `/bench full` (reserved for a future +access can run it, and only on pull requests from this repository; the command must be the whole comment, so `/bench full` (reserved for a future heavier tier) and anything else is ignored. The commenter's request is acknowledged with an eyes reaction. Workflows triggered by comments run from the default branch, so `/bench` works only once -the workflow is on `develop`. Against a fork's head the run still installs and builds the fork's code, so it needs a maintainer's -judgment about the change. - -The work is split by trigger so a `/bench` run is safe. `resolve` and `comment` run scripts from the -default branch. `bench-pr` (pull requests) uses the shared setup action with the dependency firewall -token and normal caches; GitHub withholds secrets from fork pull requests. `bench-command` (`/bench`) -references no secrets, installs the toolchain with its cache off, installs dependencies from the -public registry, and writes no caches, because it checks out pull request code, possibly from a -fork, in a context where repository secrets exist. Both call `.github/actions/cli-bench-run`. Only -`comment` can write to the pull request; it downloads the rendered markdown and never runs pull -request code. +the workflow is on `develop`. +`/bench` works only on pull requests from this repository. Those branches come from people with +write access, who already get repository secrets on a normal pull request run, so `/bench` uses the +same job, setup action, and dependency firewall token. `/bench` runs turn off the dependency cache so +they do not save a package cache under the default branch's scope. The toolchain cache is saved +before any pull request code runs. A `/bench` run's pull request code could still write a cache +entry under the default branch's scope through the runner's own cache token; this is accepted +because only same-repository branches from people with write access can be run this way. `resolve` +and `comment` run scripts from the default branch. `comment` is the only job with write permission +and never runs pull request code: it downloads the rendered markdown, sanitizes it, and posts it. To run it locally with two binaries (for example builds from two commits): From 6beaa260453a1814e3c2834619cb3d1858e6918b Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 16:59:28 +0530 Subject: [PATCH 13/15] ci: benchmark only pull requests into develop Limit the pull_request trigger and /bench to develop for now. /bench also rejects stacked pull requests based on another branch. Co-Authored-By: Claude Sonnet 5.5 --- .github/scripts/cli-bench/resolve.test.ts | 13 +++++++++++++ .github/scripts/cli-bench/resolve.ts | 9 +++++++++ .github/workflows/cli-bench.yml | 1 - apps/cli/docs/tracing-monitoring.md | 6 +++--- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/scripts/cli-bench/resolve.test.ts b/.github/scripts/cli-bench/resolve.test.ts index 87ffec9220..31dcf5a83b 100644 --- a/.github/scripts/cli-bench/resolve.test.ts +++ b/.github/scripts/cli-bench/resolve.test.ts @@ -117,4 +117,17 @@ describe("resolveBench", () => { expect(result.skipReason).toContain("from this repository"); expect(reactions).toEqual([]); }); + + test.each([ + ["a pull request into next", "next"], + ["a stacked pull request into a feature branch", "feature/stack-base"], + ])("does not run /bench on %s and does not react", async (_name, baseRef) => { + const { impl, reactions } = io({ maintainer: "write" }, pr({ baseRef })); + + const result = await resolveBench(command("/bench"), impl); + + expect(result.shouldRun).toBe(false); + expect(result.skipReason).toContain("into develop"); + expect(reactions).toEqual([]); + }); }); diff --git a/.github/scripts/cli-bench/resolve.ts b/.github/scripts/cli-bench/resolve.ts index ad27ce9223..1d385bcd8d 100644 --- a/.github/scripts/cli-bench/resolve.ts +++ b/.github/scripts/cli-bench/resolve.ts @@ -16,6 +16,8 @@ import { fetchAuthorPermission, WRITE_PERMISSIONS } from "../contribution-gate.t import { githubFetch, requireEnv } from "./github.ts"; export const BENCH_COMMAND = "/bench"; +/** Base branches the benchmark runs for; keep in sync with the workflow's `branches:` filter. */ +export const BENCH_BASE_BRANCHES: ReadonlyArray = ["develop"]; export type Trigger = "auto" | "command"; @@ -97,6 +99,13 @@ export async function resolveBench(input: ResolveInput, io: ResolveIo): Promise< return skipped("command", `${BENCH_COMMAND} runs only on pull requests from this repository.`); } + if (!BENCH_BASE_BRANCHES.includes(pr.baseRef)) { + return skipped( + "command", + `${BENCH_COMMAND} runs only on pull requests into ${BENCH_BASE_BRANCHES.join(" or ")}.`, + ); + } + // Cosmetic: a failed reaction must never fail an authorized run. try { await io.reactToComment(comment.id); diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml index dbfde3552b..0784bce266 100644 --- a/.github/workflows/cli-bench.yml +++ b/.github/workflows/cli-bench.yml @@ -18,7 +18,6 @@ on: - ready_for_review branches: - develop - - next paths: - "apps/cli/**" - "apps/cli-e2e/**" diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index 0092be6d56..ff615dc35a 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -86,7 +86,7 @@ rm -f /tmp/cli-base/mise.toml /tmp/cli-base/mise.lock # if mise refuses the untr ### Pull request benchmark -`.github/workflows/cli-bench.yml` runs on every ready pull request into `develop` or `next` that touches the CLI, its packages, or +`.github/workflows/cli-bench.yml` runs on every ready pull request into `develop` that touches the CLI, its packages, or the e2e replay fixtures. It builds the pull request and the base tip it merges into with `pnpm --filter supabase run build:binary`, runs both through `bench-cli.ts` against a local replay server, and writes a job summary with `bench-summary.ts`. The job is report only: it never blocks a pull request, and a benchmark problem is reported in the @@ -99,9 +99,9 @@ marker ``. Later pushes leave that comment alone. Pull requ the job summary only, because their token is read-only. A maintainer can comment exactly `/bench` on a pull request to run the benchmark again on its -current head commit against the merge base with its base branch (`develop` or `next`). The comment +current head commit against the merge base with its base branch, `develop`. The comment is updated in place, or created if it is missing. Only the repository owner and people with write -access can run it, and only on pull requests from this repository; the command must be the whole comment, so `/bench full` (reserved for a future +access can run it, and only on pull requests into `develop` from this repository; the command must be the whole comment, so `/bench full` (reserved for a future heavier tier) and anything else is ignored. The commenter's request is acknowledged with an eyes reaction. Workflows triggered by comments run from the default branch, so `/bench` works only once the workflow is on `develop`. From 0a1dff3240738c19f6eefb2d0e41132555e79cc9 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 17:07:22 +0530 Subject: [PATCH 14/15] ci: run bench resolve and comment jobs from the workflow commit The resolve and comment jobs checked out the default branch, which lacks the cli-bench scripts until this change merges. Checkout now uses github.sha: default branch for /bench comments, the merge commit for pull_request runs. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/cli-bench.yml | 16 +++++++++------- apps/cli/docs/tracing-monitoring.md | 7 +++++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml index 0784bce266..e10f393f9a 100644 --- a/.github/workflows/cli-bench.yml +++ b/.github/workflows/cli-bench.yml @@ -27,8 +27,8 @@ on: - ".github/workflows/cli-bench.yml" - ".github/scripts/cli-bench/**" # A `/bench` comment on a pull request starts a run. The resolve job checks who commented and - # which commit it applies to; the jobs that run pull request code hold no write permission, and - # the one job that writes comments never checks out or runs pull request code. + # which commit it applies to; for `/bench` the resolve and comment jobs run default branch code, + # and the job that builds and runs pull request code holds no write permission. # jactionlint ignore=dangerous-triggers issue_comment: types: @@ -59,7 +59,9 @@ jobs: contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association)) runs-on: ubuntu-latest timeout-minutes: 5 - # Reacts to the accepted command; runs only code from the default branch. + # Reacts to the accepted command. Checkout uses `github.sha`: default branch code for + # `issue_comment`, the pull request merge commit for `pull_request`, where a fork's token is + # read-only whatever is declared here. permissions: contents: read pull-requests: write @@ -74,7 +76,6 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.repository.default_branch }} persist-credentials: false - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: @@ -251,15 +252,16 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 continue-on-error: true - # The only job that can write to the pull request. It runs scripts from the default branch and - # reads the rendered markdown as data; it never checks out or runs pull request code. + # The only job that can write to the pull request. Checkout uses `github.sha`: default branch + # code for `/bench`, and for same-repository `pull_request` runs the merge commit, which its + # author can already run with this token by editing the workflow. It never builds or runs the + # benchmarked binaries and reads the rendered markdown as data. permissions: contents: read pull-requests: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.repository.default_branch }} persist-credentials: false - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index ff615dc35a..f6b7083c75 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -112,8 +112,11 @@ they do not save a package cache under the default branch's scope. The toolchain before any pull request code runs. A `/bench` run's pull request code could still write a cache entry under the default branch's scope through the runner's own cache token; this is accepted because only same-repository branches from people with write access can be run this way. `resolve` -and `comment` run scripts from the default branch. `comment` is the only job with write permission -and never runs pull request code: it downloads the rendered markdown, sanitizes it, and posts it. +and `comment` check out `github.sha`: for `/bench` that is the default branch, so they run default +branch scripts. On `pull_request` runs it is the merge commit, which is the workflow's own code; a +fork's token is read-only there (this assumes the repository setting that sends write tokens to fork pull request workflows stays off) and same-repository authors can already edit the workflow. `comment` +is the only job with write permission and never builds or runs the benchmarked binaries: it downloads +the rendered markdown, sanitizes it, and posts it. To run it locally with two binaries (for example builds from two commits): From 3ad7421cd9a24c34f99cff6713f54f9a50d9df48 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Wed, 28 Oct 2026 19:30:32 +0530 Subject: [PATCH 15/15] ci(cli): re-run the push benchmark for /bench and keep a run history /bench no longer checks out or runs pull request code. It re-runs the pull_request workflow run for the current head commit, which benchmarks and updates the comment, and replies on the pull request when nothing can be re-run. Later pushes skip while the comment reports a result. The comment keeps a collapsed list of earlier runs. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/cli-bench/github.ts | 44 +++- .github/scripts/cli-bench/history.test.ts | 114 +++++++++ .github/scripts/cli-bench/history.ts | 106 ++++++++ .../scripts/cli-bench/post-comment.test.ts | 54 +++- .github/scripts/cli-bench/post-comment.ts | 89 ++++--- .github/scripts/cli-bench/rerun.test.ts | 184 ++++++++++++++ .github/scripts/cli-bench/rerun.ts | 231 ++++++++++++++++++ .github/scripts/cli-bench/resolve.test.ts | 142 +++-------- .github/scripts/cli-bench/resolve.ts | 211 +++------------- .github/workflows/cli-bench.yml | 116 +++++---- apps/cli/docs/tracing-monitoring.md | 71 +++--- .../cli/scripts/bench-cli.integration.test.ts | 41 +++- apps/cli/scripts/bench-cli.ts | 37 ++- 13 files changed, 1010 insertions(+), 430 deletions(-) create mode 100644 .github/scripts/cli-bench/history.test.ts create mode 100644 .github/scripts/cli-bench/history.ts create mode 100644 .github/scripts/cli-bench/rerun.test.ts create mode 100644 .github/scripts/cli-bench/rerun.ts diff --git a/.github/scripts/cli-bench/github.ts b/.github/scripts/cli-bench/github.ts index c13b100406..4859009a68 100644 --- a/.github/scripts/cli-bench/github.ts +++ b/.github/scripts/cli-bench/github.ts @@ -6,6 +6,18 @@ export function requireEnv(name: string): string { return value; } +/** A GitHub request that did not succeed, with the HTTP status and GitHub's own message. */ +export class GithubRequestError extends Error { + constructor( + readonly status: number, + readonly detail: string, + url: string, + ) { + super(`GitHub request failed (${status}) for ${url}`); + this.name = "GithubRequestError"; + } +} + export async function githubFetch( url: string, token: string, @@ -21,7 +33,37 @@ export async function githubFetch( }, }); if (!response.ok) { - throw new Error(`GitHub request failed (${response.status}) for ${url}`); + const payload = (await response.json().catch(() => ({}))) as { message?: unknown }; + const detail = typeof payload.message === "string" ? payload.message : response.statusText; + throw new GithubRequestError(response.status, detail, url); } return response; } + +export interface IssueComment { + id: number; + body: string; + authorLogin: string; +} + +/** Every comment on an issue or pull request, oldest first. */ +export async function listIssueComments( + token: string, + base: string, + issue: number, +): Promise> { + const comments: Array = []; + for (let page = 1; ; page++) { + const batch = (await ( + await githubFetch(`${base}/issues/${issue}/comments?per_page=100&page=${page}`, token) + ).json()) as Array<{ id: number; body: string | null; user: { login: string } | null }>; + comments.push( + ...batch.map((entry) => ({ + id: entry.id, + body: entry.body ?? "", + authorLogin: entry.user?.login ?? "", + })), + ); + if (batch.length < 100) return comments; + } +} diff --git a/.github/scripts/cli-bench/history.test.ts b/.github/scripts/cli-bench/history.test.ts new file mode 100644 index 0000000000..0d096bce41 --- /dev/null +++ b/.github/scripts/cli-bench/history.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, test } from "bun:test"; +import { + decodeHistory, + headlineOf, + makeEntry, + MAX_PREVIOUS_RUNS, + mergeHistory, + renderHistory, + type RunEntry, +} from "./history.ts"; + +const entry = (n: number, overrides: Partial = {}): RunEntry => ({ + sha: n.toString(16).padStart(7, "0"), + at: "2026-10-28T09:05Z", + headline: "no changes found", + url: `https://github.com/o/r/actions/runs/${n}`, + ...overrides, +}); + +describe("headlineOf", () => { + test("reads the title of the rendered report", () => { + expect(headlineOf("\n\n## CLI benchmark: 2 things to check\n\nbody")).toBe( + "2 things to check", + ); + }); + + test("falls back when the report has no title", () => { + expect(headlineOf("nothing here")).toBe("unknown"); + }); +}); + +describe("makeEntry", () => { + test("shortens the commit and rounds the time down to the minute in UTC", () => { + const made = makeEntry({ + sha: "0123456789abcdef0123456789abcdef01234567", + now: new Date("2026-10-28T09:05:59Z"), + headline: "no changes found", + url: "https://github.com/o/r/actions/runs/9/attempts/2", + }); + + expect(made).toMatchObject({ sha: "0123456", at: "2026-10-28T09:05Z" }); + }); + + test("refuses a run link that is not a workflow run", () => { + const made = makeEntry({ + sha: "abcdef1", + now: new Date(), + headline: "h", + url: "https://evil.test/x", + }); + + expect(made).toBeUndefined(); + }); +}); + +describe("mergeHistory", () => { + test("puts the new run first", () => { + expect(mergeHistory([entry(1)], entry(2)).map((e) => e.sha)).toEqual([ + entry(2).sha, + entry(1).sha, + ]); + }); + + test("keeps the current run plus at most 20 earlier ones, dropping the oldest", () => { + const previous = Array.from({ length: 30 }, (_, i) => entry(i + 1)); + + const merged = mergeHistory(previous, entry(100)); + + expect(merged).toHaveLength(MAX_PREVIOUS_RUNS + 1); + expect(merged[0]).toEqual(entry(100)); + expect(merged.at(-1)).toEqual(entry(MAX_PREVIOUS_RUNS)); + }); +}); + +describe("renderHistory and decodeHistory", () => { + test("round-trips the entries through the hidden marker", () => { + const entries = [entry(3), entry(2), entry(1)]; + + expect(decodeHistory(`body\n${renderHistory(entries, (t) => t)}\n`)).toEqual(entries); + }); + + test("lists earlier runs newest first and leaves the current run out of the list", () => { + const rendered = renderHistory( + [entry(3), entry(2, { headline: "1 thing to check" })], + (t) => t, + ); + + expect(rendered).toContain("Previous runs"); + expect(rendered).toContain( + "- `0000002` · 2026-10-28 09:05 UTC · 1 thing to check · [run](https://github.com/o/r/actions/runs/2)", + ); + expect(rendered).not.toContain("0000003` ·"); + }); + + test("shows no list for a first run", () => { + expect(renderHistory([entry(1)], (t) => t)).not.toContain("Previous runs"); + }); + + test.each([ + ["no marker", "plain comment"], + ["not base64 json", ""], + ["wrong shape", ``], + [ + "a forged run link", + ``, + ], + [ + "too many entries", + ``, + ], + ])("treats %s as empty history", (_name, body) => { + expect(decodeHistory(body)).toEqual([]); + }); +}); diff --git a/.github/scripts/cli-bench/history.ts b/.github/scripts/cli-bench/history.ts new file mode 100644 index 0000000000..9602383119 --- /dev/null +++ b/.github/scripts/cli-bench/history.ts @@ -0,0 +1,106 @@ +/** + * Run history kept inside the benchmark comment. The newest run is the comment itself; earlier + * runs sit in a collapsed list, backed by a hidden marker so the next update can carry them on. + * The marker is read back only from the workflow bot's own comment, and anything that does not + * decode cleanly counts as no history. + */ + +/** Earlier runs kept below the current one. */ +export const MAX_PREVIOUS_RUNS = 20; +const HISTORY_MARKER = //u; +const MAX_ENCODED_CHARS = 30_000; +const MAX_HEADLINE_CHARS = 100; +const SHA = /^[0-9a-f]{7,40}$/u; +const AT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z$/u; +const RUN_URL = /^https:\/\/[^\s)<>]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?$/u; + +export interface RunEntry { + /** Short commit SHA. */ + sha: string; + /** UTC time to the minute, `YYYY-MM-DDTHH:MMZ`. */ + at: string; + headline: string; + url: string; +} + +const isEntry = (value: unknown): value is RunEntry => { + if (typeof value !== "object" || value === null) return false; + const { sha, at, headline, url } = value as Record; + return ( + typeof sha === "string" && + SHA.test(sha) && + typeof at === "string" && + AT.test(at) && + typeof headline === "string" && + headline.length <= MAX_HEADLINE_CHARS && + !/[\r\n]/u.test(headline) && + typeof url === "string" && + RUN_URL.test(url) + ); +}; + +/** The title the renderer puts after `## CLI benchmark:`, flattened to one bounded line. */ +export function headlineOf(markdown: string): string { + const title = /^## CLI benchmark: (.+)$/mu.exec(markdown)?.[1] ?? "unknown"; + return title.replaceAll(/\s+/gu, " ").trim().slice(0, MAX_HEADLINE_CHARS); +} + +/** An entry for a run, or undefined when the commit or run link is not in the expected shape. */ +export function makeEntry(input: { + sha: string; + now: Date; + headline: string; + url: string; +}): RunEntry | undefined { + const entry = { + sha: input.sha.slice(0, 7), + at: `${input.now.toISOString().slice(0, 16)}Z`, + headline: input.headline, + url: input.url, + }; + return isEntry(entry) ? entry : undefined; +} + +/** The runs recorded in a comment body, newest first; empty when the marker is absent or invalid. */ +export function decodeHistory(body: string): Array { + const encoded = HISTORY_MARKER.exec(body)?.[1]; + if (encoded === undefined || encoded.length > MAX_ENCODED_CHARS) return []; + try { + const value: unknown = JSON.parse(Buffer.from(encoded, "base64").toString("utf8")); + return Array.isArray(value) && value.length <= MAX_PREVIOUS_RUNS + 1 && value.every(isEntry) + ? value + : []; + } catch { + return []; + } +} + +/** The new run first, then the earlier ones, with at most {@link MAX_PREVIOUS_RUNS} behind it. */ +export function mergeHistory( + previous: ReadonlyArray, + current: RunEntry, +): Array { + return [current, ...previous].slice(0, MAX_PREVIOUS_RUNS + 1); +} + +const day = (at: string) => `${at.slice(0, 10)} ${at.slice(11, 16)} UTC`; + +/** + * The hidden marker for `entries` (current run first), preceded by a collapsed list of the + * earlier ones when there are any. `clean` makes a headline safe to render. + */ +export function renderHistory( + entries: ReadonlyArray, + clean: (text: string) => string, +): string { + const marker = ``; + const earlier = entries.slice(1); + if (earlier.length === 0) return marker; + const lines = earlier.map( + (entry) => + `- \`${entry.sha}\` · ${day(entry.at)} · ${clean(entry.headline)} · [run](${entry.url})`, + ); + return ["
Previous runs", "", ...lines, "", "
", marker].join( + "\n", + ); +} diff --git a/.github/scripts/cli-bench/post-comment.test.ts b/.github/scripts/cli-bench/post-comment.test.ts index cd9b74823a..fb890a8952 100644 --- a/.github/scripts/cli-bench/post-comment.test.ts +++ b/.github/scripts/cli-bench/post-comment.test.ts @@ -7,6 +7,7 @@ import { prepareBody, sanitize, } from "./post-comment.ts"; +import { decodeHistory, type RunEntry } from "./history.ts"; const BOT = "github-actions[bot]"; @@ -30,13 +31,21 @@ describe("planComment", () => { authorLogin: BOT, }; - expect(planComment("create-if-missing", [failed])).toEqual({ kind: "update", id: 5 }); + expect(planComment("create-if-missing", [failed])).toEqual({ + kind: "update", + id: 5, + previousBody: failed.body, + }); }); - test("updates the marker comment in place for /bench", () => { + test("updates the marker comment in place for a re-run", () => { const existing = { id: 5, body: `${BENCH_MARKER}\nold`, authorLogin: BOT }; - expect(planComment("upsert", [existing])).toEqual({ kind: "update", id: 5 }); + expect(planComment("upsert", [existing])).toEqual({ + kind: "update", + id: 5, + previousBody: existing.body, + }); }); test("ignores a marker pasted by someone else", () => { @@ -62,6 +71,45 @@ describe("prepareBody", () => { }); }); +describe("prepareBody history", () => { + const run = (n: number): RunEntry => ({ + sha: n.toString(16).padStart(7, "0"), + at: "2026-10-28T09:05Z", + headline: "no changes found", + url: `https://github.com/o/r/actions/runs/${n}`, + }); + + test("lists earlier runs after the run link and carries them in the marker", () => { + const body = prepareBody("report", "https://github.com/o/r/actions/runs/3", [run(3), run(2)]); + + expect(body.indexOf("[Workflow run]")).toBeLessThan(body.indexOf("Previous runs")); + expect(decodeHistory(body)).toEqual([run(3), run(2)]); + }); + + test("drops the oldest runs before cutting the report when the body is too long", () => { + const history = Array.from({ length: 21 }, (_, i) => run(21 - i)); + const report = "x".repeat(MAX_BODY_CHARS - 4_000); + + const body = prepareBody(report, "https://github.com/o/r/actions/runs/21", history); + + expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); + expect(body).not.toContain("truncated"); + const kept = decodeHistory(body); + expect(kept.length).toBeGreaterThan(0); + expect(kept.length).toBeLessThan(history.length); + expect(kept[0]).toEqual(history[0]); + }); + + test("renders an earlier run's headline through the sanitizer", () => { + const hostile = { ...run(1), headline: "@team " }; + + const body = prepareBody("report", undefined, [run(2), hostile]); + + expect(body).not.toContain(" { test("neutralizes mentions, links, images and unknown HTML but keeps the report tags", () => { const hostile = diff --git a/.github/scripts/cli-bench/post-comment.ts b/.github/scripts/cli-bench/post-comment.ts index 78d27079ec..d46572ea3c 100644 --- a/.github/scripts/cli-bench/post-comment.ts +++ b/.github/scripts/cli-bench/post-comment.ts @@ -7,11 +7,14 @@ * updates the comment. Run in CI as: `bun .github/scripts/cli-bench/post-comment.ts`. * * Modes: `create-if-missing` leaves an existing marker comment alone unless it reports a failed - * run (first run on a pull request); `upsert` updates it in place or creates it (`/bench`). + * run (first run on a pull request); `upsert` updates it in place or creates it (a re-run). + * Every post also records the run in the comment's history of earlier runs. */ import { lstatSync, readFileSync } from "node:fs"; -import { githubFetch, requireEnv } from "./github.ts"; +import { githubFetch, listIssueComments, requireEnv, type IssueComment } from "./github.ts"; +import { decodeHistory, headlineOf, makeEntry, mergeHistory, renderHistory } from "./history.ts"; +import type { RunEntry } from "./history.ts"; /** Hidden tag identifying the benchmark comment; the renderer in `bench-summary.ts` emits the same text. */ export const BENCH_MARKER = ""; @@ -24,13 +27,12 @@ const TRUNCATION_NOTICE = export type Mode = "create-if-missing" | "upsert"; -export interface ExistingComment { - id: number; - body: string; - authorLogin: string; -} +export type ExistingComment = IssueComment; -export type CommentPlan = { kind: "skip" } | { kind: "create" } | { kind: "update"; id: number }; +export type CommentPlan = + | { kind: "skip" } + | { kind: "create" } + | { kind: "update"; id: number; previousBody: string }; const ALLOWED_TAGS = new Set(["
", "
", "", "", "
"]); const ZERO_WIDTH_SPACE = "\u200b"; @@ -52,45 +54,49 @@ export function sanitize(markdown: string): string { .replaceAll("]", "\\]"); } -/** The comment body: the marker first, the sanitized report, then the trusted run link. */ -export function prepareBody(markdown: string, runUrl?: string): string { +/** + * The comment body: the marker first, the sanitized report, the trusted run link, then the run + * history. `history` lists the current run first; the oldest runs are dropped until the body fits. + */ +export function prepareBody( + markdown: string, + runUrl?: string, + history: ReadonlyArray = [], +): string { const link = runUrl === undefined || runUrl === "" ? "" : `\n\n[Workflow run](${runUrl})\n`; const content = sanitize(markdown.replaceAll(BENCH_MARKER, "")).trim(); const body = `${BENCH_MARKER}\n${content}\n`; - const limit = MAX_BODY_CHARS - link.length; + for (let kept = history.length; kept >= 1; kept--) { + const tail = `${link}\n${renderHistory(history.slice(0, kept), sanitize)}\n`; + if (body.length + tail.length <= MAX_BODY_CHARS) return body + tail; + } + const tail = `${link}\n${history.length === 0 ? "" : `${renderHistory(history.slice(0, 1), sanitize)}\n`}`; + const limit = MAX_BODY_CHARS - tail.length; const fitted = body.length <= limit ? body : body.slice(0, limit - TRUNCATION_NOTICE.length) + TRUNCATION_NOTICE; - return fitted + link; + return fitted + tail; } -/** What to do with the pull request's benchmark comment. Only the workflow bot's marker counts. */ -export function planComment(mode: Mode, comments: ReadonlyArray): CommentPlan { - const existing = comments.find( +/** The workflow bot's benchmark comment, if any. Only its marker counts. */ +export function findBenchComment( + comments: ReadonlyArray, +): ExistingComment | undefined { + return comments.find( (comment) => comment.authorLogin === WORKFLOW_BOT_LOGIN && comment.body.includes(BENCH_MARKER), ); +} + +/** What to do with the pull request's benchmark comment. */ +export function planComment(mode: Mode, comments: ReadonlyArray): CommentPlan { + const existing = findBenchComment(comments); if (existing === undefined) return { kind: "create" }; // A first run that could not compare anything must not block the next push's real result. const replaceable = mode === "upsert" || existing.body.includes(FAILURE_TITLE); - return replaceable ? { kind: "update", id: existing.id } : { kind: "skip" }; -} - -async function listComments(token: string, base: string, pr: number) { - const comments: Array = []; - for (let page = 1; ; page++) { - const batch = (await ( - await githubFetch(`${base}/issues/${pr}/comments?per_page=100&page=${page}`, token) - ).json()) as Array<{ id: number; body: string | null; user: { login: string } | null }>; - comments.push( - ...batch.map((entry) => ({ - id: entry.id, - body: entry.body ?? "", - authorLogin: entry.user?.login ?? "", - })), - ); - if (batch.length < 100) return comments; - } + return replaceable + ? { kind: "update", id: existing.id, previousBody: existing.body } + : { kind: "skip" }; } async function main(): Promise { @@ -106,9 +112,22 @@ async function main(): Promise { if (!stat.isFile() || stat.size > MAX_FILE_BYTES) { throw new Error("The benchmark comment file is not a regular file under the size limit."); } - const body = prepareBody(readFileSync(file, "utf8"), process.env.RUN_URL); + const markdown = readFileSync(file, "utf8"); + const runUrl = process.env.RUN_URL; + const entry = makeEntry({ + sha: requireEnv("HEAD_SHA"), + now: new Date(), + headline: headlineOf(markdown), + url: runUrl ?? "", + }); - const plan = planComment(mode, await listComments(token, base, pr)); + const plan = planComment(mode, await listIssueComments(token, base, pr)); + const earlier = plan.kind === "update" ? decodeHistory(plan.previousBody) : []; + const body = prepareBody( + markdown, + runUrl, + entry === undefined ? [] : mergeHistory(earlier, entry), + ); if (plan.kind === "skip") { console.log(`PR #${pr} already has a benchmark comment; leaving it untouched.`); } else if (plan.kind === "update") { diff --git a/.github/scripts/cli-bench/rerun.test.ts b/.github/scripts/cli-bench/rerun.test.ts new file mode 100644 index 0000000000..fa37cad648 --- /dev/null +++ b/.github/scripts/cli-bench/rerun.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, test } from "bun:test"; +import { GithubRequestError } from "./github.ts"; +import { + type BenchPr, + type CommandIo, + handleBenchCommand, + RERUN_WINDOW_MS, + type WorkflowRun, +} from "./rerun.ts"; + +const REPO = "supabase/cli"; +const HEAD = "a".repeat(40); +const NOW = new Date("2026-10-28T12:00:00Z"); +const pr = (overrides: Partial = {}): BenchPr => ({ + state: "open", + draft: false, + headSha: HEAD, + headRepoFullName: REPO, + baseRepoFullName: REPO, + baseRef: "develop", + ...overrides, +}); +const run = (overrides: Partial = {}): WorkflowRun => ({ + id: 11, + status: "completed", + createdAt: "2026-10-27T12:00:00Z", + ...overrides, +}); + +function setup( + options: { + permissions?: Record; + pr?: BenchPr; + runs?: ReadonlyArray; + rerunError?: Error; + } = {}, +) { + const calls = { + reactions: [] as Array, + replies: [] as Array, + reruns: [] as Array, + }; + const io: CommandIo = { + fetchPr: async () => options.pr ?? pr(), + fetchPermission: async (login) => (options.permissions ?? { maintainer: "write" })[login], + listRuns: async () => options.runs ?? [run()], + rerun: async (id) => { + if (options.rerunError !== undefined) throw options.rerunError; + calls.reruns.push(id); + }, + reactToComment: async (id) => { + calls.reactions.push(id); + }, + replyToPr: async (_n, body) => { + calls.replies.push(body); + }, + }; + return { io, calls }; +} + +const command = (body: string, association = "COLLABORATOR", login = "maintainer") => ({ + prNumber: 7, + comment: { id: 99, authorLogin: login, authorAssociation: association, body }, + now: NOW, +}); + +describe("handleBenchCommand", () => { + test("re-runs the latest run for the head commit and reacts only after GitHub accepts it", async () => { + const { io, calls } = setup({ + runs: [run({ id: 1, createdAt: "2026-10-20T00:00:00Z" }), run({ id: 2 })], + }); + + const outcome = await handleBenchCommand(command("/bench\n"), io); + + expect(outcome).toEqual({ kind: "rerun", runId: 2 }); + expect(calls.reruns).toEqual([2]); + expect(calls.reactions).toEqual([99]); + expect(calls.replies).toEqual([]); + }); + + test.each(["/bench full", "/benchmark", "please /bench", "/Bench"])( + "ignores %p without a reply", + async (body) => { + const { io, calls } = setup(); + + const outcome = await handleBenchCommand(command(body), io); + + expect(outcome.kind).toBe("ignored"); + expect(calls).toEqual({ reactions: [], replies: [], reruns: [] }); + }, + ); + + test("rejects a commenter without write permission even as a MEMBER", async () => { + const { io, calls } = setup({ permissions: { maintainer: "read" } }); + + const outcome = await handleBenchCommand(command("/bench", "MEMBER"), io); + + expect(outcome).toMatchObject({ kind: "ignored" }); + expect(calls.reruns).toEqual([]); + }); + + test("lets the repository owner run it without a permission lookup result", async () => { + const { io } = setup({ permissions: {} }); + + expect((await handleBenchCommand(command("/bench", "OWNER"), io)).kind).toBe("rerun"); + }); + + test.each([ + ["a closed pull request", pr({ state: "closed" })], + ["a fork", pr({ headRepoFullName: "someone/cli" })], + ["a deleted head repository", pr({ headRepoFullName: "" })], + ["a pull request into another branch", pr({ baseRef: "feature/stack-base" })], + ])("ignores /bench on %s", async (_name, head) => { + const { io, calls } = setup({ pr: head }); + + const outcome = await handleBenchCommand(command("/bench"), io); + + expect(outcome.kind).toBe("ignored"); + expect(calls).toEqual({ reactions: [], replies: [], reruns: [] }); + }); + + test("tells the pull request there is no run when none exists for the head commit", async () => { + const { io, calls } = setup({ runs: [] }); + + const outcome = await handleBenchCommand(command("/bench"), io); + + expect(outcome.kind).toBe("replied"); + expect(calls.replies).toHaveLength(1); + expect(calls.replies[0]).toContain("no benchmark run for commit `aaaaaaa`"); + expect(calls.reruns).toEqual([]); + expect(calls.reactions).toEqual([]); + }); + + test("says a benchmark is already running instead of re-running it", async () => { + const { io, calls } = setup({ runs: [run({ status: "in_progress" })] }); + + await handleBenchCommand(command("/bench"), io); + + expect(calls.replies[0]).toContain("already running"); + expect(calls.reruns).toEqual([]); + expect(calls.reactions).toEqual([]); + }); + + test("says the re-run could not start when the run is older than 30 days", async () => { + const old = new Date(NOW.getTime() - RERUN_WINDOW_MS - 1000).toISOString(); + const { io, calls } = setup({ runs: [run({ createdAt: old })] }); + + await handleBenchCommand(command("/bench"), io); + + expect(calls.replies[0]).toContain("older than 30 days"); + expect(calls.reruns).toEqual([]); + }); + + test("surfaces GitHub's refusal in a reply and does not react", async () => { + const { io, calls } = setup({ + rerunError: new GithubRequestError(403, "Resource not accessible by integration", "u"), + }); + + const outcome = await handleBenchCommand(command("/bench"), io); + + expect(outcome.kind).toBe("replied"); + expect(calls.replies[0]).toContain("403"); + expect(calls.replies[0]).toContain("Resource not accessible by integration"); + expect(calls.reactions).toEqual([]); + }); + + test("keeps an accepted re-run when the reaction fails", async () => { + const { io } = setup(); + io.reactToComment = async () => { + throw new Error("boom"); + }; + + expect(await handleBenchCommand(command("/bench"), io)).toEqual({ kind: "rerun", runId: 11 }); + }); + + test("explains that a draft pull request is not benchmarked", async () => { + const { io, calls } = setup({ pr: pr({ draft: true }) }); + + await handleBenchCommand(command("/bench"), io); + + expect(calls.replies[0]).toContain("draft"); + expect(calls.reruns).toEqual([]); + }); +}); diff --git a/.github/scripts/cli-bench/rerun.ts b/.github/scripts/cli-bench/rerun.ts new file mode 100644 index 0000000000..b0020185b8 --- /dev/null +++ b/.github/scripts/cli-bench/rerun.ts @@ -0,0 +1,231 @@ +/** + * `/bench` handler: re-runs the benchmark workflow run of a pull request's current head commit. + * + * Runs default branch code and never checks out or runs pull request code. The commenter must be + * the repository owner or hold write access, and the pull request must be an open, ready one from + * this repository into `develop`. The command re-runs the existing `pull_request` run, which + * benchmarks and updates the comment; the pull request gets a reply when no run can be re-run. + * Run in CI as: `bun .github/scripts/cli-bench/rerun.ts`. + */ +import { appendFileSync } from "node:fs"; + +import { fetchAuthorPermission, WRITE_PERMISSIONS } from "../contribution-gate.ts"; +import { githubFetch, GithubRequestError, requireEnv } from "./github.ts"; + +export const BENCH_COMMAND = "/bench"; +/** Base branches the benchmark runs for; keep in sync with the workflow's `branches:` filter. */ +export const BENCH_BASE_BRANCHES: ReadonlyArray = ["develop"]; +/** The workflow file whose runs `/bench` re-runs. */ +export const WORKFLOW_FILE = "cli-bench.yml"; +/** GitHub refuses to re-run a workflow run older than this. */ +export const RERUN_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; + +export interface BenchPr { + state: "open" | "closed"; + draft: boolean; + headSha: string; + /** `owner/name` of the head repository, empty when it was deleted. */ + headRepoFullName: string; + baseRepoFullName: string; + baseRef: string; +} + +export interface WorkflowRun { + id: number; + status: string; + createdAt: string; +} + +export interface CommandInput { + prNumber: number; + comment: { id: number; authorLogin: string; authorAssociation: string; body: string }; + now: Date; +} + +export interface CommandIo { + fetchPr: (prNumber: number) => Promise; + fetchPermission: (login: string) => Promise; + /** Runs of the benchmark workflow triggered by `pull_request` for this commit, any order. */ + listRuns: (headSha: string) => Promise>; + rerun: (runId: number) => Promise; + reactToComment: (commentId: number) => Promise; + replyToPr: (prNumber: number, body: string) => Promise; +} + +export type CommandOutcome = + | { kind: "ignored"; reason: string } + | { kind: "replied"; reason: string } + | { kind: "rerun"; runId: number }; + +const short = (sha: string) => sha.slice(0, 7); + +/** A GitHub error message as inline code: one short line with no backticks. */ +const quoted = (detail: string) => + `\`${detail.replaceAll("`", "'").replaceAll(/\s+/gu, " ").trim().slice(0, 200)}\``; + +export const replies = { + noRun: (sha: string) => + `There is no benchmark run for commit \`${short(sha)}\`. Runs start on a push that changes the ` + + "CLI, its packages, or the benchmark workflow, and GitHub's path filter looks at only the " + + "first 300 changed files, so this pull request may not have one.", + running: (sha: string) => + `A benchmark is already running for commit \`${short(sha)}\`. The comment updates when it finishes.`, + draft: () => `This pull request is a draft. The benchmark runs once it is ready for review.`, + expired: (sha: string) => + `The benchmark could not be re-run for commit \`${short(sha)}\`: its run is older than 30 days. ` + + "Push a new commit to benchmark again.", + refused: (sha: string, status: number, detail: string) => + `The benchmark could not be re-run for commit \`${short(sha)}\`: GitHub refused the request ` + + `(${status} ${quoted(detail)}).`, +}; + +/** Handles one `/bench` comment; a rejected command is ignored without a reply. */ +export async function handleBenchCommand( + input: CommandInput, + io: CommandIo, +): Promise { + const { comment, prNumber } = input; + // The workflow's `if:` only pre-filters on `startsWith`, so `/bench full` must be rejected here. + if (comment.body.trim() !== BENCH_COMMAND) { + return { kind: "ignored", reason: `Comment is not exactly ${BENCH_COMMAND}.` }; + } + // Only the repository OWNER skips the lookup: MEMBER and COLLABORATOR do not imply push access. + const permission = await io.fetchPermission(comment.authorLogin); + const authorized = + comment.authorAssociation === "OWNER" || + (permission !== undefined && WRITE_PERMISSIONS.has(permission)); + if (!authorized) { + return { + kind: "ignored", + reason: + `Commenter @${comment.authorLogin} is not authorized to run ${BENCH_COMMAND} ` + + `(author_association=${comment.authorAssociation}, permission=${permission ?? "n/a"}).`, + }; + } + const pr = await io.fetchPr(prNumber); + if (pr.state === "closed") return { kind: "ignored", reason: `PR #${prNumber} is closed.` }; + if (pr.headRepoFullName !== pr.baseRepoFullName) { + return { + kind: "ignored", + reason: `${BENCH_COMMAND} runs only on pull requests from this repository.`, + }; + } + if (!BENCH_BASE_BRANCHES.includes(pr.baseRef)) { + return { + kind: "ignored", + reason: `${BENCH_COMMAND} runs only on pull requests into ${BENCH_BASE_BRANCHES.join(" or ")}.`, + }; + } + + const reply = async (text: string): Promise => { + await io.replyToPr(prNumber, text); + return { kind: "replied", reason: text }; + }; + if (pr.draft) return reply(replies.draft()); + + const latest = [...(await io.listRuns(pr.headSha))].sort((a, b) => + b.createdAt.localeCompare(a.createdAt), + )[0]; + if (latest === undefined) return reply(replies.noRun(pr.headSha)); + if (latest.status !== "completed") return reply(replies.running(pr.headSha)); + if (input.now.getTime() - Date.parse(latest.createdAt) > RERUN_WINDOW_MS) { + return reply(replies.expired(pr.headSha)); + } + + try { + await io.rerun(latest.id); + } catch (error) { + if (error instanceof GithubRequestError) { + return reply(replies.refused(pr.headSha, error.status, error.detail)); + } + throw error; + } + // Cosmetic: a failed reaction must never hide an accepted re-run. + try { + await io.reactToComment(comment.id); + } catch (error) { + console.warn(`Could not react to comment ${comment.id}: ${String(error)}`); + } + return { kind: "rerun", runId: latest.id }; +} + +async function main(): Promise { + const token = requireEnv("GITHUB_TOKEN"); + const [owner = "", repo = ""] = requireEnv("GITHUB_REPOSITORY").split("/"); + const base = `https://api.github.com/repos/${owner}/${repo}`; + const prNumber = Number(requireEnv("PR_NUMBER")); + + const io: CommandIo = { + fetchPr: async (n) => { + const value = (await (await githubFetch(`${base}/pulls/${n}`, token)).json()) as { + state: "open" | "closed"; + draft: boolean; + head: { sha: string; repo: { full_name: string } | null }; + base: { ref: string; repo: { full_name: string } }; + }; + return { + state: value.state, + draft: value.draft, + headSha: value.head.sha, + headRepoFullName: value.head.repo?.full_name ?? "", + baseRepoFullName: value.base.repo.full_name, + baseRef: value.base.ref, + }; + }, + fetchPermission: (login) => fetchAuthorPermission(token, owner, repo, login), + listRuns: async (headSha) => { + const url = + `${base}/actions/workflows/${WORKFLOW_FILE}/runs` + + `?event=pull_request&head_sha=${headSha}&per_page=100`; + const payload = (await (await githubFetch(url, token)).json()) as { + workflow_runs: Array<{ id: number; status: string; created_at: string; head_sha: string }>; + }; + return payload.workflow_runs + .filter((run) => run.head_sha === headSha) + .map((run) => ({ id: run.id, status: run.status, createdAt: run.created_at })); + }, + rerun: async (runId) => { + await githubFetch(`${base}/actions/runs/${runId}/rerun`, token, { method: "POST" }); + }, + reactToComment: async (commentId) => { + await githubFetch(`${base}/issues/comments/${commentId}/reactions`, token, { + method: "POST", + body: JSON.stringify({ content: "eyes" }), + }); + }, + replyToPr: async (n, body) => { + await githubFetch(`${base}/issues/${n}/comments`, token, { + method: "POST", + body: JSON.stringify({ body }), + }); + }, + }; + + const outcome = await handleBenchCommand( + { + prNumber, + comment: { + id: Number(requireEnv("COMMENT_ID")), + authorLogin: requireEnv("COMMENT_AUTHOR_LOGIN"), + authorAssociation: requireEnv("COMMENT_AUTHOR_ASSOCIATION"), + body: requireEnv("COMMENT_BODY"), + }, + now: new Date(), + }, + io, + ); + const message = + outcome.kind === "rerun" + ? `Re-running benchmark run ${outcome.runId} for PR #${prNumber}.` + : `${outcome.kind === "replied" ? "Replied on" : "Ignored command on"} PR #${prNumber}: ${outcome.reason}`; + console.log(message); + const summaryFile = process.env.GITHUB_STEP_SUMMARY; + if (summaryFile) appendFileSync(summaryFile, `${message}\n`); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/.github/scripts/cli-bench/resolve.test.ts b/.github/scripts/cli-bench/resolve.test.ts index 31dcf5a83b..7ac55287b2 100644 --- a/.github/scripts/cli-bench/resolve.test.ts +++ b/.github/scripts/cli-bench/resolve.test.ts @@ -1,133 +1,47 @@ import { describe, expect, test } from "bun:test"; -import { type BenchPr, resolveBench, type ResolveIo } from "./resolve.ts"; +import { BENCH_MARKER, FAILURE_TITLE } from "./post-comment.ts"; +import { resolveBench, SKIP_REASON } from "./resolve.ts"; -const REPO = "supabase/cli"; -const pr = (overrides: Partial = {}): BenchPr => ({ - state: "open", - headSha: "a".repeat(40), - headRepoFullName: REPO, - baseRepoFullName: REPO, - baseRef: "develop", - ...overrides, -}); - -function io(permissions: Record, head: BenchPr = pr()) { - const reactions: Array = []; - const impl: ResolveIo = { - fetchPr: async () => head, - fetchPermission: async (login) => permissions[login], - reactToComment: async (id) => { - reactions.push(id); - }, - }; - return { impl, reactions }; -} - -const command = (body: string, association = "COLLABORATOR", login = "maintainer") => - ({ - eventName: "issue_comment", - prNumber: 7, - comment: { id: 99, authorLogin: login, authorAssociation: association, body }, - }) as const; +const BOT = "github-actions[bot]"; +const reported = { + id: 5, + body: `${BENCH_MARKER}\n## CLI benchmark: no changes found`, + authorLogin: BOT, +}; describe("resolveBench", () => { - test("runs a pull_request event for its own head commit", async () => { - const { impl } = io({}); - - const result = await resolveBench( - { eventName: "pull_request", prNumber: 7, pr: pr({ baseRef: "next" }) }, - impl, - ); - - expect(result).toMatchObject({ - shouldRun: true, - trigger: "auto", - baseRef: "next", - isFork: false, - }); + test("benchmarks the first push of a pull request that has no comment yet", () => { + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [] }).shouldRun).toBe(true); }); - test("marks a pull request from another repository as a fork", async () => { - const { impl } = io({}); - - const result = await resolveBench( - { eventName: "pull_request", prNumber: 7, pr: pr({ headRepoFullName: "someone/cli" }) }, - impl, - ); + test("skips a later push while the comment reports a result, and says how to refresh it", () => { + const result = resolveBench({ runAttempt: 1, isFork: false, comments: [reported] }); - expect(result.isFork).toBe(true); + expect(result).toEqual({ shouldRun: false, skipReason: SKIP_REASON }); + expect(SKIP_REASON).toContain("`/bench`"); }); - test("accepts /bench from a writer, runs the current head, and reacts", async () => { - const { impl, reactions } = io({ maintainer: "write" }, pr({ headSha: "b".repeat(40) })); - - const result = await resolveBench(command("/bench\n"), impl); + test("benchmarks again when the comment reports a failed run", () => { + const failed = { ...reported, body: `${BENCH_MARKER}\n${FAILURE_TITLE}` }; - expect(result).toMatchObject({ shouldRun: true, trigger: "command", headSha: "b".repeat(40) }); - expect(reactions).toEqual([99]); + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [failed] }).shouldRun).toBe(true); }); - test.each(["/bench full", "/benchmark", "please /bench", "/Bench"])( - "ignores %p", - async (body) => { - const { impl, reactions } = io({ maintainer: "write" }); - - const result = await resolveBench(command(body), impl); - - expect(result.shouldRun).toBe(false); - expect(reactions).toEqual([]); - }, - ); - - test("rejects a commenter without write permission even as a MEMBER", async () => { - const { impl, reactions } = io({ maintainer: "read" }); - - const result = await resolveBench(command("/bench", "MEMBER"), impl); - - expect(result.shouldRun).toBe(false); - expect(result.skipReason).toContain("not authorized"); - expect(reactions).toEqual([]); - }); - - test("lets the repository owner run it without a permission lookup result", async () => { - const { impl } = io({}); - - const result = await resolveBench(command("/bench", "OWNER"), impl); - - expect(result.shouldRun).toBe(true); - }); - - test("does not run on a closed pull request", async () => { - const { impl } = io({ maintainer: "admin" }, pr({ state: "closed" })); - - const result = await resolveBench(command("/bench"), impl); - - expect(result.shouldRun).toBe(false); + test("benchmarks every re-run even when the comment reports a result", () => { + expect(resolveBench({ runAttempt: 2, isFork: false, comments: [reported] }).shouldRun).toBe( + true, + ); }); - test.each([ - ["a fork", "someone/cli"], - ["a deleted head repository", ""], - ])("does not run /bench on %s and does not react", async (_name, headRepo) => { - const { impl, reactions } = io({ maintainer: "write" }, pr({ headRepoFullName: headRepo })); - - const result = await resolveBench(command("/bench"), impl); + test("ignores a marker pasted by someone else", () => { + const forged = { ...reported, authorLogin: "mallory" }; - expect(result.shouldRun).toBe(false); - expect(result.skipReason).toContain("from this repository"); - expect(reactions).toEqual([]); + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [forged] }).shouldRun).toBe(true); }); - test.each([ - ["a pull request into next", "next"], - ["a stacked pull request into a feature branch", "feature/stack-base"], - ])("does not run /bench on %s and does not react", async (_name, baseRef) => { - const { impl, reactions } = io({ maintainer: "write" }, pr({ baseRef })); - - const result = await resolveBench(command("/bench"), impl); - - expect(result.shouldRun).toBe(false); - expect(result.skipReason).toContain("into develop"); - expect(reactions).toEqual([]); + test("benchmarks every push of a fork pull request, which never has a comment", () => { + expect(resolveBench({ runAttempt: 1, isFork: true, comments: [reported] }).shouldRun).toBe( + true, + ); }); }); diff --git a/.github/scripts/cli-bench/resolve.ts b/.github/scripts/cli-bench/resolve.ts index 1d385bcd8d..03882d2450 100644 --- a/.github/scripts/cli-bench/resolve.ts +++ b/.github/scripts/cli-bench/resolve.ts @@ -1,204 +1,61 @@ /** - * CLI benchmark resolver: decides whether the benchmark runs and for which commit. + * CLI benchmark resolver for `pull_request` runs: decides whether the benchmark job runs. * - * Two triggers: - * - `pull_request`: runs for the event's head commit; the event payload is the only input. - * - a maintainer's `/bench` comment on a pull request from this repository: runs for the pull request's head commit - * when this workflow accepts the command. The comment must be exactly `/bench`, and the - * commenter must be the repository owner or hold write access. - * - * `resolveBench` is pure orchestration with injected I/O; `main()` wires the real GitHub calls and - * writes the outputs the workflow reads. Run in CI as: `bun .github/scripts/cli-bench/resolve.ts`. + * The first run on a pull request benchmarks and comments. Later pushes skip while the comment + * exists and reports a result; a re-run (`run_attempt` above 1), a missing comment, a failed first + * run, or a fork pull request, which never has a comment, benchmarks. Needs read access only. + * Run in CI as: `bun .github/scripts/cli-bench/resolve.ts`. */ import { appendFileSync } from "node:fs"; -import { fetchAuthorPermission, WRITE_PERMISSIONS } from "../contribution-gate.ts"; -import { githubFetch, requireEnv } from "./github.ts"; - -export const BENCH_COMMAND = "/bench"; -/** Base branches the benchmark runs for; keep in sync with the workflow's `branches:` filter. */ -export const BENCH_BASE_BRANCHES: ReadonlyArray = ["develop"]; - -export type Trigger = "auto" | "command"; - -export interface BenchPr { - state: "open" | "closed"; - headSha: string; - /** `owner/name` of the head repository, empty when it was deleted. */ - headRepoFullName: string; - baseRepoFullName: string; - baseRef: string; -} - -export type ResolveInput = - | { eventName: "pull_request"; prNumber: number; pr: BenchPr } - | { - eventName: "issue_comment"; - prNumber: number; - comment: { id: number; authorLogin: string; authorAssociation: string; body: string }; - }; +import { listIssueComments, requireEnv, type IssueComment } from "./github.ts"; +import { FAILURE_TITLE, findBenchComment } from "./post-comment.ts"; -export interface ResolveIo { - fetchPr: (prNumber: number) => Promise; - fetchPermission: (login: string) => Promise; - reactToComment: (commentId: number) => Promise; +export interface ResolveInput { + runAttempt: number; + isFork: boolean; + comments: ReadonlyArray; } export interface ResolveResult { shouldRun: boolean; + /** Shown in the job summary when the benchmark is skipped. */ skipReason?: string; - trigger: Trigger; - headSha: string; - baseRef: string; - /** Whether the head commit lives in a repository other than the base repository. */ - isFork: boolean; -} - -const skipped = (trigger: Trigger, skipReason: string): ResolveResult => ({ - shouldRun: false, - skipReason, - trigger, - headSha: "", - baseRef: "", - isFork: false, -}); - -const accepted = (trigger: Trigger, pr: BenchPr): ResolveResult => ({ - shouldRun: true, - trigger, - headSha: pr.headSha, - baseRef: pr.baseRef, - isFork: pr.headRepoFullName !== pr.baseRepoFullName, -}); - -/** Decides whether the benchmark runs; never throws for an unauthorized or malformed command. */ -export async function resolveBench(input: ResolveInput, io: ResolveIo): Promise { - if (input.eventName === "pull_request") return accepted("auto", input.pr); - - const { comment } = input; - // The workflow's `if:` only pre-filters on `startsWith`, so `/bench full` must be rejected here. - if (comment.body.trim() !== BENCH_COMMAND) { - return skipped("command", `Comment is not exactly ${BENCH_COMMAND}.`); - } - // Only the repository OWNER skips the lookup: MEMBER and COLLABORATOR do not imply push access. - const permission = await io.fetchPermission(comment.authorLogin); - const authorized = - comment.authorAssociation === "OWNER" || - (permission !== undefined && WRITE_PERMISSIONS.has(permission)); - if (!authorized) { - return skipped( - "command", - `Commenter @${comment.authorLogin} is not authorized to run ${BENCH_COMMAND} ` + - `(author_association=${comment.authorAssociation}, permission=${permission ?? "n/a"}).`, - ); - } - const pr = await io.fetchPr(input.prNumber); - if (pr.state === "closed") return skipped("command", `PR #${input.prNumber} is closed.`); - // Only same-repository branches run: their authors already get repository secrets on a pull request. - if (pr.headRepoFullName !== pr.baseRepoFullName) { - return skipped("command", `${BENCH_COMMAND} runs only on pull requests from this repository.`); - } - - if (!BENCH_BASE_BRANCHES.includes(pr.baseRef)) { - return skipped( - "command", - `${BENCH_COMMAND} runs only on pull requests into ${BENCH_BASE_BRANCHES.join(" or ")}.`, - ); - } - - // Cosmetic: a failed reaction must never fail an authorized run. - try { - await io.reactToComment(comment.id); - } catch (error) { - console.warn(`Could not react to comment ${comment.id}: ${String(error)}`); - } - return accepted("command", pr); } -async function fetchPr(token: string, base: string, prNumber: number): Promise { - const value = (await (await githubFetch(`${base}/pulls/${prNumber}`, token)).json()) as { - state: "open" | "closed"; - head: { sha: string; repo: { full_name: string } | null }; - base: { ref: string; repo: { full_name: string } }; - }; - return { - state: value.state, - headSha: value.head.sha, - headRepoFullName: value.head.repo?.full_name ?? "", - baseRepoFullName: value.base.repo.full_name, - baseRef: value.base.ref, - }; -} +export const SKIP_REASON = + "A benchmark comment already exists on this pull request. Comment `/bench` to refresh it."; -/** Writes each output with a random delimiter so a value can never inject extra output lines. */ -function writeOutputs(result: ResolveResult, prNumber: number): void { - const entries: Record = { - should_run: String(result.shouldRun), - pr_number: String(prNumber), - head_sha: result.headSha, - base_ref: result.baseRef, - is_fork: String(result.isFork), - trigger: result.trigger, - }; - const lines = Object.entries(entries).map(([name, value]) => { - const delimiter = `ghadelim_${crypto.randomUUID()}`; - return `${name}<<${delimiter}\n${value}\n${delimiter}`; - }); - appendFileSync(requireEnv("GITHUB_OUTPUT"), `${lines.join("\n")}\n`); +/** Skips only a first attempt on a pull request whose benchmark comment reports a result. */ +export function resolveBench(input: ResolveInput): ResolveResult { + if (input.isFork || input.runAttempt > 1) return { shouldRun: true }; + const existing = findBenchComment(input.comments); + if (existing === undefined || existing.body.includes(FAILURE_TITLE)) return { shouldRun: true }; + return { shouldRun: false, skipReason: SKIP_REASON }; } async function main(): Promise { const token = requireEnv("GITHUB_TOKEN"); - const [owner, repo] = requireEnv("GITHUB_REPOSITORY").split("/"); - const base = `https://api.github.com/repos/${owner}/${repo}`; + const base = `https://api.github.com/repos/${requireEnv("GITHUB_REPOSITORY")}`; const prNumber = Number(requireEnv("PR_NUMBER")); - const eventName = requireEnv("EVENT_NAME"); - - const io: ResolveIo = { - fetchPr: (n) => fetchPr(token, base, n), - fetchPermission: (login) => fetchAuthorPermission(token, owner!, repo!, login), - reactToComment: async (commentId) => { - await githubFetch(`${base}/issues/comments/${commentId}/reactions`, token, { - method: "POST", - body: JSON.stringify({ content: "eyes" }), - }); - }, - }; - - let input: ResolveInput; - if (eventName === "pull_request") { - input = { - eventName, - prNumber, - pr: { - state: "open", - headSha: requireEnv("HEAD_SHA"), - headRepoFullName: process.env.HEAD_REPO ?? "", - baseRepoFullName: requireEnv("GITHUB_REPOSITORY"), - baseRef: requireEnv("BASE_REF"), - }, - }; - } else if (eventName === "issue_comment") { - input = { - eventName, - prNumber, - comment: { - id: Number(requireEnv("COMMENT_ID")), - authorLogin: requireEnv("COMMENT_AUTHOR_LOGIN"), - authorAssociation: requireEnv("COMMENT_AUTHOR_ASSOCIATION"), - body: requireEnv("COMMENT_BODY"), - }, - }; - } else { - throw new Error(`Unsupported EVENT_NAME "${eventName}".`); + const runAttempt = Number(requireEnv("RUN_ATTEMPT")); + const isFork = requireEnv("HEAD_REPO") !== requireEnv("GITHUB_REPOSITORY"); + + let comments: ReadonlyArray = []; + if (!isFork && runAttempt === 1) { + // A comment that cannot be listed must not hide a benchmark: run it instead. + comments = await listIssueComments(token, base, prNumber).catch((error: unknown) => { + console.warn(`Could not list comments on PR #${prNumber}: ${String(error)}`); + return []; + }); } - const result = await resolveBench(input, io); + const result = resolveBench({ runAttempt, isFork, comments }); console.log( `CLI benchmark resolve for PR #${prNumber}: should_run=${result.shouldRun} ` + - `trigger=${result.trigger}${result.skipReason ? ` (${result.skipReason})` : ""}`, + `attempt=${runAttempt} fork=${isFork}`, ); - writeOutputs(result, prNumber); + appendFileSync(requireEnv("GITHUB_OUTPUT"), `should_run=${result.shouldRun}\n`); const summaryFile = process.env.GITHUB_STEP_SUMMARY; if (result.skipReason && summaryFile) appendFileSync(summaryFile, `${result.skipReason}\n`); } diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml index e10f393f9a..73f76a6ac5 100644 --- a/.github/workflows/cli-bench.yml +++ b/.github/workflows/cli-bench.yml @@ -5,10 +5,14 @@ run-name: "CLI benchmark #${{ github.event.pull_request.number || github.event.i # workload runs against the replay server, so no Docker, database, or network is involved. The # job never blocks a pull request and is not a required check. # -# Each run writes a job summary. The first run on a pull request from this repository also posts -# one comment tagged ``; later pushes leave it alone. A maintainer commenting -# exactly `/bench` re-runs the benchmark on the pull request's current head and updates that -# comment. See apps/cli/docs/tracing-monitoring.md. +# The first push to a pull request from this repository benchmarks and posts one comment tagged +# ``; later pushes skip while that comment reports a result. A maintainer +# commenting exactly `/bench` re-runs the run for the pull request's current head commit, as does +# "Re-run all jobs" in the Actions UI: a re-run benchmarks again and updates the comment, keeping +# earlier runs in a collapsed list. Pull requests from forks benchmark every push and write a job +# summary only. A pull request that changes no benchmarked path, or more than the 300 files +# GitHub's path filter inspects, has no run to re-run, and `/bench` says so. +# See apps/cli/docs/tracing-monitoring.md. on: pull_request: types: @@ -26,9 +30,7 @@ on: - "turbo.json" - ".github/workflows/cli-bench.yml" - ".github/scripts/cli-bench/**" - # A `/bench` comment on a pull request starts a run. The resolve job checks who commented and - # which commit it applies to; for `/bench` the resolve and comment jobs run default branch code, - # and the job that builds and runs pull request code holds no write permission. + # `/bench` runs default branch code only: it never checks out or runs pull request code. # jactionlint ignore=dangerous-triggers issue_comment: types: @@ -36,43 +38,26 @@ on: permissions: {} -# Comments that are not a `/bench` from a maintainer get a per-run group, so they cannot cancel an -# in-flight benchmark; `/bench full` is one of them. +# Comment runs get a group of their own, so a comment can never cancel a pull request's benchmark +# nor be cancelled by one. concurrency: group: >- - cli-bench-${{ github.event.pull_request.number || github.event.issue.number }}-${{ - (github.event_name == 'issue_comment' && (github.event.comment.body != '/bench' || - !contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association))) - && github.run_id || 'run' }} + cli-bench-${{ github.event.pull_request.number || github.event.issue.number }}${{ + github.event_name == 'issue_comment' && format('-comment-{0}', github.run_id) || '' }} cancel-in-progress: true jobs: + # Decides whether this push benchmarks. Read-only: it only lists the pull request's comments. resolve: name: Resolve - # Cheap, non-authoritative pre-filter; resolve.ts does the exact command match and the - # effective-permission lookup. - if: > - (github.event_name == 'pull_request' && github.event.pull_request.draft == false) || - (github.event_name == 'issue_comment' && - github.event.issue.pull_request != null && - startsWith(github.event.comment.body, '/bench') && - contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association)) + if: github.event_name == 'pull_request' && github.event.pull_request.draft == false runs-on: ubuntu-latest timeout-minutes: 5 - # Reacts to the accepted command. Checkout uses `github.sha`: default branch code for - # `issue_comment`, the pull request merge commit for `pull_request`, where a fork's token is - # read-only whatever is declared here. permissions: contents: read - pull-requests: write + pull-requests: read outputs: should_run: ${{ steps.resolve.outputs.should_run }} - pr_number: ${{ steps.resolve.outputs.pr_number }} - # Every job benchmarks this commit, so a push during the run is never reported as measured. - head_sha: ${{ steps.resolve.outputs.head_sha }} - base_ref: ${{ steps.resolve.outputs.base_ref }} - is_fork: ${{ steps.resolve.outputs.is_fork }} - trigger: ${{ steps.resolve.outputs.trigger }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -86,16 +71,46 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} - EVENT_NAME: ${{ github.event_name }} - PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} - BASE_REF: ${{ github.event.pull_request.base.ref }} + RUN_ATTEMPT: ${{ github.run_attempt }} + run: bun .github/scripts/cli-bench/resolve.ts + + # `/bench`: re-runs this workflow's run for the pull request's current head commit. The only job + # with `actions: write`; it runs default branch code and no pull request code. + rerun: + name: Re-run benchmark + # Cheap, non-authoritative pre-filter; rerun.ts does the exact command match and the + # effective-permission lookup. + if: > + github.event_name == 'issue_comment' && + github.event.issue.pull_request != null && + startsWith(github.event.comment.body, '/bench') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: ".bun-version" + no-cache: true + - name: Re-run + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.issue.number }} COMMENT_ID: ${{ github.event.comment.id }} COMMENT_AUTHOR_LOGIN: ${{ github.event.comment.user.login }} COMMENT_AUTHOR_ASSOCIATION: ${{ github.event.comment.author_association }} COMMENT_BODY: ${{ github.event.comment.body }} - run: bun .github/scripts/cli-bench/resolve.ts + run: bun .github/scripts/cli-bench/rerun.ts bench: name: Benchmark CLI against merge base @@ -109,17 +124,14 @@ jobs: - name: Checkout pull request head uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ needs.resolve.outputs.head_sha }} + ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 persist-credentials: false - # `/bench` runs only for same-repository branches. It skips the dependency cache so it saves - # no package cache under the default branch's scope. - name: Setup uses: ./.github/actions/setup with: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} - dependency-cache: ${{ needs.resolve.outputs.trigger == 'auto' && 'true' || 'false' }} - name: Prepare benchmark directory shell: bash @@ -142,7 +154,7 @@ jobs: if: steps.branch_build.outcome == 'success' env: HUSKY: "0" - BASE_REF: ${{ needs.resolve.outputs.base_ref }} + BASE_REF: ${{ github.event.pull_request.base.ref }} shell: bash run: | merge_base="$(git merge-base HEAD "origin/$BASE_REF")" @@ -150,7 +162,7 @@ jobs: echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" git worktree add "$RUNNER_TEMP/base-src" "$merge_base" cd "$RUNNER_TEMP/base-src" - # The setup action writes this file only when the Dependency Firewall token is available. + # The setup action writes this file only when the Dependency Firewall token is available. if [ -f "$RUNNER_TEMP/dependency-firewall.npmrc" ]; then export NPM_CONFIG_USERCONFIG="$RUNNER_TEMP/dependency-firewall.npmrc" fi @@ -205,8 +217,8 @@ jobs: BASE_BUILD: ${{ steps.base_build.outcome == 'success' && 'ok' || 'failed' }} REPLAY: ${{ steps.replay.outcome == 'success' && 'ok' || 'failed' }} MERGE_BASE: ${{ steps.base_build.outputs.merge_base }} - HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} - BASE_REF: ${{ needs.resolve.outputs.base_ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} shell: bash run: | bun apps/cli/scripts/bench-summary.ts \ @@ -248,14 +260,13 @@ jobs: # post never fails the workflow. if: >- ${{ !cancelled() && needs.resolve.outputs.should_run == 'true' && needs.bench.result == 'success' && - (needs.resolve.outputs.trigger == 'command' || needs.resolve.outputs.is_fork == 'false') }} + github.event.pull_request.head.repo.full_name == github.repository }} runs-on: ubuntu-latest timeout-minutes: 5 continue-on-error: true - # The only job that can write to the pull request. Checkout uses `github.sha`: default branch - # code for `/bench`, and for same-repository `pull_request` runs the merge commit, which its - # author can already run with this token by editing the workflow. It never builds or runs the - # benchmarked binaries and reads the rendered markdown as data. + # The only pull_request job that can write to the pull request. It checks out the merge commit, + # which its author can already run with this token by editing the workflow. It never builds or + # runs the benchmarked binaries and reads the rendered markdown as data. permissions: contents: read pull-requests: write @@ -277,10 +288,11 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} - PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} - MODE: ${{ needs.resolve.outputs.trigger == 'command' && 'upsert' || 'create-if-missing' }} + PR_NUMBER: ${{ github.event.pull_request.number }} + MODE: ${{ github.run_attempt > 1 && 'upsert' || 'create-if-missing' }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} BODY_FILE: ${{ runner.temp }}/bench-comment/comment.md - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} run: | if [ ! -f "$BODY_FILE" ]; then echo "The benchmark job produced no comment." diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index f6b7083c75..fe78022672 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -94,29 +94,36 @@ summary instead of failing the job: a failed branch or base build or a replay se start replaces the comparison with a notice naming the step. The workloads use no Docker, database, or network, and no prompts. -The first run on a pull request from this repository also posts one comment, tagged with the hidden -marker ``. Later pushes leave that comment alone. Pull requests from forks get -the job summary only, because their token is read-only. - -A maintainer can comment exactly `/bench` on a pull request to run the benchmark again on its -current head commit against the merge base with its base branch, `develop`. The comment -is updated in place, or created if it is missing. Only the repository owner and people with write -access can run it, and only on pull requests into `develop` from this repository; the command must be the whole comment, so `/bench full` (reserved for a future -heavier tier) and anything else is ignored. The commenter's request is acknowledged with an eyes -reaction. Workflows triggered by comments run from the default branch, so `/bench` works only once -the workflow is on `develop`. -`/bench` works only on pull requests from this repository. Those branches come from people with -write access, who already get repository secrets on a normal pull request run, so `/bench` uses the -same job, setup action, and dependency firewall token. `/bench` runs turn off the dependency cache so -they do not save a package cache under the default branch's scope. The toolchain cache is saved -before any pull request code runs. A `/bench` run's pull request code could still write a cache -entry under the default branch's scope through the runner's own cache token; this is accepted -because only same-repository branches from people with write access can be run this way. `resolve` -and `comment` check out `github.sha`: for `/bench` that is the default branch, so they run default -branch scripts. On `pull_request` runs it is the merge commit, which is the workflow's own code; a -fork's token is read-only there (this assumes the repository setting that sends write tokens to fork pull request workflows stays off) and same-repository authors can already edit the workflow. `comment` -is the only job with write permission and never builds or runs the benchmarked binaries: it downloads -the rendered markdown, sanitizes it, and posts it. +The first push to a pull request from this repository benchmarks and posts one comment, tagged with +the hidden marker ``. Later pushes skip the benchmark, and the job summary says +so, while that comment reports a result; a comment that reports a failed run is benchmarked again +on the next push. Pull requests from forks have a read-only token, so they get the job summary only +and benchmark every push. + +A maintainer can comment exactly `/bench` to run the benchmark again on the pull request's current +head commit. The command re-runs that commit's `pull_request` workflow run, as "Re-run all jobs" in +the Actions UI does, and the re-run updates the comment in place, or creates it if it is missing. +The comment keeps a collapsed "Previous runs" list below the report with the commit, time (UTC), +headline, and link of each earlier run, newest first and up to 20, stored in a hidden marker in the +comment. Only the repository owner and people with write access can run it, and only on open, +non-draft pull requests into `develop` from this repository; the command must be the whole comment, +so `/bench full` and anything else is ignored. An eyes reaction on the comment means GitHub accepted +the re-run and a benchmark is running; the result arrives as the updated benchmark comment, whether +it succeeded or failed. When no re-run can start, the pull request gets a reply saying why: a +benchmark is already running for the commit, the run is older than 30 days, GitHub refused the +re-run (the reply includes the status and message), or there is no run for the commit. + +Known gap: a pull request whose changes match none of the workflow's path filters, or that changes +more files than GitHub's path filter inspects (300), gets no run, so there is nothing for `/bench` +to re-run and it replies that there is none. + +Workflows triggered by comments run from the default branch, so `/bench` works only once the +workflow is on `develop`. The `/bench` job runs default branch code only: it never checks out or +runs pull request code, because a comment-triggered run shares the default branch's cache scope. +Benchmarks always run in the `pull_request` run. `resolve` (read-only) decides whether a push +benchmarks; `rerun` is the only job with `actions: write`. `bench` has read-only permissions, and +`comment` is the only `pull_request` job that can write to the pull request. It never builds or +runs the benchmarked binaries: it downloads the rendered markdown, sanitizes it, and posts it. To run it locally with two binaries (for example builds from two commits): @@ -139,7 +146,8 @@ Management API client sends requests, so it is never set. Every launch gets the `--version` and `init`, so an unexpected new API call reaches the server and is counted. Before each launch the harness clears the server's request log and scenario, then selects the workload's scenario, so scenarios that are used up as they are served start fresh every time. A failing control -call aborts the benchmark. +call aborts the benchmark. A launch that runs longer than 60 seconds is killed and counted as a +failed launch (exit code 124), so that workload is reported as not compared and the rest finish. API calls are counted from the traced launch's HTTP client spans: spans to the `--api-url` host and port are API calls, and spans to any other host are `externalCalls`. The count is cross-checked @@ -166,7 +174,7 @@ the timing ranges, API calls per command, and the slowest phases. It reports: - Phase deltas: the three largest span deltas for each workload with a slower or faster verdict. - Binary size of `dist/supabase`. -The baseline API calls per workload on both `develop` and `next` builds (a project-scoped command +The baseline API calls per workload on `develop` (a project-scoped command also refreshes the linked-project cache, so it makes two calls): | Workload | API calls | @@ -205,15 +213,10 @@ GROUP BY cmd ORDER BY users DESC LIMIT 40 Caveats: -- Flags differ between branches. The manifest uses the `develop` form (`--project-ref`, - `--project-id`). `next` accepts these as deprecated aliases until v4 and prints a warning in both - builds. After the v3 cut, switch the manifest to `--remote[=]`. -- `db query` is not benchmarked yet, although it is one of the most used commands. `next` requires - `db query --remote --read-only`, whose request the `develop` replay scenario does not match, so one - manifest cannot run it on both branches. Add it after the v3 cut with the `db-query-remote-*` - scenario. -- Telemetry records the command path but no flags or arguments, so `bench` and the - TypeScript target only approximate real use. +- Telemetry records the command path and the flags set on the command line, with values redacted + unless the flag is boolean, a choice, or listed in the command's `safeFlags` (see + [analytics.md](analytics.md)); positional arguments and free-form values are not recorded, so + `bench` and the TypeScript target only approximate real use. - `device_id` counts machines, not people, short-lived CI runners inflate it, and a large share of runs come from agents. Telemetry durations include network, Docker, and wait time, so they guided the choice but are not what the benchmark measures. diff --git a/apps/cli/scripts/bench-cli.integration.test.ts b/apps/cli/scripts/bench-cli.integration.test.ts index 244408070d..6caeed1fcb 100644 --- a/apps/cli/scripts/bench-cli.integration.test.ts +++ b/apps/cli/scripts/bench-cli.integration.test.ts @@ -1,9 +1,14 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, FileSystem, Layer, Ref } from "effect"; +import { Duration, Effect, FileSystem, Layer, Ref } from "effect"; import { FetchHttpClient } from "effect/http"; import { startReplayServer } from "../../cli-e2e/src/server/replay-server.ts"; -import { runWorkloadBench, type BenchOptions } from "./bench-cli.ts"; +import { + LAUNCH_TIMEOUT_EXIT_CODE, + runLaunch, + runWorkloadBench, + type BenchOptions, +} from "./bench-cli.ts"; const fixturesDir = new URL("../../cli-e2e/fixtures", import.meta.url).pathname; @@ -70,3 +75,35 @@ describe("runWorkloadBench", () => { ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), ); }); + +describe("runLaunch", () => { + it.live("kills a launch that outlives the timeout and records a failing exit code", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "bench-hung-cli-" }); + const pidFile = `${dir}/pid`; + const binary = `${dir}/supabase`; + yield* fs.writeFileString( + binary, + `#!/bin/sh\necho $$ > "${pidFile}"\nexec /bin/sleep 60\n`, + ); + yield* fs.chmod(binary, 0o755); + + const result = yield* runLaunch({ + binary, + args: [], + cwd: dir, + env: {}, + timeout: Duration.millis(500), + }); + + expect(result.exitCode).toBe(LAUNCH_TIMEOUT_EXIT_CODE); + expect(result.wallMs).toBeLessThan(30_000); + const pid = Number((yield* fs.readFileString(pidFile)).trim()); + expect(Number.isInteger(pid)).toBe(true); + expect(() => process.kill(pid, 0)).toThrow(); + }), + ).pipe(Effect.provide(BunServices.layer)), + ); +}); diff --git a/apps/cli/scripts/bench-cli.ts b/apps/cli/scripts/bench-cli.ts index 8ec2d862d1..5699c7aba8 100644 --- a/apps/cli/scripts/bench-cli.ts +++ b/apps/cli/scripts/bench-cli.ts @@ -13,6 +13,7 @@ import { BunPath, BunRuntime, BunServices } from "@effect/platform-bun"; import { Clock, Console, + Duration, Effect, FileSystem, Layer, @@ -230,16 +231,22 @@ interface LaunchResult { readonly exitCode: number; } -const runLaunch = (params: { +/** Longest a single launch may run before it is killed and counted as a failed launch. */ +export const LAUNCH_TIMEOUT = Duration.seconds(60); +/** Exit code recorded for a launch that hit {@link LAUNCH_TIMEOUT}. */ +export const LAUNCH_TIMEOUT_EXIT_CODE = 124; + +export const runLaunch = Effect.fn("BenchCli.runLaunch")(function* (params: { readonly binary: string; readonly args: ReadonlyArray; readonly cwd: string; readonly env: Record; -}) => - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const start = yield* Clock.currentTimeNanos; - const exitCode = yield* spawner.exitCode( + readonly timeout?: Duration.Duration; +}) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const start = yield* Clock.currentTimeNanos; + const exit = yield* spawner + .exitCode( ChildProcess.make(params.binary, params.args, { cwd: params.cwd, env: params.env, @@ -248,13 +255,19 @@ const runLaunch = (params: { stdout: "ignore", stderr: "ignore", }), + ) + .pipe(Effect.timeoutOption(params.timeout ?? LAUNCH_TIMEOUT)); + const end = yield* Clock.currentTimeNanos; + if (Option.isNone(exit)) { + yield* Console.error( + `warning: ${params.binary} ${params.args.join(" ")} did not exit within ${Duration.format(params.timeout ?? LAUNCH_TIMEOUT)}; it was killed`, ); - const end = yield* Clock.currentTimeNanos; - return { - wallMs: Number(end - start) / 1_000_000, - exitCode: Number(exitCode), - } satisfies LaunchResult; - }); + } + return { + wallMs: Number(end - start) / 1_000_000, + exitCode: Option.isSome(exit) ? Number(exit.value) : LAUNCH_TIMEOUT_EXIT_CODE, + } satisfies LaunchResult; +}); const PROFILE_FILE = "bench-profile.yaml";