diff --git a/.github/scripts/stage-vitest-results.sh b/.github/scripts/stage-vitest-results.sh new file mode 100644 index 0000000000..5a88d07aa0 --- /dev/null +++ b/.github/scripts/stage-vitest-results.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Stages each workspace's Vitest results cache under ///results.json +# before upload, so actions/upload-artifact's common-ancestor stripping can't drop the workspace +# from the artifact's paths. +# +# Usage: stage-vitest-results.sh ... +set -euo pipefail +shopt -s nullglob + +staging_dir="$1" +shift + +mkdir -p "$staging_dir" +for pattern in "$@"; do + for cache in $pattern; do + workspace="${cache%%/node_modules/*}" + hash_dir="$(basename "$(dirname "$cache")")" + dest="$staging_dir/$workspace/$hash_dir" + mkdir -p "$dest" + cp "$cache" "$dest/results.json" + done +done diff --git a/.github/scripts/test-health-report/logic.test.ts b/.github/scripts/test-health-report/logic.test.ts new file mode 100644 index 0000000000..c5c5b6ad91 --- /dev/null +++ b/.github/scripts/test-health-report/logic.test.ts @@ -0,0 +1,237 @@ +import { describe, expect, test } from "bun:test"; +import { + aggregateLocByWorkspaceTier, + classifyTestTier, + diffLocByWorkspaceTier, + fastestGrowingFiles, + flakyFiles, + renderMarkdownReport, + retriedJobFailures, + slowestFiles, + workspaceForPath, +} from "./logic.ts"; + +describe("classifyTestTier", () => { + test.each([ + ["src/foo.unit.test.ts", "unit"], + ["src/foo.integration.test.ts", "integration"], + ["src/foo.e2e.test.ts", "e2e"], + ["src/foo.live.test.ts", "live"], + [".github/scripts/contribution-gate.test.ts", "other"], + ] as const)("classifies %s as %s", (path, tier) => { + expect(classifyTestTier(path)).toBe(tier); + }); +}); + +describe("workspaceForPath", () => { + test.each([ + ["apps/cli/src/foo.unit.test.ts", "apps/cli"], + ["packages/api/src/foo.integration.test.ts", "packages/api"], + [".github/scripts/contribution-gate.test.ts", "root"], + ["tools/lib/test-health-report.unit.test.ts", "root"], + ])("maps %s to workspace %s", (path, workspace) => { + expect(workspaceForPath(path)).toBe(workspace); + }); +}); + +describe("aggregateLocByWorkspaceTier", () => { + test("sums LOC per workspace/tier and counts files", () => { + const totals = aggregateLocByWorkspaceTier([ + { path: "apps/cli/src/a.unit.test.ts", loc: 10 }, + { path: "apps/cli/src/b.unit.test.ts", loc: 20 }, + { path: "apps/cli/src/c.integration.test.ts", loc: 5 }, + { path: "packages/api/src/a.unit.test.ts", loc: 7 }, + ]); + + expect(totals).toEqual([ + { workspace: "apps/cli", tier: "integration", loc: 5, fileCount: 1 }, + { workspace: "apps/cli", tier: "unit", loc: 30, fileCount: 2 }, + { workspace: "packages/api", tier: "unit", loc: 7, fileCount: 1 }, + ]); + }); +}); + +describe("diffLocByWorkspaceTier", () => { + test("computes week-over-week change and keeps groups that emptied out", () => { + const current = [{ workspace: "apps/cli", tier: "unit" as const, loc: 30, fileCount: 2 }]; + const previous = [ + { workspace: "apps/cli", tier: "unit" as const, loc: 10, fileCount: 1 }, + { workspace: "apps/cli", tier: "integration" as const, loc: 40, fileCount: 3 }, + ]; + + expect(diffLocByWorkspaceTier(current, previous)).toEqual([ + { + workspace: "apps/cli", + tier: "integration", + loc: 0, + fileCount: 0, + previousLoc: 40, + change: -40, + }, + { workspace: "apps/cli", tier: "unit", loc: 30, fileCount: 2, previousLoc: 10, change: 20 }, + ]); + }); +}); + +describe("fastestGrowingFiles", () => { + test("ranks growth descending and treats a new file as growth from zero", () => { + const current = [ + { path: "src/a.unit.test.ts", loc: 100 }, + { path: "src/b.unit.test.ts", loc: 50 }, + { path: "src/new.unit.test.ts", loc: 30 }, + { path: "src/shrunk.unit.test.ts", loc: 5 }, + ]; + const previous = [ + { path: "src/a.unit.test.ts", loc: 40 }, + { path: "src/b.unit.test.ts", loc: 45 }, + { path: "src/shrunk.unit.test.ts", loc: 50 }, + ]; + + expect(fastestGrowingFiles(current, previous, 2)).toEqual([ + { path: "src/a.unit.test.ts", loc: 100, previousLoc: 40, growth: 60 }, + { path: "src/new.unit.test.ts", loc: 30, previousLoc: 0, growth: 30 }, + ]); + }); + + test("excludes files that shrank or stayed the same", () => { + const current = [{ path: "src/a.unit.test.ts", loc: 10 }]; + const previous = [{ path: "src/a.unit.test.ts", loc: 10 }]; + + expect(fastestGrowingFiles(current, previous)).toEqual([]); + }); +}); + +describe("slowestFiles", () => { + test("keeps the max duration per file across runs and ranks descending", () => { + const entries = [ + { workspace: "apps/cli", path: "src/a.e2e.test.ts", duration: 1200.4, failed: false }, + { workspace: "apps/cli", path: "src/a.e2e.test.ts", duration: 900, failed: false }, + { workspace: "apps/cli", path: "src/b.e2e.test.ts", duration: 5000, failed: false }, + ]; + + expect(slowestFiles(entries, 10)).toEqual([ + { workspace: "apps/cli", path: "src/b.e2e.test.ts", durationMs: 5000 }, + { workspace: "apps/cli", path: "src/a.e2e.test.ts", durationMs: 1200 }, + ]); + }); + + test("limits to the requested count", () => { + const entries = Array.from({ length: 15 }, (_, index) => ({ + workspace: "apps/cli", + path: `src/f${index}.e2e.test.ts`, + duration: index, + failed: false, + })); + + expect(slowestFiles(entries, 10)).toHaveLength(10); + }); +}); + +describe("flakyFiles", () => { + test("counts failures per file and ignores passing entries", () => { + const entries = [ + { workspace: "apps/cli", path: "src/a.e2e.test.ts", duration: 100, failed: true }, + { workspace: "apps/cli", path: "src/a.e2e.test.ts", duration: 100, failed: true }, + { workspace: "apps/cli", path: "src/b.e2e.test.ts", duration: 100, failed: false }, + ]; + + expect(flakyFiles(entries)).toEqual([ + { workspace: "apps/cli", path: "src/a.e2e.test.ts", failureCount: 2 }, + ]); + }); +}); + +describe("retriedJobFailures", () => { + test("reports attempt-1 failures only for runs that ultimately succeeded after a retry", () => { + const runs = [ + { + runId: 1, + runUrl: "https://example.com/1", + finalAttempt: 2, + finalConclusion: "success", + attempt1Jobs: [ + { name: "Run unit tests", conclusion: "failure" }, + { name: "Run integration tests", conclusion: "success" }, + ], + }, + { + // Never retried — attempt 1 is the only (and final) attempt. + runId: 2, + runUrl: "https://example.com/2", + finalAttempt: 1, + finalConclusion: "success", + attempt1Jobs: [{ name: "Run unit tests", conclusion: "failure" }], + }, + { + // Retried but still failing — not a flaky "later succeeded" case. + runId: 3, + runUrl: "https://example.com/3", + finalAttempt: 2, + finalConclusion: "failure", + attempt1Jobs: [{ name: "Run unit tests", conclusion: "failure" }], + }, + ]; + + expect(retriedJobFailures(runs)).toEqual([ + { runId: 1, runUrl: "https://example.com/1", finalAttempt: 2, jobName: "Run unit tests" }, + ]); + }); +}); + +describe("renderMarkdownReport", () => { + test("renders every section with an empty-state placeholder when there is no data", () => { + const markdown = renderMarkdownReport({ + metadata: { + headSha: "abcdef123456789", + weekAgoSha: "111111222222333", + weekAgoDate: "2026-09-22", + runsAnalyzed: 3, + notes: ["sampled the last 3 runs for this demo"], + }, + locChanges: [], + fastestGrowing: [], + slowest: [], + flakyFiles: [], + retriedJobFailures: [], + }); + + expect(markdown).toContain("# Weekly test-suite health report"); + expect(markdown).toContain("abcdef123456"); + expect(markdown).toContain("> sampled the last 3 runs for this demo"); + expect(markdown).toContain("## Test LOC by workspace and tier"); + expect(markdown).toContain("_none_"); + }); + + test("renders populated tables with formatted durations and changes", () => { + const markdown = renderMarkdownReport({ + metadata: { + headSha: "abcdef123456789", + weekAgoSha: "111111222222333", + weekAgoDate: "2026-09-22", + runsAnalyzed: 12, + notes: [], + }, + locChanges: [ + { + workspace: "apps/cli", + tier: "unit", + loc: 120, + fileCount: 4, + previousLoc: 100, + change: 20, + }, + ], + fastestGrowing: [{ path: "src/a.unit.test.ts", loc: 120, previousLoc: 100, growth: 20 }], + slowest: [{ workspace: "apps/cli", path: "src/a.e2e.test.ts", durationMs: 12345 }], + flakyFiles: [{ workspace: "apps/cli", path: "src/a.e2e.test.ts", failureCount: 3 }], + retriedJobFailures: [ + { runId: 42, runUrl: "https://example.com/42", finalAttempt: 2, jobName: "Run unit tests" }, + ], + }); + + expect(markdown).toContain("| apps/cli | unit | 120 | 4 | +20 |"); + expect(markdown).toContain("12.3s"); + expect(markdown).toContain("| apps/cli | src/a.e2e.test.ts | 3 |"); + expect(markdown).toContain("[42](https://example.com/42)"); + }); +}); diff --git a/.github/scripts/test-health-report/logic.ts b/.github/scripts/test-health-report/logic.ts new file mode 100644 index 0000000000..062cb7c47a --- /dev/null +++ b/.github/scripts/test-health-report/logic.ts @@ -0,0 +1,336 @@ +/** + * Pure pieces of the weekly test-suite health report: tier classification, LOC diffing, ranking, + * and flaky-test detection. `report.ts` holds the git/API/artifact-fetching half. + */ + +export type TestTier = "unit" | "integration" | "e2e" | "live" | "other"; + +const TIER_SUFFIXES: ReadonlyArray = [ + [".unit.test.ts", "unit"], + [".integration.test.ts", "integration"], + [".e2e.test.ts", "e2e"], + [".live.test.ts", "live"], +]; + +/** Tiers follow the `*.unit.test.ts` / `*.integration.test.ts` / `*.e2e.test.ts` / `*.live.test.ts` + * naming convention; a bare `*.test.ts` file (e.g. under `.github/scripts`) is `"other"`. */ +export function classifyTestTier(path: string): TestTier { + for (const [suffix, tier] of TIER_SUFFIXES) { + if (path.endsWith(suffix)) { + return tier; + } + } + return "other"; +} + +/** The owning workspace for a repo-relative path, e.g. `apps/cli` or `packages/api`; anything + * outside `apps/*` or `packages/*` (root scripts, `.github/scripts`, `tools/`) is `"root"`. */ +export function workspaceForPath(path: string): string { + const match = /^(apps|packages)\/([^/]+)\//.exec(path); + return match ? `${match[1]}/${match[2]}` : "root"; +} + +export interface TestFileLoc { + readonly path: string; + readonly loc: number; +} + +export interface WorkspaceTierLoc { + readonly workspace: string; + readonly tier: TestTier; + readonly loc: number; + readonly fileCount: number; +} + +function groupKey(workspace: string, tier: string): string { + return `${workspace}\u0000${tier}`; +} + +/** Sums LOC per workspace/tier pair, sorted by workspace then tier for stable rendering. */ +export function aggregateLocByWorkspaceTier(files: readonly TestFileLoc[]): WorkspaceTierLoc[] { + const totals = new Map(); + for (const file of files) { + const key = groupKey(workspaceForPath(file.path), classifyTestTier(file.path)); + const existing = totals.get(key) ?? { loc: 0, fileCount: 0 }; + totals.set(key, { loc: existing.loc + file.loc, fileCount: existing.fileCount + 1 }); + } + return [...totals.entries()] + .map(([key, value]) => { + const [workspace = "root", tier = "other"] = key.split("\u0000"); + return { workspace, tier: tier as TestTier, ...value }; + }) + .sort((a, b) => a.workspace.localeCompare(b.workspace) || a.tier.localeCompare(b.tier)); +} + +export interface WorkspaceTierChange extends WorkspaceTierLoc { + readonly previousLoc: number; + readonly change: number; +} + +/** Joins this week's per-workspace/tier LOC against last week's, keeping every group that had + * LOC in either snapshot (a tier that emptied out this week still shows its drop to zero). */ +export function diffLocByWorkspaceTier( + current: readonly WorkspaceTierLoc[], + previous: readonly WorkspaceTierLoc[], +): WorkspaceTierChange[] { + const previousByKey = new Map( + previous.map((entry) => [groupKey(entry.workspace, entry.tier), entry.loc]), + ); + const currentByKey = new Map( + current.map((entry) => [groupKey(entry.workspace, entry.tier), entry]), + ); + const keys = new Set([...currentByKey.keys(), ...previousByKey.keys()]); + + const changes: WorkspaceTierChange[] = []; + for (const key of keys) { + const [workspace = "root", tier = "other"] = key.split("\u0000"); + const entry = currentByKey.get(key); + const previousLoc = previousByKey.get(key) ?? 0; + const loc = entry?.loc ?? 0; + changes.push({ + workspace, + tier: tier as TestTier, + loc, + fileCount: entry?.fileCount ?? 0, + previousLoc, + change: loc - previousLoc, + }); + } + return changes.sort( + (a, b) => a.workspace.localeCompare(b.workspace) || a.tier.localeCompare(b.tier), + ); +} + +export interface FileGrowth { + readonly path: string; + readonly loc: number; + readonly previousLoc: number; + readonly growth: number; +} + +/** Files ranked by LOC growth over the week; a file absent from `previous` counts as growth from + * zero, and files that shrank or disappeared are excluded. */ +export function fastestGrowingFiles( + current: readonly TestFileLoc[], + previous: readonly TestFileLoc[], + limit = 10, +): FileGrowth[] { + const previousByPath = new Map(previous.map((entry) => [entry.path, entry.loc])); + return current + .map((file) => { + const previousLoc = previousByPath.get(file.path) ?? 0; + return { path: file.path, loc: file.loc, previousLoc, growth: file.loc - previousLoc }; + }) + .filter((entry) => entry.growth > 0) + .sort((a, b) => b.growth - a.growth || a.path.localeCompare(b.path)) + .slice(0, limit); +} + +export interface TimingEntry { + readonly workspace: string; + readonly path: string; + readonly duration: number; + readonly failed: boolean; +} + +export interface FileDuration { + readonly workspace: string; + readonly path: string; + readonly durationMs: number; +} + +/** The slowest test files over the week, by the longest duration recorded for each across runs. */ +export function slowestFiles(entries: readonly TimingEntry[], limit = 10): FileDuration[] { + const slowestByKey = new Map(); + for (const entry of entries) { + const key = groupKey(entry.workspace, entry.path); + const existing = slowestByKey.get(key); + const durationMs = Math.round(entry.duration); + if (!existing || durationMs > existing.durationMs) { + slowestByKey.set(key, { workspace: entry.workspace, path: entry.path, durationMs }); + } + } + return [...slowestByKey.values()] + .sort((a, b) => b.durationMs - a.durationMs || a.path.localeCompare(b.path)) + .slice(0, limit); +} + +export interface FlakyFile { + readonly workspace: string; + readonly path: string; + readonly failureCount: number; +} + +/** Files that failed in at least one run this week; can't distinguish flaky from broken-then-fixed. */ +export function flakyFiles(entries: readonly TimingEntry[]): FlakyFile[] { + const failuresByKey = new Map(); + for (const entry of entries) { + if (!entry.failed) { + continue; + } + const key = groupKey(entry.workspace, entry.path); + const existing = failuresByKey.get(key); + failuresByKey.set(key, { + workspace: entry.workspace, + path: entry.path, + failureCount: (existing?.failureCount ?? 0) + 1, + }); + } + return [...failuresByKey.values()].sort( + (a, b) => b.failureCount - a.failureCount || a.path.localeCompare(b.path), + ); +} + +export interface RunAttemptJobs { + readonly runId: number; + readonly runUrl: string; + /** The attempt number of the run's current (latest) state. */ + readonly finalAttempt: number; + readonly finalConclusion: string | null; + /** Job name/conclusion pairs from the run's first attempt. */ + readonly attempt1Jobs: ReadonlyArray<{ + readonly name: string; + readonly conclusion: string | null; + }>; +} + +export interface RetriedJobFailure { + readonly runId: number; + readonly runUrl: string; + readonly finalAttempt: number; + readonly jobName: string; +} + +/** Jobs that failed on attempt 1 of a run that later succeeded; job granularity only, since the + * Actions API has no per-test-file outcome for a past attempt. */ +export function retriedJobFailures(runs: readonly RunAttemptJobs[]): RetriedJobFailure[] { + const failures: RetriedJobFailure[] = []; + for (const run of runs) { + if (run.finalConclusion !== "success" || run.finalAttempt <= 1) { + continue; + } + for (const job of run.attempt1Jobs) { + if (job.conclusion === "failure") { + failures.push({ + runId: run.runId, + runUrl: run.runUrl, + finalAttempt: run.finalAttempt, + jobName: job.name, + }); + } + } + } + return failures; +} + +function formatDuration(ms: number): string { + return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${ms}ms`; +} + +function formatChange(change: number): string { + if (change === 0) { + return "0"; + } + return change > 0 ? `+${change}` : `${change}`; +} + +function markdownTable(headers: readonly string[], rows: ReadonlyArray): string { + if (rows.length === 0) { + return "_none_"; + } + const headerRow = `| ${headers.join(" | ")} |`; + const separatorRow = `| ${headers.map(() => "---").join(" | ")} |`; + const bodyRows = rows.map((row) => `| ${row.join(" | ")} |`); + return [headerRow, separatorRow, ...bodyRows].join("\n"); +} + +interface ReportMetadata { + readonly headSha: string; + readonly weekAgoSha: string; + readonly weekAgoDate: string; + readonly runsAnalyzed: number; + /** Caveats surfaced to the reader, e.g. sampling limits or missing artifact kinds. */ + readonly notes: readonly string[]; +} + +export interface ReportInput { + readonly metadata: ReportMetadata; + readonly locChanges: readonly WorkspaceTierChange[]; + readonly fastestGrowing: readonly FileGrowth[]; + readonly slowest: readonly FileDuration[]; + readonly flakyFiles: readonly FlakyFile[]; + readonly retriedJobFailures: readonly RetriedJobFailure[]; +} + +/** Renders the full report as GitHub-flavored markdown, suitable for `$GITHUB_STEP_SUMMARY`. */ +export function renderMarkdownReport(input: ReportInput): string { + const { metadata } = input; + const lines: string[] = [ + "# Weekly test-suite health report", + "", + `Comparing \`${metadata.headSha.slice(0, 12)}\` against \`${metadata.weekAgoSha.slice(0, 12)}\` (${metadata.weekAgoDate}), across ${metadata.runsAnalyzed} develop/merge-queue Test run(s) from the past week.`, + "", + ]; + + if (metadata.notes.length > 0) { + lines.push(...metadata.notes.map((note) => `> ${note}`), ""); + } + + lines.push( + "## Test LOC by workspace and tier", + "", + markdownTable( + ["Workspace", "Tier", "LOC", "Files", "Δ vs last week"], + input.locChanges + .filter((entry) => entry.loc > 0 || entry.previousLoc > 0) + .map((entry) => [ + entry.workspace, + entry.tier, + String(entry.loc), + String(entry.fileCount), + formatChange(entry.change), + ]), + ), + "", + "## Fastest-growing test files (by LOC)", + "", + markdownTable( + ["Path", "LOC", "Δ vs last week"], + input.fastestGrowing.map((entry) => [ + entry.path, + String(entry.loc), + formatChange(entry.growth), + ]), + ), + "", + "## Slowest test files", + "", + markdownTable( + ["Workspace", "Path", "Duration"], + input.slowest.map((entry) => [entry.workspace, entry.path, formatDuration(entry.durationMs)]), + ), + "", + "## Flaky tests", + "", + "### Files that failed in a develop/merge-queue run", + "", + markdownTable( + ["Workspace", "Path", "Failures"], + input.flakyFiles.map((entry) => [entry.workspace, entry.path, String(entry.failureCount)]), + ), + "", + "### Runs where attempt 1 failed and a later attempt succeeded", + "", + markdownTable( + ["Run", "Job", "Succeeded on attempt"], + input.retriedJobFailures.map((entry) => [ + `[${entry.runId}](${entry.runUrl})`, + entry.jobName, + String(entry.finalAttempt), + ]), + ), + "", + ); + + return lines.join("\n"); +} diff --git a/.github/scripts/test-health-report/report.ts b/.github/scripts/test-health-report/report.ts new file mode 100644 index 0000000000..fe8c652127 --- /dev/null +++ b/.github/scripts/test-health-report/report.ts @@ -0,0 +1,300 @@ +/** + * Builds the weekly test-suite health report from git history plus the `*-timings-*` artifacts + * uploaded by `.github/workflows/test.yml`. Advisory only: never exits non-zero for a data + * finding, only for a tool failure. + * + * Usage: bun .github/scripts/test-health-report/report.ts [--repo owner/name] [--days 7] [--max-runs N] + */ + +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path, { join } from "node:path"; +import process from "node:process"; +import { parseArgs } from "node:util"; +import { + aggregateLocByWorkspaceTier, + diffLocByWorkspaceTier, + fastestGrowingFiles, + flakyFiles, + renderMarkdownReport, + retriedJobFailures, + slowestFiles, + type RunAttemptJobs, + type TestFileLoc, + type TimingEntry, +} from "./logic.ts"; +import { readResultEntries } from "../../../apps/cli/scripts/vitest-results-cache.ts"; + +const repoRoot = path.resolve(import.meta.dir, "../../.."); + +interface ProcessResult { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; +} + +async function run(command: readonly string[]): Promise { + const proc = Bun.spawn([...command], { cwd: repoRoot, stdout: "pipe", stderr: "pipe" }); + const [exitCode, stdout, stderr] = await Promise.all([ + proc.exited, + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + return { exitCode, stdout, stderr }; +} + +async function runGit(args: readonly string[]): Promise { + return run(["git", ...args]); +} + +async function ghApiJson(apiPath: string): Promise { + const result = await run(["gh", "api", apiPath]); + if (result.exitCode !== 0) { + throw new Error(`gh api ${apiPath} failed: ${result.stderr.trim()}`); + } + return JSON.parse(result.stdout) as T; +} + +/** Every tracked `*.test.ts` file and its line count at `ref`, via one `git grep -c` call — + * matches a manual `git ls-files '*.test.ts' | xargs wc -l` count at that ref. */ +async function listTestFilesWithLoc(ref: string): Promise { + const result = await runGit(["grep", "--no-color", "-I", "-c", "", ref, "--", "*.test.ts"]); + // git grep exits 1 (with empty stdout) when nothing matches; that's not a tool failure. + if (result.exitCode !== 0 && result.stdout.trim() === "") { + if (result.exitCode === 1) { + return []; + } + throw new Error(`git grep -c '' ${ref} -- '*.test.ts' failed: ${result.stderr.trim()}`); + } + const files: TestFileLoc[] = []; + for (const line of result.stdout.split("\n")) { + if (line === "") { + continue; + } + const match = /^[0-9a-f]+:(.+):(\d+)$/.exec(line); + if (!match) { + continue; + } + const [, filePath, loc] = match; + files.push({ path: filePath ?? "", loc: Number(loc) }); + } + return files; +} + +async function resolveWeekAgoSha(headSha: string, days: number): Promise { + const result = await runGit(["log", "-1", `--before=${days} days ago`, "--format=%H", headSha]); + const sha = result.stdout.trim(); + return sha === "" ? headSha : sha; +} + +async function commitDate(sha: string): Promise { + const result = await runGit(["log", "-1", "--format=%cs", sha]); + return result.stdout.trim(); +} + +interface WorkflowRunSummary { + readonly id: number; + readonly html_url: string; + readonly created_at: string; + readonly conclusion: string | null; + readonly run_attempt: number; +} + +async function listWorkflowRuns( + repo: string, + event: string, + sinceIso: string, + branch?: string, +): Promise { + const runs: WorkflowRunSummary[] = []; + for (let page = 1; page <= 50; page++) { + const params = new URLSearchParams({ + event, + per_page: "100", + page: String(page), + created: `>=${sinceIso}`, + }); + if (branch) { + params.set("branch", branch); + } + const data = await ghApiJson<{ workflow_runs: WorkflowRunSummary[] }>( + `repos/${repo}/actions/workflows/test.yml/runs?${params.toString()}`, + ); + runs.push(...data.workflow_runs); + if (data.workflow_runs.length < 100) { + break; + } + } + return runs; +} + +/** The week's develop pushes and merge-queue runs of the Test workflow, deduped and time-ordered. */ +async function listRunsForWindow(repo: string, sinceIso: string): Promise { + const [mergeQueueRuns, pushRuns] = await Promise.all([ + listWorkflowRuns(repo, "merge_group", sinceIso), + listWorkflowRuns(repo, "push", sinceIso, "develop"), + ]); + const byId = new Map(); + for (const run of [...mergeQueueRuns, ...pushRuns]) { + byId.set(run.id, run); + } + return [...byId.values()].sort((a, b) => a.created_at.localeCompare(b.created_at)); +} + +/** Downloads a run's `*-timings-*` artifacts into `destDir`; returns false when the run has none + * (e.g. it was cancelled before the upload step), which is not a failure. */ +async function downloadTimingArtifacts( + repo: string, + runId: number, + destDir: string, +): Promise { + const result = await run([ + "gh", + "run", + "download", + String(runId), + "--repo", + repo, + "--pattern", + "*-timings-*", + "--dir", + destDir, + ]); + if (result.exitCode !== 0) { + if (!/no artifacts? found|no valid artifacts/i.test(result.stderr)) { + console.warn(`[test-health-report] gh run download ${runId} failed: ${result.stderr.trim()}`); + } + return false; + } + return true; +} + +async function attempt1Jobs( + repo: string, + runId: number, +): Promise> { + const data = await ghApiJson<{ jobs: Array<{ name: string; conclusion: string | null }> }>( + `repos/${repo}/actions/runs/${runId}/attempts/1/jobs`, + ); + return data.jobs.map((job) => ({ name: job.name, conclusion: job.conclusion })); +} + +async function writeStepSummary(markdown: string): Promise { + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (!summaryPath) { + return; + } + await Bun.write(summaryPath, `${markdown}\n`, { createPath: true }); +} + +const usage = `Usage: bun .github/scripts/test-health-report/report.ts [--repo owner/name] [--days 7] [--max-runs N] + + --max-runs caps how many matching runs are downloaded, for local dry runs or rate-limit safety.`; + +if (import.meta.main) { + const { values } = parseArgs({ + args: process.argv.slice(2), + options: { + repo: { type: "string", default: "supabase/cli" }, + days: { type: "string", default: "7" }, + "max-runs": { type: "string" }, + help: { type: "boolean", default: false }, + }, + }); + + if (values.help) { + console.log(usage); + process.exit(0); + } + + const repo = values.repo ?? "supabase/cli"; + const days = Number(values.days ?? "7"); + const maxRuns = values["max-runs"] !== undefined ? Number(values["max-runs"]) : undefined; + const notes: string[] = []; + + try { + const headSha = (await runGit(["rev-parse", "HEAD"])).stdout.trim(); + const weekAgoSha = await resolveWeekAgoSha(headSha, days); + const weekAgoDate = await commitDate(weekAgoSha); + + const [currentFiles, previousFiles] = await Promise.all([ + listTestFilesWithLoc(headSha), + listTestFilesWithLoc(weekAgoSha), + ]); + const locChanges = diffLocByWorkspaceTier( + aggregateLocByWorkspaceTier(currentFiles), + aggregateLocByWorkspaceTier(previousFiles), + ); + const fastestGrowing = fastestGrowingFiles(currentFiles, previousFiles); + + const sinceIso = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString(); + let runs = await listRunsForWindow(repo, sinceIso); + if (maxRuns !== undefined && runs.length > maxRuns) { + notes.push( + `sampled the ${maxRuns} most recent of ${runs.length} matching runs from the past ` + + `${days} day(s) (--max-runs).`, + ); + runs = runs.slice(-maxRuns); + } + + const timingEntries: TimingEntry[] = []; + const runAttemptInfos: RunAttemptJobs[] = []; + const downloadRoot = mkdtempSync(join(tmpdir(), "test-health-report-")); + try { + for (const workflowRun of runs) { + const runDir = join(downloadRoot, String(workflowRun.id)); + if (await downloadTimingArtifacts(repo, workflowRun.id, runDir)) { + try { + timingEntries.push(...readResultEntries(runDir)); + } catch (cause) { + console.warn( + `[test-health-report] could not parse timing artifacts for run ${workflowRun.id}: ` + + `${cause instanceof Error ? cause.message : String(cause)}`, + ); + } + } + + if (workflowRun.run_attempt > 1) { + runAttemptInfos.push({ + runId: workflowRun.id, + runUrl: workflowRun.html_url, + finalAttempt: workflowRun.run_attempt, + finalConclusion: workflowRun.conclusion, + attempt1Jobs: await attempt1Jobs(repo, workflowRun.id), + }); + } + } + } finally { + rmSync(downloadRoot, { recursive: true, force: true }); + } + + if (timingEntries.length === 0 && runs.length > 0) { + notes.push( + "no timing artifacts were found for this week's runs — unit/integration timing uploads " + + "only exist once this report's own workflow changes have merged; only e2e timings are " + + "available until then.", + ); + } else if (timingEntries.some((entry) => entry.workspace === ".")) { + notes.push( + 'some timing entries show workspace "." — they came from an e2e-timings artifact ' + + "uploaded before the per-workspace staging step landed, so their workspace could not " + + "be recovered.", + ); + } + + const markdown = renderMarkdownReport({ + metadata: { headSha, weekAgoSha, weekAgoDate, runsAnalyzed: runs.length, notes }, + locChanges, + fastestGrowing, + slowest: slowestFiles(timingEntries), + flakyFiles: flakyFiles(timingEntries), + retriedJobFailures: retriedJobFailures(runAttemptInfos), + }); + + console.log(markdown); + await writeStepSummary(markdown); + } catch (cause) { + console.error(`error: ${cause instanceof Error ? cause.message : String(cause)}`); + process.exitCode = 1; + } +} diff --git a/.github/workflows/github-scripts-ci.yml b/.github/workflows/github-scripts-ci.yml index c2f12fa74b..01cb497ace 100644 --- a/.github/workflows/github-scripts-ci.yml +++ b/.github/workflows/github-scripts-ci.yml @@ -12,6 +12,7 @@ on: - "apps/cli/src/shared/services/Dockerfile" - "apps/cli/src/shared/services/parse-dockerfile-service-images.ts" - "apps/cli/src/shared/services/slim-images.ts" + - "apps/cli/scripts/vitest-results-cache.ts" - "packages/stack/src/Artifacts.ts" - ".github/workflows/ai-review.yml" - ".github/workflows/github-scripts-ci.yml" diff --git a/.github/workflows/test-health-report.yml b/.github/workflows/test-health-report.yml new file mode 100644 index 0000000000..391df8bd6d --- /dev/null +++ b/.github/workflows/test-health-report.yml @@ -0,0 +1,43 @@ +name: Weekly test-suite health report + +# Advisory only — the report script never fails the job for what it finds, only for a tool +# failure, and this workflow gates nothing. +on: + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: test-health-report + cancel-in-progress: false + +jobs: + report: + name: Build report + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + actions: read + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The report diffs test LOC between HEAD and the last commit at least a week old. + fetch-depth: 0 + + # Bun and git/gh (preinstalled on the runner) only — no workspace install needed. + - name: Install toolchains + uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 + with: + version: 2026.9.0 + + - name: Build report + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bun .github/scripts/test-health-report/report.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 391542b18e..ee083bc58a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -100,6 +100,23 @@ jobs: - name: Run unit tests run: pnpm run test:unit + # Stage per workspace so the artifact keeps the workspace in each path. + - name: Stage unit test timings + if: always() + run: >- + bash .github/scripts/stage-vitest-results.sh .timings-staging + 'apps/*/node_modules/.vite/vitest/*/results.json' + 'packages/*/node_modules/.vite/vitest/*/results.json' + + - name: Upload unit test timings + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: unit-timings + path: .timings-staging + retention-days: 14 + if-no-files-found: warn + test-integration: if: | !startsWith(github.head_ref, 'release-notes/') && @@ -123,6 +140,23 @@ jobs: - name: Run integration tests run: pnpm run test:integration + # Stage per workspace so the artifact keeps the workspace in each path. + - name: Stage integration test timings + if: always() + run: >- + bash .github/scripts/stage-vitest-results.sh .timings-staging + 'apps/*/node_modules/.vite/vitest/*/results.json' + 'packages/*/node_modules/.vite/vitest/*/results.json' + + - name: Upload integration test timings + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: integration-timings + path: .timings-staging + retention-days: 14 + if-no-files-found: warn + test-stack-ports: if: | !startsWith(github.head_ref, 'release-notes/') && @@ -248,15 +282,29 @@ jobs: env: SUPABASE_GO_BINARY: ${{ github.workspace }}/apps/cli-go/supabase-go - # Vitest writes its results cache under a sha1-named directory. The refresh script - # (apps/cli/scripts/refresh-e2e-timings.ts) merges these into tests/e2e-timings.json. + # Stage per workspace so the artifact keeps the workspace in each path. + - name: Stage e2e test timings + if: always() + run: >- + bash .github/scripts/stage-vitest-results.sh .timings-staging + 'apps/*/node_modules/.vite/vitest/*/results.json' + - name: Upload e2e timings if: always() && matrix.suite == 'cli' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-timings-${{ matrix.shard }} - path: apps/cli/node_modules/.vite/vitest/**/results.json - retention-days: 7 + path: .timings-staging + retention-days: 14 + if-no-files-found: warn + + - name: Upload cli-e2e timings + if: always() && matrix.suite == 'cli-e2e' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-e2e-timings + path: .timings-staging + retention-days: 14 if-no-files-found: warn test-stack-e2e: diff --git a/apps/cli/scripts/refresh-e2e-timings.ts b/apps/cli/scripts/refresh-e2e-timings.ts index dd5396410b..d9196edc0d 100644 --- a/apps/cli/scripts/refresh-e2e-timings.ts +++ b/apps/cli/scripts/refresh-e2e-timings.ts @@ -1,24 +1,16 @@ // Rewrites apps/cli/tests/e2e-timings.json from the Vitest results caches that the test-e2e // workflow job uploads as `e2e-timings-` artifacts. import { spawnSync } from "node:child_process"; -import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import process from "node:process"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; import { version as installedVitestVersion } from "vitest/node"; +import { readResultsCaches, type VitestResultsCache } from "./vitest-results-cache.ts"; -export interface VitestFileResult { - readonly duration: number; - readonly failed: boolean; -} - -/** Shape of Vitest's `results.json` cache: `[:, result]` entries. */ -export interface VitestResultsCache { - readonly version: string; - readonly results: ReadonlyArray; -} +export type { VitestResultsCache } from "./vitest-results-cache.ts"; export interface MergeOptions { /** Version of the Vitest that will consume the timings; caches from another major are rejected. */ @@ -63,51 +55,6 @@ export function mergeTimings( return Object.fromEntries([...merged].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); } -function isFileResult(value: unknown): value is VitestFileResult { - return ( - typeof value === "object" && - value !== null && - "duration" in value && - typeof value.duration === "number" && - "failed" in value && - typeof value.failed === "boolean" - ); -} - -function isResultsEntry(value: unknown): value is readonly [string, VitestFileResult] { - return ( - Array.isArray(value) && - value.length === 2 && - typeof value[0] === "string" && - isFileResult(value[1]) - ); -} - -function isResultsCache(value: unknown): value is VitestResultsCache { - return ( - typeof value === "object" && - value !== null && - "version" in value && - typeof value.version === "string" && - "results" in value && - Array.isArray(value.results) && - value.results.every(isResultsEntry) - ); -} - -function readResultsCaches(dir: string): VitestResultsCache[] { - return readdirSync(dir, { recursive: true, withFileTypes: true }) - .filter((entry) => entry.isFile() && entry.name === "results.json") - .map((entry) => { - const file = join(entry.parentPath, entry.name); - const parsed: unknown = JSON.parse(readFileSync(file, "utf8")); - if (!isResultsCache(parsed)) { - throw new Error(`${file} is not a Vitest results cache`); - } - return parsed; - }); -} - const usage = `Usage: pnpm exec bun apps/cli/scripts/refresh-e2e-timings.ts --run [--repo supabase/cli] Downloads the e2e-timings-* artifacts of a test workflow run and rewrites diff --git a/apps/cli/scripts/vitest-results-cache.ts b/apps/cli/scripts/vitest-results-cache.ts new file mode 100644 index 0000000000..df42ca7c04 --- /dev/null +++ b/apps/cli/scripts/vitest-results-cache.ts @@ -0,0 +1,107 @@ +// Shared parsing for Vitest's results cache, staged before upload as +// `//results.json` so the workspace survives artifact upload. +import { readdirSync, readFileSync } from "node:fs"; +import { join, relative } from "node:path"; + +interface VitestFileResult { + readonly duration: number; + readonly failed: boolean; +} + +/** Shape of Vitest's `results.json` cache: `[:, result]` entries. */ +export interface VitestResultsCache { + readonly version: string; + readonly results: ReadonlyArray; +} + +function isFileResult(value: unknown): value is VitestFileResult { + return ( + typeof value === "object" && + value !== null && + "duration" in value && + typeof value.duration === "number" && + "failed" in value && + typeof value.failed === "boolean" + ); +} + +function isResultsEntry(value: unknown): value is readonly [string, VitestFileResult] { + return ( + Array.isArray(value) && + value.length === 2 && + typeof value[0] === "string" && + isFileResult(value[1]) + ); +} + +export function isResultsCache(value: unknown): value is VitestResultsCache { + return ( + typeof value === "object" && + value !== null && + "version" in value && + typeof value.version === "string" && + "results" in value && + Array.isArray(value.results) && + value.results.every(isResultsEntry) + ); +} + +/** Reads every `results.json` found recursively under `dir`. */ +export function readResultsCaches(dir: string): VitestResultsCache[] { + return readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name === "results.json") + .map((entry) => { + const file = join(entry.parentPath, entry.name); + const parsed: unknown = JSON.parse(readFileSync(file, "utf8")); + if (!isResultsCache(parsed)) { + throw new Error(`${file} is not a Vitest results cache`); + } + return parsed; + }); +} + +/** A single project entry from a results cache, with the `:` prefix split out and the + * owning workspace (e.g. `apps/cli`) derived from where the cache file was found under `dir`. */ +export interface VitestResultEntry { + readonly workspace: string; + readonly project: string; + readonly path: string; + readonly duration: number; + readonly failed: boolean; +} + +function splitKey(key: string): { readonly project: string; readonly path: string } { + const separator = key.indexOf(":"); + return separator === -1 + ? { project: "", path: key } + : { project: key.slice(0, separator), path: key.slice(separator + 1) }; +} + +/** The workspace is the `apps/` or `packages/` segment pair in the cache's directory + * path; a path with neither (e.g. an artifact predating staging) is `"."`. */ +function workspaceFromCachePath(relativeDir: string): string { + const segments = relativeDir.split(/[/\\]+/); + const index = segments.findIndex((segment) => segment === "apps" || segment === "packages"); + return index === -1 || segments[index + 1] === undefined + ? "." + : `${segments[index]}/${segments[index + 1]}`; +} + +/** Flattens every results cache found (recursively) under `dir` into entries carrying the + * workspace, project, path, duration, and failed flag. */ +export function readResultEntries(dir: string): VitestResultEntry[] { + return readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name === "results.json") + .flatMap((entry) => { + const file = join(entry.parentPath, entry.name); + const parsed: unknown = JSON.parse(readFileSync(file, "utf8")); + if (!isResultsCache(parsed)) { + throw new Error(`${file} is not a Vitest results cache`); + } + const workspace = workspaceFromCachePath(relative(dir, entry.parentPath)); + return parsed.results.map(([key, result]) => { + const { project, path } = splitKey(key); + return { workspace, project, path, duration: result.duration, failed: result.failed }; + }); + }); +} diff --git a/apps/cli/scripts/vitest-results-cache.unit.test.ts b/apps/cli/scripts/vitest-results-cache.unit.test.ts new file mode 100644 index 0000000000..d6ef90469e --- /dev/null +++ b/apps/cli/scripts/vitest-results-cache.unit.test.ts @@ -0,0 +1,122 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; +import { isResultsCache, readResultEntries, readResultsCaches } from "./vitest-results-cache.ts"; + +let dir: string | undefined; + +afterEach(() => { + if (dir) { + rmSync(dir, { recursive: true, force: true }); + dir = undefined; + } +}); + +function writeCache(relativeDir: string, cache: unknown): void { + const cacheDir = join( + dir ?? (dir = mkdtempSync(join(tmpdir(), "vitest-results-cache-"))), + relativeDir, + ); + mkdirSync(cacheDir, { recursive: true }); + writeFileSync(join(cacheDir, "results.json"), JSON.stringify(cache)); +} + +describe("isResultsCache", () => { + test("accepts a well-formed cache", () => { + expect( + isResultsCache({ + version: "5.0.0", + results: [["unit:src/a.unit.test.ts", { duration: 1, failed: false }]], + }), + ).toBe(true); + }); + + test.each([ + [undefined], + [{}], + [{ version: "5.0.0" }], + [{ version: "5.0.0", results: "nope" }], + [{ version: "5.0.0", results: [["unit:a.ts", { duration: "1", failed: false }]] }], + ])("rejects %j", (value) => { + expect(isResultsCache(value)).toBe(false); + }); +}); + +describe("readResultsCaches", () => { + test("finds every results.json recursively under a directory", () => { + writeCache("apps/cli/aaa", { + version: "5.0.0", + results: [["unit:src/a.unit.test.ts", { duration: 10, failed: false }]], + }); + writeCache("packages/api/bbb", { + version: "5.0.0", + results: [["unit:src/b.unit.test.ts", { duration: 20, failed: false }]], + }); + + const caches = readResultsCaches(dir!); + + expect(caches).toHaveLength(2); + expect(caches.flatMap((cache) => cache.results)).toEqual( + expect.arrayContaining([ + ["unit:src/a.unit.test.ts", { duration: 10, failed: false }], + ["unit:src/b.unit.test.ts", { duration: 20, failed: false }], + ]), + ); + }); +}); + +describe("readResultEntries", () => { + test("derives the workspace from the staged apps/ or packages/ path segment and splits the project prefix", () => { + writeCache("apps/cli/aaa", { + version: "5.0.0", + results: [ + ["unit:src/a.unit.test.ts", { duration: 10, failed: false }], + ["e2e:src/a.e2e.test.ts", { duration: 30, failed: true }], + ], + }); + writeCache("packages/api/bbb", { + version: "5.0.0", + results: [["integration:src/b.integration.test.ts", { duration: 20, failed: false }]], + }); + + const entries = readResultEntries(dir!); + + expect(entries).toEqual( + expect.arrayContaining([ + { + workspace: "apps/cli", + project: "unit", + path: "src/a.unit.test.ts", + duration: 10, + failed: false, + }, + { + workspace: "apps/cli", + project: "e2e", + path: "src/a.e2e.test.ts", + duration: 30, + failed: true, + }, + { + workspace: "packages/api", + project: "integration", + path: "src/b.integration.test.ts", + duration: 20, + failed: false, + }, + ]), + ); + }); + + test('falls back to "." for an artifact predating the staging step', () => { + writeCache("da39a3ee5e6b4b0d3255bfef95601890afd80709", { + version: "5.0.0", + results: [["e2e:src/a.e2e.test.ts", { duration: 10, failed: false }]], + }); + + expect(readResultEntries(dir!)).toEqual([ + { workspace: ".", project: "e2e", path: "src/a.e2e.test.ts", duration: 10, failed: false }, + ]); + }); +}); diff --git a/package.json b/package.json index 8f86e271b2..0098e58620 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "knip:check": "knip-bun", "knip:fix": "knip-bun --fix", "check:config-api": "bun tools/config-api-compare.ts", + "test-health-report": "bun .github/scripts/test-health-report/report.ts", "repos:install": "git submodule sync && git submodule update --init", "repos:pull": "git submodule sync && git submodule update --init --remote", "local-registry": "bun tools/release/local-registry.ts",