From d21b41cad3f2448549ef598cf63f00c87437fee1 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 29 Sep 2026 16:44:43 +0200 Subject: [PATCH 1/2] ci(repo): publish a weekly test-suite health report Makes suite growth, speed, and flakiness visible from CI data so later pruning routines and ratchets have something to act on. - Uploads Vitest results-cache timings from the unit and integration jobs, and the cli-e2e suite, alongside the existing e2e timings, staged under a per-workspace path so they stay attributable; raises retention to 14 days. - Shares the results-cache parser between the e2e timings refresh script and the new report instead of duplicating it. - Adds a weekly scheduled workflow that builds a job-summary report: test LOC by workspace/tier with week-over-week change, the fastest growing and slowest test files, and flaky-test signals from the week's develop/merge-queue Test runs. Advisory only. --- .github/workflows/test-health-report.yml | 44 +++ .github/workflows/test.yml | 86 ++++- apps/cli/scripts/refresh-e2e-timings.ts | 59 +-- apps/cli/scripts/vitest-results-cache.ts | 111 ++++++ .../scripts/vitest-results-cache.unit.test.ts | 122 ++++++ package.json | 1 + tools/lib/test-health-report.ts | 347 ++++++++++++++++++ tools/lib/test-health-report.unit.test.ts | 237 ++++++++++++ tools/test-health-report.ts | 304 +++++++++++++++ 9 files changed, 1251 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/test-health-report.yml create mode 100644 apps/cli/scripts/vitest-results-cache.ts create mode 100644 apps/cli/scripts/vitest-results-cache.unit.test.ts create mode 100644 tools/lib/test-health-report.ts create mode 100644 tools/lib/test-health-report.unit.test.ts create mode 100644 tools/test-health-report.ts diff --git a/.github/workflows/test-health-report.yml b/.github/workflows/test-health-report.yml new file mode 100644 index 0000000000..01f03269e0 --- /dev/null +++ b/.github/workflows/test-health-report.yml @@ -0,0 +1,44 @@ +name: Weekly test-suite health report + +# CLI-2541: makes suite growth, speed, and flakiness visible so later pruning routines and +# ratchets have data to act on. Advisory only — tools/test-health-report.ts never fails the job +# for what it finds, only for a tool failure, and this workflow gates nothing. +on: + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: test-health-report + cancel-in-progress: false + +jobs: + report: + name: Build report + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + actions: read + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The report diffs test LOC between HEAD and the last commit at least a week old. + fetch-depth: 0 + + # Bun and git/gh (preinstalled on the runner) only — no workspace install needed. + - name: Install toolchains + uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 + with: + version: 2026.9.0 + + - name: Build report + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bun tools/test-health-report.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 391542b18e..c08b298b6b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -100,6 +100,34 @@ jobs: - name: Run unit tests run: pnpm run test:unit + # Every workspace's Vitest results cache lands under its own + # `node_modules/.vite/vitest/`; staging them under one directory first (rather than + # uploading the raw glob) keeps each workspace's subtree in the artifact regardless of + # how many workspaces ran, so the weekly report (tools/test-health-report.ts) can tell + # them apart. See `.github/workflows/test.yml`'s e2e-timings upload for the sibling case. + - name: Stage unit test timings + if: always() + run: | + set -euo pipefail + mkdir -p .timings-staging + shopt -s nullglob + for cache in apps/*/node_modules/.vite/vitest/*/results.json packages/*/node_modules/.vite/vitest/*/results.json; do + workspace="${cache%%/node_modules/*}" + hash_dir="$(basename "$(dirname "$cache")")" + dest=".timings-staging/${workspace}/${hash_dir}" + mkdir -p "$dest" + cp "$cache" "$dest/results.json" + done + + - name: Upload unit test timings + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: unit-timings + path: .timings-staging + retention-days: 14 + if-no-files-found: warn + test-integration: if: | !startsWith(github.head_ref, 'release-notes/') && @@ -123,6 +151,30 @@ jobs: - name: Run integration tests run: pnpm run test:integration + # See the matching step in test-unit for why results caches are staged before upload. + - name: Stage integration test timings + if: always() + run: | + set -euo pipefail + mkdir -p .timings-staging + shopt -s nullglob + for cache in apps/*/node_modules/.vite/vitest/*/results.json packages/*/node_modules/.vite/vitest/*/results.json; do + workspace="${cache%%/node_modules/*}" + hash_dir="$(basename "$(dirname "$cache")")" + dest=".timings-staging/${workspace}/${hash_dir}" + mkdir -p "$dest" + cp "$cache" "$dest/results.json" + done + + - name: Upload integration test timings + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: integration-timings + path: .timings-staging + retention-days: 14 + if-no-files-found: warn + test-stack-ports: if: | !startsWith(github.head_ref, 'release-notes/') && @@ -248,15 +300,41 @@ jobs: env: SUPABASE_GO_BINARY: ${{ github.workspace }}/apps/cli-go/supabase-go - # Vitest writes its results cache under a sha1-named directory. The refresh script - # (apps/cli/scripts/refresh-e2e-timings.ts) merges these into tests/e2e-timings.json. + # Vitest writes its results cache under a sha1-named directory, under whichever + # workspace's suite ran on this matrix leg. Staging first (see the matching step in + # test-unit) keeps that workspace identifiable in the artifact. The refresh script + # (apps/cli/scripts/refresh-e2e-timings.ts) and the weekly report + # (tools/test-health-report.ts) both merge these back in from the results caches. + - name: Stage e2e test timings + if: always() + run: | + set -euo pipefail + mkdir -p .timings-staging + shopt -s nullglob + for cache in apps/*/node_modules/.vite/vitest/*/results.json; do + workspace="${cache%%/node_modules/*}" + hash_dir="$(basename "$(dirname "$cache")")" + dest=".timings-staging/${workspace}/${hash_dir}" + mkdir -p "$dest" + cp "$cache" "$dest/results.json" + done + - name: Upload e2e timings if: always() && matrix.suite == 'cli' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-timings-${{ matrix.shard }} - path: apps/cli/node_modules/.vite/vitest/**/results.json - retention-days: 7 + path: .timings-staging + retention-days: 14 + if-no-files-found: warn + + - name: Upload cli-e2e timings + if: always() && matrix.suite == 'cli-e2e' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-e2e-timings + path: .timings-staging + retention-days: 14 if-no-files-found: warn test-stack-e2e: diff --git a/apps/cli/scripts/refresh-e2e-timings.ts b/apps/cli/scripts/refresh-e2e-timings.ts index dd5396410b..d9196edc0d 100644 --- a/apps/cli/scripts/refresh-e2e-timings.ts +++ b/apps/cli/scripts/refresh-e2e-timings.ts @@ -1,24 +1,16 @@ // Rewrites apps/cli/tests/e2e-timings.json from the Vitest results caches that the test-e2e // workflow job uploads as `e2e-timings-` artifacts. import { spawnSync } from "node:child_process"; -import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import process from "node:process"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; import { version as installedVitestVersion } from "vitest/node"; +import { readResultsCaches, type VitestResultsCache } from "./vitest-results-cache.ts"; -export interface VitestFileResult { - readonly duration: number; - readonly failed: boolean; -} - -/** Shape of Vitest's `results.json` cache: `[:, result]` entries. */ -export interface VitestResultsCache { - readonly version: string; - readonly results: ReadonlyArray; -} +export type { VitestResultsCache } from "./vitest-results-cache.ts"; export interface MergeOptions { /** Version of the Vitest that will consume the timings; caches from another major are rejected. */ @@ -63,51 +55,6 @@ export function mergeTimings( return Object.fromEntries([...merged].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); } -function isFileResult(value: unknown): value is VitestFileResult { - return ( - typeof value === "object" && - value !== null && - "duration" in value && - typeof value.duration === "number" && - "failed" in value && - typeof value.failed === "boolean" - ); -} - -function isResultsEntry(value: unknown): value is readonly [string, VitestFileResult] { - return ( - Array.isArray(value) && - value.length === 2 && - typeof value[0] === "string" && - isFileResult(value[1]) - ); -} - -function isResultsCache(value: unknown): value is VitestResultsCache { - return ( - typeof value === "object" && - value !== null && - "version" in value && - typeof value.version === "string" && - "results" in value && - Array.isArray(value.results) && - value.results.every(isResultsEntry) - ); -} - -function readResultsCaches(dir: string): VitestResultsCache[] { - return readdirSync(dir, { recursive: true, withFileTypes: true }) - .filter((entry) => entry.isFile() && entry.name === "results.json") - .map((entry) => { - const file = join(entry.parentPath, entry.name); - const parsed: unknown = JSON.parse(readFileSync(file, "utf8")); - if (!isResultsCache(parsed)) { - throw new Error(`${file} is not a Vitest results cache`); - } - return parsed; - }); -} - const usage = `Usage: pnpm exec bun apps/cli/scripts/refresh-e2e-timings.ts --run [--repo supabase/cli] Downloads the e2e-timings-* artifacts of a test workflow run and rewrites diff --git a/apps/cli/scripts/vitest-results-cache.ts b/apps/cli/scripts/vitest-results-cache.ts new file mode 100644 index 0000000000..c356ca916e --- /dev/null +++ b/apps/cli/scripts/vitest-results-cache.ts @@ -0,0 +1,111 @@ +// Shared parsing for Vitest's results cache (normally at +// `/node_modules/.vite/vitest//results.json`, staged by `test.yml` as +// `//results.json` before upload — see the "Stage ... test timings" steps), +// used both by `apps/cli/scripts/refresh-e2e-timings.ts` and the weekly test-health report. +import { readdirSync, readFileSync } from "node:fs"; +import { join, relative } from "node:path"; + +interface VitestFileResult { + readonly duration: number; + readonly failed: boolean; +} + +/** Shape of Vitest's `results.json` cache: `[:, result]` entries. */ +export interface VitestResultsCache { + readonly version: string; + readonly results: ReadonlyArray; +} + +function isFileResult(value: unknown): value is VitestFileResult { + return ( + typeof value === "object" && + value !== null && + "duration" in value && + typeof value.duration === "number" && + "failed" in value && + typeof value.failed === "boolean" + ); +} + +function isResultsEntry(value: unknown): value is readonly [string, VitestFileResult] { + return ( + Array.isArray(value) && + value.length === 2 && + typeof value[0] === "string" && + isFileResult(value[1]) + ); +} + +export function isResultsCache(value: unknown): value is VitestResultsCache { + return ( + typeof value === "object" && + value !== null && + "version" in value && + typeof value.version === "string" && + "results" in value && + Array.isArray(value.results) && + value.results.every(isResultsEntry) + ); +} + +/** Reads every `results.json` found recursively under `dir`. */ +export function readResultsCaches(dir: string): VitestResultsCache[] { + return readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name === "results.json") + .map((entry) => { + const file = join(entry.parentPath, entry.name); + const parsed: unknown = JSON.parse(readFileSync(file, "utf8")); + if (!isResultsCache(parsed)) { + throw new Error(`${file} is not a Vitest results cache`); + } + return parsed; + }); +} + +/** A single project entry from a results cache, with the `:` prefix split out and the + * owning workspace (e.g. `apps/cli`) derived from where the cache file was found under `dir`. */ +export interface VitestResultEntry { + readonly workspace: string; + readonly project: string; + readonly path: string; + readonly duration: number; + readonly failed: boolean; +} + +function splitKey(key: string): { readonly project: string; readonly path: string } { + const separator = key.indexOf(":"); + return separator === -1 + ? { project: "", path: key } + : { project: key.slice(0, separator), path: key.slice(separator + 1) }; +} + +/** The `unit-timings`/`integration-timings`/`e2e-timings-*` artifacts stage each workspace's + * results caches under `//results.json` before upload (see `test.yml`), so the + * workspace is the `apps/` or `packages/` segment pair found in the cache's directory + * path; a path with neither (e.g. an artifact predating that staging step) is `"."`. */ +function workspaceFromCachePath(relativeDir: string): string { + const segments = relativeDir.split(/[/\\]+/); + const index = segments.findIndex((segment) => segment === "apps" || segment === "packages"); + return index === -1 || segments[index + 1] === undefined + ? "." + : `${segments[index]}/${segments[index + 1]}`; +} + +/** Flattens every results cache found (recursively) under `dir` into entries carrying the + * workspace, project, path, duration, and failed flag. */ +export function readResultEntries(dir: string): VitestResultEntry[] { + return readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name === "results.json") + .flatMap((entry) => { + const file = join(entry.parentPath, entry.name); + const parsed: unknown = JSON.parse(readFileSync(file, "utf8")); + if (!isResultsCache(parsed)) { + throw new Error(`${file} is not a Vitest results cache`); + } + const workspace = workspaceFromCachePath(relative(dir, entry.parentPath)); + return parsed.results.map(([key, result]) => { + const { project, path } = splitKey(key); + return { workspace, project, path, duration: result.duration, failed: result.failed }; + }); + }); +} diff --git a/apps/cli/scripts/vitest-results-cache.unit.test.ts b/apps/cli/scripts/vitest-results-cache.unit.test.ts new file mode 100644 index 0000000000..d6ef90469e --- /dev/null +++ b/apps/cli/scripts/vitest-results-cache.unit.test.ts @@ -0,0 +1,122 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; +import { isResultsCache, readResultEntries, readResultsCaches } from "./vitest-results-cache.ts"; + +let dir: string | undefined; + +afterEach(() => { + if (dir) { + rmSync(dir, { recursive: true, force: true }); + dir = undefined; + } +}); + +function writeCache(relativeDir: string, cache: unknown): void { + const cacheDir = join( + dir ?? (dir = mkdtempSync(join(tmpdir(), "vitest-results-cache-"))), + relativeDir, + ); + mkdirSync(cacheDir, { recursive: true }); + writeFileSync(join(cacheDir, "results.json"), JSON.stringify(cache)); +} + +describe("isResultsCache", () => { + test("accepts a well-formed cache", () => { + expect( + isResultsCache({ + version: "5.0.0", + results: [["unit:src/a.unit.test.ts", { duration: 1, failed: false }]], + }), + ).toBe(true); + }); + + test.each([ + [undefined], + [{}], + [{ version: "5.0.0" }], + [{ version: "5.0.0", results: "nope" }], + [{ version: "5.0.0", results: [["unit:a.ts", { duration: "1", failed: false }]] }], + ])("rejects %j", (value) => { + expect(isResultsCache(value)).toBe(false); + }); +}); + +describe("readResultsCaches", () => { + test("finds every results.json recursively under a directory", () => { + writeCache("apps/cli/aaa", { + version: "5.0.0", + results: [["unit:src/a.unit.test.ts", { duration: 10, failed: false }]], + }); + writeCache("packages/api/bbb", { + version: "5.0.0", + results: [["unit:src/b.unit.test.ts", { duration: 20, failed: false }]], + }); + + const caches = readResultsCaches(dir!); + + expect(caches).toHaveLength(2); + expect(caches.flatMap((cache) => cache.results)).toEqual( + expect.arrayContaining([ + ["unit:src/a.unit.test.ts", { duration: 10, failed: false }], + ["unit:src/b.unit.test.ts", { duration: 20, failed: false }], + ]), + ); + }); +}); + +describe("readResultEntries", () => { + test("derives the workspace from the staged apps/ or packages/ path segment and splits the project prefix", () => { + writeCache("apps/cli/aaa", { + version: "5.0.0", + results: [ + ["unit:src/a.unit.test.ts", { duration: 10, failed: false }], + ["e2e:src/a.e2e.test.ts", { duration: 30, failed: true }], + ], + }); + writeCache("packages/api/bbb", { + version: "5.0.0", + results: [["integration:src/b.integration.test.ts", { duration: 20, failed: false }]], + }); + + const entries = readResultEntries(dir!); + + expect(entries).toEqual( + expect.arrayContaining([ + { + workspace: "apps/cli", + project: "unit", + path: "src/a.unit.test.ts", + duration: 10, + failed: false, + }, + { + workspace: "apps/cli", + project: "e2e", + path: "src/a.e2e.test.ts", + duration: 30, + failed: true, + }, + { + workspace: "packages/api", + project: "integration", + path: "src/b.integration.test.ts", + duration: 20, + failed: false, + }, + ]), + ); + }); + + test('falls back to "." for an artifact predating the staging step', () => { + writeCache("da39a3ee5e6b4b0d3255bfef95601890afd80709", { + version: "5.0.0", + results: [["e2e:src/a.e2e.test.ts", { duration: 10, failed: false }]], + }); + + expect(readResultEntries(dir!)).toEqual([ + { workspace: ".", project: "e2e", path: "src/a.e2e.test.ts", duration: 10, failed: false }, + ]); + }); +}); diff --git a/package.json b/package.json index 8f86e271b2..130d8b7765 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "knip:check": "knip-bun", "knip:fix": "knip-bun --fix", "check:config-api": "bun tools/config-api-compare.ts", + "test-health-report": "bun tools/test-health-report.ts", "repos:install": "git submodule sync && git submodule update --init", "repos:pull": "git submodule sync && git submodule update --init --remote", "local-registry": "bun tools/release/local-registry.ts", diff --git a/tools/lib/test-health-report.ts b/tools/lib/test-health-report.ts new file mode 100644 index 0000000000..a5ce8918c4 --- /dev/null +++ b/tools/lib/test-health-report.ts @@ -0,0 +1,347 @@ +/** + * Pure pieces of the weekly test-suite health report (CLI-2541): test-file tier + * classification, LOC aggregation and week-over-week diffing, growth/duration ranking, and + * flaky-test detection. The impure half — git history, `gh` API calls, and artifact downloads — + * lives in `tools/test-health-report.ts`. + */ + +export type TestTier = "unit" | "integration" | "e2e" | "live" | "other"; + +const TIER_SUFFIXES: ReadonlyArray = [ + [".unit.test.ts", "unit"], + [".integration.test.ts", "integration"], + [".e2e.test.ts", "e2e"], + [".live.test.ts", "live"], +]; + +/** Tiers follow the `*.unit.test.ts` / `*.integration.test.ts` / `*.e2e.test.ts` / `*.live.test.ts` + * naming convention; a bare `*.test.ts` file (e.g. under `.github/scripts`) is `"other"`. */ +export function classifyTestTier(path: string): TestTier { + for (const [suffix, tier] of TIER_SUFFIXES) { + if (path.endsWith(suffix)) { + return tier; + } + } + return "other"; +} + +/** The owning workspace for a repo-relative path, e.g. `apps/cli` or `packages/api`; anything + * outside `apps/*` or `packages/*` (root scripts, `.github/scripts`, `tools/`) is `"root"`. */ +export function workspaceForPath(path: string): string { + const match = /^(apps|packages)\/([^/]+)\//.exec(path); + return match ? `${match[1]}/${match[2]}` : "root"; +} + +export interface TestFileLoc { + readonly path: string; + readonly loc: number; +} + +export interface WorkspaceTierLoc { + readonly workspace: string; + readonly tier: TestTier; + readonly loc: number; + readonly fileCount: number; +} + +function groupKey(workspace: string, tier: string): string { + return `${workspace}\u0000${tier}`; +} + +/** Sums LOC per workspace/tier pair, sorted by workspace then tier for stable rendering. */ +export function aggregateLocByWorkspaceTier(files: readonly TestFileLoc[]): WorkspaceTierLoc[] { + const totals = new Map(); + for (const file of files) { + const key = groupKey(workspaceForPath(file.path), classifyTestTier(file.path)); + const existing = totals.get(key) ?? { loc: 0, fileCount: 0 }; + totals.set(key, { loc: existing.loc + file.loc, fileCount: existing.fileCount + 1 }); + } + return [...totals.entries()] + .map(([key, value]) => { + const [workspace = "root", tier = "other"] = key.split("\u0000"); + return { workspace, tier: tier as TestTier, ...value }; + }) + .sort((a, b) => a.workspace.localeCompare(b.workspace) || a.tier.localeCompare(b.tier)); +} + +export interface WorkspaceTierChange extends WorkspaceTierLoc { + readonly previousLoc: number; + readonly change: number; +} + +/** Joins this week's per-workspace/tier LOC against last week's, keeping every group that had + * LOC in either snapshot (a tier that emptied out this week still shows its drop to zero). */ +export function diffLocByWorkspaceTier( + current: readonly WorkspaceTierLoc[], + previous: readonly WorkspaceTierLoc[], +): WorkspaceTierChange[] { + const previousByKey = new Map( + previous.map((entry) => [groupKey(entry.workspace, entry.tier), entry.loc]), + ); + const currentByKey = new Map( + current.map((entry) => [groupKey(entry.workspace, entry.tier), entry]), + ); + const keys = new Set([...currentByKey.keys(), ...previousByKey.keys()]); + + const changes: WorkspaceTierChange[] = []; + for (const key of keys) { + const [workspace = "root", tier = "other"] = key.split("\u0000"); + const entry = currentByKey.get(key); + const previousLoc = previousByKey.get(key) ?? 0; + const loc = entry?.loc ?? 0; + changes.push({ + workspace, + tier: tier as TestTier, + loc, + fileCount: entry?.fileCount ?? 0, + previousLoc, + change: loc - previousLoc, + }); + } + return changes.sort( + (a, b) => a.workspace.localeCompare(b.workspace) || a.tier.localeCompare(b.tier), + ); +} + +export interface FileGrowth { + readonly path: string; + readonly loc: number; + readonly previousLoc: number; + readonly growth: number; +} + +/** The files whose LOC grew the most over the week, largest growth first; ties broken by path. + * A file absent from `previous` counts as growth from zero. Files that shrank or disappeared are + * excluded — this ranks growth, not overall churn. */ +export function fastestGrowingFiles( + current: readonly TestFileLoc[], + previous: readonly TestFileLoc[], + limit = 10, +): FileGrowth[] { + const previousByPath = new Map(previous.map((entry) => [entry.path, entry.loc])); + return current + .map((file) => { + const previousLoc = previousByPath.get(file.path) ?? 0; + return { path: file.path, loc: file.loc, previousLoc, growth: file.loc - previousLoc }; + }) + .filter((entry) => entry.growth > 0) + .sort((a, b) => b.growth - a.growth || a.path.localeCompare(b.path)) + .slice(0, limit); +} + +export interface TimingEntry { + readonly workspace: string; + readonly path: string; + readonly duration: number; + readonly failed: boolean; +} + +export interface FileDuration { + readonly workspace: string; + readonly path: string; + readonly durationMs: number; +} + +/** The slowest test files over the week, by the longest duration recorded for each file across + * every run's results cache (a file's duration only reflects its own run, so the max is the best + * single estimate of how slow it can be). */ +export function slowestFiles(entries: readonly TimingEntry[], limit = 10): FileDuration[] { + const slowestByKey = new Map(); + for (const entry of entries) { + const key = groupKey(entry.workspace, entry.path); + const existing = slowestByKey.get(key); + const durationMs = Math.round(entry.duration); + if (!existing || durationMs > existing.durationMs) { + slowestByKey.set(key, { workspace: entry.workspace, path: entry.path, durationMs }); + } + } + return [...slowestByKey.values()] + .sort((a, b) => b.durationMs - a.durationMs || a.path.localeCompare(b.path)) + .slice(0, limit); +} + +export interface FlakyFile { + readonly workspace: string; + readonly path: string; + readonly failureCount: number; +} + +/** Files that failed in at least one of the week's develop/merge-queue runs, most-failed first. + * Only detects a file failing within a run's own results cache — it can't distinguish "flaky" from + * "broken and later fixed", since both look identical from a single week of run outcomes. */ +export function flakyFiles(entries: readonly TimingEntry[]): FlakyFile[] { + const failuresByKey = new Map(); + for (const entry of entries) { + if (!entry.failed) { + continue; + } + const key = groupKey(entry.workspace, entry.path); + const existing = failuresByKey.get(key); + failuresByKey.set(key, { + workspace: entry.workspace, + path: entry.path, + failureCount: (existing?.failureCount ?? 0) + 1, + }); + } + return [...failuresByKey.values()].sort( + (a, b) => b.failureCount - a.failureCount || a.path.localeCompare(b.path), + ); +} + +export interface RunAttemptJobs { + readonly runId: number; + readonly runUrl: string; + /** The attempt number of the run's current (latest) state. */ + readonly finalAttempt: number; + readonly finalConclusion: string | null; + /** Job name/conclusion pairs from the run's first attempt. */ + readonly attempt1Jobs: ReadonlyArray<{ + readonly name: string; + readonly conclusion: string | null; + }>; +} + +export interface RetriedJobFailure { + readonly runId: number; + readonly runUrl: string; + readonly finalAttempt: number; + readonly jobName: string; +} + +/** + * Runs that ultimately succeeded but needed a re-run, with the job(s) that failed on attempt 1. + * This detects retry-flakiness at job granularity only: the Actions API reports per-job, not + * per-test-file, outcomes for a past attempt, so a job spanning many test files (e.g. "Run unit + * tests") can't be narrowed down to the one file that failed without parsing job logs. + */ +export function retriedJobFailures(runs: readonly RunAttemptJobs[]): RetriedJobFailure[] { + const failures: RetriedJobFailure[] = []; + for (const run of runs) { + if (run.finalConclusion !== "success" || run.finalAttempt <= 1) { + continue; + } + for (const job of run.attempt1Jobs) { + if (job.conclusion === "failure") { + failures.push({ + runId: run.runId, + runUrl: run.runUrl, + finalAttempt: run.finalAttempt, + jobName: job.name, + }); + } + } + } + return failures; +} + +function formatDuration(ms: number): string { + return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${ms}ms`; +} + +function formatChange(change: number): string { + if (change === 0) { + return "0"; + } + return change > 0 ? `+${change}` : `${change}`; +} + +function markdownTable(headers: readonly string[], rows: ReadonlyArray): string { + if (rows.length === 0) { + return "_none_"; + } + const headerRow = `| ${headers.join(" | ")} |`; + const separatorRow = `| ${headers.map(() => "---").join(" | ")} |`; + const bodyRows = rows.map((row) => `| ${row.join(" | ")} |`); + return [headerRow, separatorRow, ...bodyRows].join("\n"); +} + +interface ReportMetadata { + readonly headSha: string; + readonly weekAgoSha: string; + readonly weekAgoDate: string; + readonly runsAnalyzed: number; + /** Caveats surfaced to the reader, e.g. sampling limits or missing artifact kinds. */ + readonly notes: readonly string[]; +} + +export interface ReportInput { + readonly metadata: ReportMetadata; + readonly locChanges: readonly WorkspaceTierChange[]; + readonly fastestGrowing: readonly FileGrowth[]; + readonly slowest: readonly FileDuration[]; + readonly flakyFiles: readonly FlakyFile[]; + readonly retriedJobFailures: readonly RetriedJobFailure[]; +} + +/** Renders the full report as GitHub-flavored markdown, suitable for `$GITHUB_STEP_SUMMARY`. */ +export function renderMarkdownReport(input: ReportInput): string { + const { metadata } = input; + const lines: string[] = [ + "# Weekly test-suite health report", + "", + `Comparing \`${metadata.headSha.slice(0, 12)}\` against \`${metadata.weekAgoSha.slice(0, 12)}\` (${metadata.weekAgoDate}), across ${metadata.runsAnalyzed} develop/merge-queue Test run(s) from the past week.`, + "", + ]; + + if (metadata.notes.length > 0) { + lines.push(...metadata.notes.map((note) => `> ${note}`), ""); + } + + lines.push( + "## Test LOC by workspace and tier", + "", + markdownTable( + ["Workspace", "Tier", "LOC", "Files", "Δ vs last week"], + input.locChanges + .filter((entry) => entry.loc > 0 || entry.previousLoc > 0) + .map((entry) => [ + entry.workspace, + entry.tier, + String(entry.loc), + String(entry.fileCount), + formatChange(entry.change), + ]), + ), + "", + "## Fastest-growing test files (by LOC)", + "", + markdownTable( + ["Path", "LOC", "Δ vs last week"], + input.fastestGrowing.map((entry) => [ + entry.path, + String(entry.loc), + formatChange(entry.growth), + ]), + ), + "", + "## Slowest test files", + "", + markdownTable( + ["Workspace", "Path", "Duration"], + input.slowest.map((entry) => [entry.workspace, entry.path, formatDuration(entry.durationMs)]), + ), + "", + "## Flaky tests", + "", + "### Files that failed in a develop/merge-queue run", + "", + markdownTable( + ["Workspace", "Path", "Failures"], + input.flakyFiles.map((entry) => [entry.workspace, entry.path, String(entry.failureCount)]), + ), + "", + "### Runs where attempt 1 failed and a later attempt succeeded", + "", + markdownTable( + ["Run", "Job", "Succeeded on attempt"], + input.retriedJobFailures.map((entry) => [ + `[${entry.runId}](${entry.runUrl})`, + entry.jobName, + String(entry.finalAttempt), + ]), + ), + "", + ); + + return lines.join("\n"); +} diff --git a/tools/lib/test-health-report.unit.test.ts b/tools/lib/test-health-report.unit.test.ts new file mode 100644 index 0000000000..3adeb309ed --- /dev/null +++ b/tools/lib/test-health-report.unit.test.ts @@ -0,0 +1,237 @@ +import { describe, expect, test } from "bun:test"; +import { + aggregateLocByWorkspaceTier, + classifyTestTier, + diffLocByWorkspaceTier, + fastestGrowingFiles, + flakyFiles, + renderMarkdownReport, + retriedJobFailures, + slowestFiles, + workspaceForPath, +} from "./test-health-report.ts"; + +describe("classifyTestTier", () => { + test.each([ + ["src/foo.unit.test.ts", "unit"], + ["src/foo.integration.test.ts", "integration"], + ["src/foo.e2e.test.ts", "e2e"], + ["src/foo.live.test.ts", "live"], + [".github/scripts/contribution-gate.test.ts", "other"], + ])("classifies %s as %s", (path, tier) => { + expect(classifyTestTier(path)).toBe(tier); + }); +}); + +describe("workspaceForPath", () => { + test.each([ + ["apps/cli/src/foo.unit.test.ts", "apps/cli"], + ["packages/api/src/foo.integration.test.ts", "packages/api"], + [".github/scripts/contribution-gate.test.ts", "root"], + ["tools/lib/test-health-report.unit.test.ts", "root"], + ])("maps %s to workspace %s", (path, workspace) => { + expect(workspaceForPath(path)).toBe(workspace); + }); +}); + +describe("aggregateLocByWorkspaceTier", () => { + test("sums LOC per workspace/tier and counts files", () => { + const totals = aggregateLocByWorkspaceTier([ + { path: "apps/cli/src/a.unit.test.ts", loc: 10 }, + { path: "apps/cli/src/b.unit.test.ts", loc: 20 }, + { path: "apps/cli/src/c.integration.test.ts", loc: 5 }, + { path: "packages/api/src/a.unit.test.ts", loc: 7 }, + ]); + + expect(totals).toEqual([ + { workspace: "apps/cli", tier: "integration", loc: 5, fileCount: 1 }, + { workspace: "apps/cli", tier: "unit", loc: 30, fileCount: 2 }, + { workspace: "packages/api", tier: "unit", loc: 7, fileCount: 1 }, + ]); + }); +}); + +describe("diffLocByWorkspaceTier", () => { + test("computes week-over-week change and keeps groups that emptied out", () => { + const current = [{ workspace: "apps/cli", tier: "unit" as const, loc: 30, fileCount: 2 }]; + const previous = [ + { workspace: "apps/cli", tier: "unit" as const, loc: 10, fileCount: 1 }, + { workspace: "apps/cli", tier: "integration" as const, loc: 40, fileCount: 3 }, + ]; + + expect(diffLocByWorkspaceTier(current, previous)).toEqual([ + { + workspace: "apps/cli", + tier: "integration", + loc: 0, + fileCount: 0, + previousLoc: 40, + change: -40, + }, + { workspace: "apps/cli", tier: "unit", loc: 30, fileCount: 2, previousLoc: 10, change: 20 }, + ]); + }); +}); + +describe("fastestGrowingFiles", () => { + test("ranks growth descending and treats a new file as growth from zero", () => { + const current = [ + { path: "src/a.unit.test.ts", loc: 100 }, + { path: "src/b.unit.test.ts", loc: 50 }, + { path: "src/new.unit.test.ts", loc: 30 }, + { path: "src/shrunk.unit.test.ts", loc: 5 }, + ]; + const previous = [ + { path: "src/a.unit.test.ts", loc: 40 }, + { path: "src/b.unit.test.ts", loc: 45 }, + { path: "src/shrunk.unit.test.ts", loc: 50 }, + ]; + + expect(fastestGrowingFiles(current, previous, 2)).toEqual([ + { path: "src/a.unit.test.ts", loc: 100, previousLoc: 40, growth: 60 }, + { path: "src/new.unit.test.ts", loc: 30, previousLoc: 0, growth: 30 }, + ]); + }); + + test("excludes files that shrank or stayed the same", () => { + const current = [{ path: "src/a.unit.test.ts", loc: 10 }]; + const previous = [{ path: "src/a.unit.test.ts", loc: 10 }]; + + expect(fastestGrowingFiles(current, previous)).toEqual([]); + }); +}); + +describe("slowestFiles", () => { + test("keeps the max duration per file across runs and ranks descending", () => { + const entries = [ + { workspace: "apps/cli", path: "src/a.e2e.test.ts", duration: 1200.4, failed: false }, + { workspace: "apps/cli", path: "src/a.e2e.test.ts", duration: 900, failed: false }, + { workspace: "apps/cli", path: "src/b.e2e.test.ts", duration: 5000, failed: false }, + ]; + + expect(slowestFiles(entries, 10)).toEqual([ + { workspace: "apps/cli", path: "src/b.e2e.test.ts", durationMs: 5000 }, + { workspace: "apps/cli", path: "src/a.e2e.test.ts", durationMs: 1200 }, + ]); + }); + + test("limits to the requested count", () => { + const entries = Array.from({ length: 15 }, (_, index) => ({ + workspace: "apps/cli", + path: `src/f${index}.e2e.test.ts`, + duration: index, + failed: false, + })); + + expect(slowestFiles(entries, 10)).toHaveLength(10); + }); +}); + +describe("flakyFiles", () => { + test("counts failures per file and ignores passing entries", () => { + const entries = [ + { workspace: "apps/cli", path: "src/a.e2e.test.ts", duration: 100, failed: true }, + { workspace: "apps/cli", path: "src/a.e2e.test.ts", duration: 100, failed: true }, + { workspace: "apps/cli", path: "src/b.e2e.test.ts", duration: 100, failed: false }, + ]; + + expect(flakyFiles(entries)).toEqual([ + { workspace: "apps/cli", path: "src/a.e2e.test.ts", failureCount: 2 }, + ]); + }); +}); + +describe("retriedJobFailures", () => { + test("reports attempt-1 failures only for runs that ultimately succeeded after a retry", () => { + const runs = [ + { + runId: 1, + runUrl: "https://example.com/1", + finalAttempt: 2, + finalConclusion: "success", + attempt1Jobs: [ + { name: "Run unit tests", conclusion: "failure" }, + { name: "Run integration tests", conclusion: "success" }, + ], + }, + { + // Never retried — attempt 1 is the only (and final) attempt. + runId: 2, + runUrl: "https://example.com/2", + finalAttempt: 1, + finalConclusion: "success", + attempt1Jobs: [{ name: "Run unit tests", conclusion: "failure" }], + }, + { + // Retried but still failing — not a flaky "later succeeded" case. + runId: 3, + runUrl: "https://example.com/3", + finalAttempt: 2, + finalConclusion: "failure", + attempt1Jobs: [{ name: "Run unit tests", conclusion: "failure" }], + }, + ]; + + expect(retriedJobFailures(runs)).toEqual([ + { runId: 1, runUrl: "https://example.com/1", finalAttempt: 2, jobName: "Run unit tests" }, + ]); + }); +}); + +describe("renderMarkdownReport", () => { + test("renders every section with an empty-state placeholder when there is no data", () => { + const markdown = renderMarkdownReport({ + metadata: { + headSha: "abcdef123456789", + weekAgoSha: "111111222222333", + weekAgoDate: "2026-09-22", + runsAnalyzed: 3, + notes: ["sampled the last 3 runs for this demo"], + }, + locChanges: [], + fastestGrowing: [], + slowest: [], + flakyFiles: [], + retriedJobFailures: [], + }); + + expect(markdown).toContain("# Weekly test-suite health report"); + expect(markdown).toContain("abcdef123456"); + expect(markdown).toContain("> sampled the last 3 runs for this demo"); + expect(markdown).toContain("## Test LOC by workspace and tier"); + expect(markdown).toContain("_none_"); + }); + + test("renders populated tables with formatted durations and changes", () => { + const markdown = renderMarkdownReport({ + metadata: { + headSha: "abcdef123456789", + weekAgoSha: "111111222222333", + weekAgoDate: "2026-09-22", + runsAnalyzed: 12, + notes: [], + }, + locChanges: [ + { + workspace: "apps/cli", + tier: "unit", + loc: 120, + fileCount: 4, + previousLoc: 100, + change: 20, + }, + ], + fastestGrowing: [{ path: "src/a.unit.test.ts", loc: 120, previousLoc: 100, growth: 20 }], + slowest: [{ workspace: "apps/cli", path: "src/a.e2e.test.ts", durationMs: 12345 }], + flakyFiles: [{ workspace: "apps/cli", path: "src/a.e2e.test.ts", failureCount: 3 }], + retriedJobFailures: [ + { runId: 42, runUrl: "https://example.com/42", finalAttempt: 2, jobName: "Run unit tests" }, + ], + }); + + expect(markdown).toContain("| apps/cli | unit | 120 | 4 | +20 |"); + expect(markdown).toContain("12.3s"); + expect(markdown).toContain("| apps/cli | src/a.e2e.test.ts | 3 |"); + expect(markdown).toContain("[42](https://example.com/42)"); + }); +}); diff --git a/tools/test-health-report.ts b/tools/test-health-report.ts new file mode 100644 index 0000000000..8994a00428 --- /dev/null +++ b/tools/test-health-report.ts @@ -0,0 +1,304 @@ +/** + * Builds the weekly test-suite health report (CLI-2541): test LOC by workspace/tier with + * week-over-week change, the fastest-growing and slowest test files, and flaky-test signals — + * all from git history plus the `test-*-timings-*` artifacts uploaded by `.github/workflows/test.yml`. + * Writes to `$GITHUB_STEP_SUMMARY` when set, and always prints to stdout. Advisory only; never + * exits non-zero for data findings, only for a tool failure. + * + * Usage: bun tools/test-health-report.ts [--repo owner/name] [--days 7] [--max-runs N] + */ + +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path, { join } from "node:path"; +import process from "node:process"; +import { parseArgs } from "node:util"; +import { + aggregateLocByWorkspaceTier, + diffLocByWorkspaceTier, + fastestGrowingFiles, + flakyFiles, + renderMarkdownReport, + retriedJobFailures, + slowestFiles, + type RunAttemptJobs, + type TestFileLoc, + type TimingEntry, +} from "./lib/test-health-report.ts"; +import { readResultEntries } from "../apps/cli/scripts/vitest-results-cache.ts"; + +const repoRoot = path.resolve(import.meta.dir, ".."); + +interface ProcessResult { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; +} + +async function run(command: readonly string[]): Promise { + const proc = Bun.spawn([...command], { cwd: repoRoot, stdout: "pipe", stderr: "pipe" }); + const [exitCode, stdout, stderr] = await Promise.all([ + proc.exited, + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + return { exitCode, stdout, stderr }; +} + +async function runGit(args: readonly string[]): Promise { + return run(["git", ...args]); +} + +async function ghApiJson(apiPath: string): Promise { + const result = await run(["gh", "api", apiPath]); + if (result.exitCode !== 0) { + throw new Error(`gh api ${apiPath} failed: ${result.stderr.trim()}`); + } + return JSON.parse(result.stdout) as T; +} + +/** Every tracked `*.test.ts` file and its line count at `ref`, via one `git grep -c` call — + * matches a manual `git ls-files '*.test.ts' | xargs wc -l` count at that ref. */ +async function listTestFilesWithLoc(ref: string): Promise { + const result = await runGit(["grep", "--no-color", "-I", "-c", "", ref, "--", "*.test.ts"]); + // git grep exits 1 (with empty stdout) when nothing matches; that's not a tool failure. + if (result.exitCode !== 0 && result.stdout.trim() === "") { + if (result.exitCode === 1) { + return []; + } + throw new Error(`git grep -c '' ${ref} -- '*.test.ts' failed: ${result.stderr.trim()}`); + } + const files: TestFileLoc[] = []; + for (const line of result.stdout.split("\n")) { + if (line === "") { + continue; + } + const match = /^[0-9a-f]+:(.+):(\d+)$/.exec(line); + if (!match) { + continue; + } + const [, filePath, loc] = match; + files.push({ path: filePath ?? "", loc: Number(loc) }); + } + return files; +} + +async function resolveWeekAgoSha(headSha: string, days: number): Promise { + const result = await runGit(["log", "-1", `--before=${days} days ago`, "--format=%H", headSha]); + const sha = result.stdout.trim(); + return sha === "" ? headSha : sha; +} + +async function commitDate(sha: string): Promise { + const result = await runGit(["log", "-1", "--format=%cs", sha]); + return result.stdout.trim(); +} + +interface WorkflowRunSummary { + readonly id: number; + readonly html_url: string; + readonly created_at: string; + readonly conclusion: string | null; + readonly run_attempt: number; +} + +async function listWorkflowRuns( + repo: string, + event: string, + sinceIso: string, + branch?: string, +): Promise { + const runs: WorkflowRunSummary[] = []; + for (let page = 1; page <= 50; page++) { + const params = new URLSearchParams({ + event, + per_page: "100", + page: String(page), + created: `>=${sinceIso}`, + }); + if (branch) { + params.set("branch", branch); + } + const data = await ghApiJson<{ workflow_runs: WorkflowRunSummary[] }>( + `repos/${repo}/actions/workflows/test.yml/runs?${params.toString()}`, + ); + runs.push(...data.workflow_runs); + if (data.workflow_runs.length < 100) { + break; + } + } + return runs; +} + +/** The week's develop pushes and merge-queue runs of the Test workflow, deduped and time-ordered. */ +async function listRunsForWindow(repo: string, sinceIso: string): Promise { + const [mergeQueueRuns, pushRuns] = await Promise.all([ + listWorkflowRuns(repo, "merge_group", sinceIso), + listWorkflowRuns(repo, "push", sinceIso, "develop"), + ]); + const byId = new Map(); + for (const run of [...mergeQueueRuns, ...pushRuns]) { + byId.set(run.id, run); + } + return [...byId.values()].sort((a, b) => a.created_at.localeCompare(b.created_at)); +} + +/** Downloads a run's `*-timings-*` artifacts into `destDir`; returns false when the run has none + * (e.g. it was cancelled before the upload step), which is not a failure. */ +async function downloadTimingArtifacts( + repo: string, + runId: number, + destDir: string, +): Promise { + const result = await run([ + "gh", + "run", + "download", + String(runId), + "--repo", + repo, + "--pattern", + "*-timings-*", + "--dir", + destDir, + ]); + if (result.exitCode !== 0) { + if (!/no artifacts? found|no valid artifacts/i.test(result.stderr)) { + console.warn(`[test-health-report] gh run download ${runId} failed: ${result.stderr.trim()}`); + } + return false; + } + return true; +} + +async function attempt1Jobs( + repo: string, + runId: number, +): Promise> { + const data = await ghApiJson<{ jobs: Array<{ name: string; conclusion: string | null }> }>( + `repos/${repo}/actions/runs/${runId}/attempts/1/jobs`, + ); + return data.jobs.map((job) => ({ name: job.name, conclusion: job.conclusion })); +} + +async function writeStepSummary(markdown: string): Promise { + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (!summaryPath) { + return; + } + await Bun.write(summaryPath, `${markdown}\n`, { createPath: true }); +} + +const usage = `Usage: bun tools/test-health-report.ts [--repo owner/name] [--days 7] [--max-runs N] + + Builds the weekly test-suite health report from git history and the week's + develop/merge-queue Test workflow run artifacts. --max-runs caps how many + matching runs are downloaded, for local dry runs or rate-limit safety.`; + +if (import.meta.main) { + const { values } = parseArgs({ + args: process.argv.slice(2), + options: { + repo: { type: "string", default: "supabase/cli" }, + days: { type: "string", default: "7" }, + "max-runs": { type: "string" }, + help: { type: "boolean", default: false }, + }, + }); + + if (values.help) { + console.log(usage); + process.exit(0); + } + + const repo = values.repo ?? "supabase/cli"; + const days = Number(values.days ?? "7"); + const maxRuns = values["max-runs"] !== undefined ? Number(values["max-runs"]) : undefined; + const notes: string[] = []; + + try { + const headSha = (await runGit(["rev-parse", "HEAD"])).stdout.trim(); + const weekAgoSha = await resolveWeekAgoSha(headSha, days); + const weekAgoDate = await commitDate(weekAgoSha); + + const [currentFiles, previousFiles] = await Promise.all([ + listTestFilesWithLoc(headSha), + listTestFilesWithLoc(weekAgoSha), + ]); + const locChanges = diffLocByWorkspaceTier( + aggregateLocByWorkspaceTier(currentFiles), + aggregateLocByWorkspaceTier(previousFiles), + ); + const fastestGrowing = fastestGrowingFiles(currentFiles, previousFiles); + + const sinceIso = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString(); + let runs = await listRunsForWindow(repo, sinceIso); + if (maxRuns !== undefined && runs.length > maxRuns) { + notes.push( + `sampled the ${maxRuns} most recent of ${runs.length} matching runs from the past ` + + `${days} day(s) (--max-runs).`, + ); + runs = runs.slice(-maxRuns); + } + + const timingEntries: TimingEntry[] = []; + const runAttemptInfos: RunAttemptJobs[] = []; + const downloadRoot = mkdtempSync(join(tmpdir(), "test-health-report-")); + try { + for (const workflowRun of runs) { + const runDir = join(downloadRoot, String(workflowRun.id)); + if (await downloadTimingArtifacts(repo, workflowRun.id, runDir)) { + try { + timingEntries.push(...readResultEntries(runDir)); + } catch (cause) { + console.warn( + `[test-health-report] could not parse timing artifacts for run ${workflowRun.id}: ` + + `${cause instanceof Error ? cause.message : String(cause)}`, + ); + } + } + + if (workflowRun.run_attempt > 1) { + runAttemptInfos.push({ + runId: workflowRun.id, + runUrl: workflowRun.html_url, + finalAttempt: workflowRun.run_attempt, + finalConclusion: workflowRun.conclusion, + attempt1Jobs: await attempt1Jobs(repo, workflowRun.id), + }); + } + } + } finally { + rmSync(downloadRoot, { recursive: true, force: true }); + } + + if (timingEntries.length === 0 && runs.length > 0) { + notes.push( + "no timing artifacts were found for this week's runs — unit/integration timing uploads " + + "only exist once this report's own workflow changes have merged; only e2e timings are " + + "available until then.", + ); + } else if (timingEntries.some((entry) => entry.workspace === ".")) { + notes.push( + 'some timing entries show workspace "." — they came from an e2e-timings artifact ' + + "uploaded before the per-workspace staging step landed, so their workspace could not " + + "be recovered.", + ); + } + + const markdown = renderMarkdownReport({ + metadata: { headSha, weekAgoSha, weekAgoDate, runsAnalyzed: runs.length, notes }, + locChanges, + fastestGrowing, + slowest: slowestFiles(timingEntries), + flakyFiles: flakyFiles(timingEntries), + retriedJobFailures: retriedJobFailures(runAttemptInfos), + }); + + console.log(markdown); + await writeStepSummary(markdown); + } catch (cause) { + console.error(`error: ${cause instanceof Error ? cause.message : String(cause)}`); + process.exitCode = 1; + } +} From 1db6966f31dcdd2d80c40d36772fcb2f3baeab00 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 29 Sep 2026 16:52:17 +0200 Subject: [PATCH 2/2] ci(repo): move the weekly test-health report under .github/scripts Puts the report where bun test ./.github/scripts and its type-check already run it, and de-duplicates the per-job results-cache staging into one script. - Moves tools/test-health-report.ts and tools/lib/test-health-report.ts into .github/scripts/test-health-report/ as report.ts and logic.ts, with logic.test.ts running under the existing github-scripts-ci.yml. - Replaces the copy-pasted staging loop in the unit, integration, and e2e jobs with .github/scripts/stage-vitest-results.sh. - Adds apps/cli/scripts/vitest-results-cache.ts to github-scripts-ci.yml's trigger paths, since the report imports it. - Trims comments to the repo's one-or-two-line, why-only convention. --- .github/scripts/stage-vitest-results.sh | 22 +++++++ .../scripts/test-health-report/logic.test.ts | 4 +- .../scripts/test-health-report/logic.ts | 27 +++------ .../scripts/test-health-report/report.ts | 22 +++---- .github/workflows/github-scripts-ci.yml | 1 + .github/workflows/test-health-report.yml | 7 +-- .github/workflows/test.yml | 58 +++++-------------- apps/cli/scripts/vitest-results-cache.ts | 12 ++-- package.json | 2 +- 9 files changed, 64 insertions(+), 91 deletions(-) create mode 100644 .github/scripts/stage-vitest-results.sh rename tools/lib/test-health-report.unit.test.ts => .github/scripts/test-health-report/logic.test.ts (99%) rename tools/lib/test-health-report.ts => .github/scripts/test-health-report/logic.ts (88%) rename tools/test-health-report.ts => .github/scripts/test-health-report/report.ts (90%) diff --git a/.github/scripts/stage-vitest-results.sh b/.github/scripts/stage-vitest-results.sh new file mode 100644 index 0000000000..5a88d07aa0 --- /dev/null +++ b/.github/scripts/stage-vitest-results.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Stages each workspace's Vitest results cache under ///results.json +# before upload, so actions/upload-artifact's common-ancestor stripping can't drop the workspace +# from the artifact's paths. +# +# Usage: stage-vitest-results.sh ... +set -euo pipefail +shopt -s nullglob + +staging_dir="$1" +shift + +mkdir -p "$staging_dir" +for pattern in "$@"; do + for cache in $pattern; do + workspace="${cache%%/node_modules/*}" + hash_dir="$(basename "$(dirname "$cache")")" + dest="$staging_dir/$workspace/$hash_dir" + mkdir -p "$dest" + cp "$cache" "$dest/results.json" + done +done diff --git a/tools/lib/test-health-report.unit.test.ts b/.github/scripts/test-health-report/logic.test.ts similarity index 99% rename from tools/lib/test-health-report.unit.test.ts rename to .github/scripts/test-health-report/logic.test.ts index 3adeb309ed..c5c5b6ad91 100644 --- a/tools/lib/test-health-report.unit.test.ts +++ b/.github/scripts/test-health-report/logic.test.ts @@ -9,7 +9,7 @@ import { retriedJobFailures, slowestFiles, workspaceForPath, -} from "./test-health-report.ts"; +} from "./logic.ts"; describe("classifyTestTier", () => { test.each([ @@ -18,7 +18,7 @@ describe("classifyTestTier", () => { ["src/foo.e2e.test.ts", "e2e"], ["src/foo.live.test.ts", "live"], [".github/scripts/contribution-gate.test.ts", "other"], - ])("classifies %s as %s", (path, tier) => { + ] as const)("classifies %s as %s", (path, tier) => { expect(classifyTestTier(path)).toBe(tier); }); }); diff --git a/tools/lib/test-health-report.ts b/.github/scripts/test-health-report/logic.ts similarity index 88% rename from tools/lib/test-health-report.ts rename to .github/scripts/test-health-report/logic.ts index a5ce8918c4..062cb7c47a 100644 --- a/tools/lib/test-health-report.ts +++ b/.github/scripts/test-health-report/logic.ts @@ -1,8 +1,6 @@ /** - * Pure pieces of the weekly test-suite health report (CLI-2541): test-file tier - * classification, LOC aggregation and week-over-week diffing, growth/duration ranking, and - * flaky-test detection. The impure half — git history, `gh` API calls, and artifact downloads — - * lives in `tools/test-health-report.ts`. + * Pure pieces of the weekly test-suite health report: tier classification, LOC diffing, ranking, + * and flaky-test detection. `report.ts` holds the git/API/artifact-fetching half. */ export type TestTier = "unit" | "integration" | "e2e" | "live" | "other"; @@ -110,9 +108,8 @@ export interface FileGrowth { readonly growth: number; } -/** The files whose LOC grew the most over the week, largest growth first; ties broken by path. - * A file absent from `previous` counts as growth from zero. Files that shrank or disappeared are - * excluded — this ranks growth, not overall churn. */ +/** Files ranked by LOC growth over the week; a file absent from `previous` counts as growth from + * zero, and files that shrank or disappeared are excluded. */ export function fastestGrowingFiles( current: readonly TestFileLoc[], previous: readonly TestFileLoc[], @@ -142,9 +139,7 @@ export interface FileDuration { readonly durationMs: number; } -/** The slowest test files over the week, by the longest duration recorded for each file across - * every run's results cache (a file's duration only reflects its own run, so the max is the best - * single estimate of how slow it can be). */ +/** The slowest test files over the week, by the longest duration recorded for each across runs. */ export function slowestFiles(entries: readonly TimingEntry[], limit = 10): FileDuration[] { const slowestByKey = new Map(); for (const entry of entries) { @@ -166,9 +161,7 @@ export interface FlakyFile { readonly failureCount: number; } -/** Files that failed in at least one of the week's develop/merge-queue runs, most-failed first. - * Only detects a file failing within a run's own results cache — it can't distinguish "flaky" from - * "broken and later fixed", since both look identical from a single week of run outcomes. */ +/** Files that failed in at least one run this week; can't distinguish flaky from broken-then-fixed. */ export function flakyFiles(entries: readonly TimingEntry[]): FlakyFile[] { const failuresByKey = new Map(); for (const entry of entries) { @@ -208,12 +201,8 @@ export interface RetriedJobFailure { readonly jobName: string; } -/** - * Runs that ultimately succeeded but needed a re-run, with the job(s) that failed on attempt 1. - * This detects retry-flakiness at job granularity only: the Actions API reports per-job, not - * per-test-file, outcomes for a past attempt, so a job spanning many test files (e.g. "Run unit - * tests") can't be narrowed down to the one file that failed without parsing job logs. - */ +/** Jobs that failed on attempt 1 of a run that later succeeded; job granularity only, since the + * Actions API has no per-test-file outcome for a past attempt. */ export function retriedJobFailures(runs: readonly RunAttemptJobs[]): RetriedJobFailure[] { const failures: RetriedJobFailure[] = []; for (const run of runs) { diff --git a/tools/test-health-report.ts b/.github/scripts/test-health-report/report.ts similarity index 90% rename from tools/test-health-report.ts rename to .github/scripts/test-health-report/report.ts index 8994a00428..fe8c652127 100644 --- a/tools/test-health-report.ts +++ b/.github/scripts/test-health-report/report.ts @@ -1,11 +1,9 @@ /** - * Builds the weekly test-suite health report (CLI-2541): test LOC by workspace/tier with - * week-over-week change, the fastest-growing and slowest test files, and flaky-test signals — - * all from git history plus the `test-*-timings-*` artifacts uploaded by `.github/workflows/test.yml`. - * Writes to `$GITHUB_STEP_SUMMARY` when set, and always prints to stdout. Advisory only; never - * exits non-zero for data findings, only for a tool failure. + * Builds the weekly test-suite health report from git history plus the `*-timings-*` artifacts + * uploaded by `.github/workflows/test.yml`. Advisory only: never exits non-zero for a data + * finding, only for a tool failure. * - * Usage: bun tools/test-health-report.ts [--repo owner/name] [--days 7] [--max-runs N] + * Usage: bun .github/scripts/test-health-report/report.ts [--repo owner/name] [--days 7] [--max-runs N] */ import { mkdtempSync, rmSync } from "node:fs"; @@ -24,10 +22,10 @@ import { type RunAttemptJobs, type TestFileLoc, type TimingEntry, -} from "./lib/test-health-report.ts"; -import { readResultEntries } from "../apps/cli/scripts/vitest-results-cache.ts"; +} from "./logic.ts"; +import { readResultEntries } from "../../../apps/cli/scripts/vitest-results-cache.ts"; -const repoRoot = path.resolve(import.meta.dir, ".."); +const repoRoot = path.resolve(import.meta.dir, "../../.."); interface ProcessResult { readonly exitCode: number; @@ -189,11 +187,9 @@ async function writeStepSummary(markdown: string): Promise { await Bun.write(summaryPath, `${markdown}\n`, { createPath: true }); } -const usage = `Usage: bun tools/test-health-report.ts [--repo owner/name] [--days 7] [--max-runs N] +const usage = `Usage: bun .github/scripts/test-health-report/report.ts [--repo owner/name] [--days 7] [--max-runs N] - Builds the weekly test-suite health report from git history and the week's - develop/merge-queue Test workflow run artifacts. --max-runs caps how many - matching runs are downloaded, for local dry runs or rate-limit safety.`; + --max-runs caps how many matching runs are downloaded, for local dry runs or rate-limit safety.`; if (import.meta.main) { const { values } = parseArgs({ diff --git a/.github/workflows/github-scripts-ci.yml b/.github/workflows/github-scripts-ci.yml index c2f12fa74b..01cb497ace 100644 --- a/.github/workflows/github-scripts-ci.yml +++ b/.github/workflows/github-scripts-ci.yml @@ -12,6 +12,7 @@ on: - "apps/cli/src/shared/services/Dockerfile" - "apps/cli/src/shared/services/parse-dockerfile-service-images.ts" - "apps/cli/src/shared/services/slim-images.ts" + - "apps/cli/scripts/vitest-results-cache.ts" - "packages/stack/src/Artifacts.ts" - ".github/workflows/ai-review.yml" - ".github/workflows/github-scripts-ci.yml" diff --git a/.github/workflows/test-health-report.yml b/.github/workflows/test-health-report.yml index 01f03269e0..391df8bd6d 100644 --- a/.github/workflows/test-health-report.yml +++ b/.github/workflows/test-health-report.yml @@ -1,8 +1,7 @@ name: Weekly test-suite health report -# CLI-2541: makes suite growth, speed, and flakiness visible so later pruning routines and -# ratchets have data to act on. Advisory only — tools/test-health-report.ts never fails the job -# for what it finds, only for a tool failure, and this workflow gates nothing. +# Advisory only — the report script never fails the job for what it finds, only for a tool +# failure, and this workflow gates nothing. on: schedule: - cron: "0 6 * * 1" @@ -41,4 +40,4 @@ jobs: - name: Build report env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: bun tools/test-health-report.ts + run: bun .github/scripts/test-health-report/report.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c08b298b6b..ee083bc58a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -100,24 +100,13 @@ jobs: - name: Run unit tests run: pnpm run test:unit - # Every workspace's Vitest results cache lands under its own - # `node_modules/.vite/vitest/`; staging them under one directory first (rather than - # uploading the raw glob) keeps each workspace's subtree in the artifact regardless of - # how many workspaces ran, so the weekly report (tools/test-health-report.ts) can tell - # them apart. See `.github/workflows/test.yml`'s e2e-timings upload for the sibling case. + # Stage per workspace so the artifact keeps the workspace in each path. - name: Stage unit test timings if: always() - run: | - set -euo pipefail - mkdir -p .timings-staging - shopt -s nullglob - for cache in apps/*/node_modules/.vite/vitest/*/results.json packages/*/node_modules/.vite/vitest/*/results.json; do - workspace="${cache%%/node_modules/*}" - hash_dir="$(basename "$(dirname "$cache")")" - dest=".timings-staging/${workspace}/${hash_dir}" - mkdir -p "$dest" - cp "$cache" "$dest/results.json" - done + run: >- + bash .github/scripts/stage-vitest-results.sh .timings-staging + 'apps/*/node_modules/.vite/vitest/*/results.json' + 'packages/*/node_modules/.vite/vitest/*/results.json' - name: Upload unit test timings if: always() @@ -151,20 +140,13 @@ jobs: - name: Run integration tests run: pnpm run test:integration - # See the matching step in test-unit for why results caches are staged before upload. + # Stage per workspace so the artifact keeps the workspace in each path. - name: Stage integration test timings if: always() - run: | - set -euo pipefail - mkdir -p .timings-staging - shopt -s nullglob - for cache in apps/*/node_modules/.vite/vitest/*/results.json packages/*/node_modules/.vite/vitest/*/results.json; do - workspace="${cache%%/node_modules/*}" - hash_dir="$(basename "$(dirname "$cache")")" - dest=".timings-staging/${workspace}/${hash_dir}" - mkdir -p "$dest" - cp "$cache" "$dest/results.json" - done + run: >- + bash .github/scripts/stage-vitest-results.sh .timings-staging + 'apps/*/node_modules/.vite/vitest/*/results.json' + 'packages/*/node_modules/.vite/vitest/*/results.json' - name: Upload integration test timings if: always() @@ -300,24 +282,12 @@ jobs: env: SUPABASE_GO_BINARY: ${{ github.workspace }}/apps/cli-go/supabase-go - # Vitest writes its results cache under a sha1-named directory, under whichever - # workspace's suite ran on this matrix leg. Staging first (see the matching step in - # test-unit) keeps that workspace identifiable in the artifact. The refresh script - # (apps/cli/scripts/refresh-e2e-timings.ts) and the weekly report - # (tools/test-health-report.ts) both merge these back in from the results caches. + # Stage per workspace so the artifact keeps the workspace in each path. - name: Stage e2e test timings if: always() - run: | - set -euo pipefail - mkdir -p .timings-staging - shopt -s nullglob - for cache in apps/*/node_modules/.vite/vitest/*/results.json; do - workspace="${cache%%/node_modules/*}" - hash_dir="$(basename "$(dirname "$cache")")" - dest=".timings-staging/${workspace}/${hash_dir}" - mkdir -p "$dest" - cp "$cache" "$dest/results.json" - done + run: >- + bash .github/scripts/stage-vitest-results.sh .timings-staging + 'apps/*/node_modules/.vite/vitest/*/results.json' - name: Upload e2e timings if: always() && matrix.suite == 'cli' diff --git a/apps/cli/scripts/vitest-results-cache.ts b/apps/cli/scripts/vitest-results-cache.ts index c356ca916e..df42ca7c04 100644 --- a/apps/cli/scripts/vitest-results-cache.ts +++ b/apps/cli/scripts/vitest-results-cache.ts @@ -1,7 +1,5 @@ -// Shared parsing for Vitest's results cache (normally at -// `/node_modules/.vite/vitest//results.json`, staged by `test.yml` as -// `//results.json` before upload — see the "Stage ... test timings" steps), -// used both by `apps/cli/scripts/refresh-e2e-timings.ts` and the weekly test-health report. +// Shared parsing for Vitest's results cache, staged before upload as +// `//results.json` so the workspace survives artifact upload. import { readdirSync, readFileSync } from "node:fs"; import { join, relative } from "node:path"; @@ -79,10 +77,8 @@ function splitKey(key: string): { readonly project: string; readonly path: strin : { project: key.slice(0, separator), path: key.slice(separator + 1) }; } -/** The `unit-timings`/`integration-timings`/`e2e-timings-*` artifacts stage each workspace's - * results caches under `//results.json` before upload (see `test.yml`), so the - * workspace is the `apps/` or `packages/` segment pair found in the cache's directory - * path; a path with neither (e.g. an artifact predating that staging step) is `"."`. */ +/** The workspace is the `apps/` or `packages/` segment pair in the cache's directory + * path; a path with neither (e.g. an artifact predating staging) is `"."`. */ function workspaceFromCachePath(relativeDir: string): string { const segments = relativeDir.split(/[/\\]+/); const index = segments.findIndex((segment) => segment === "apps" || segment === "packages"); diff --git a/package.json b/package.json index 130d8b7765..0098e58620 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "knip:check": "knip-bun", "knip:fix": "knip-bun --fix", "check:config-api": "bun tools/config-api-compare.ts", - "test-health-report": "bun tools/test-health-report.ts", + "test-health-report": "bun .github/scripts/test-health-report/report.ts", "repos:install": "git submodule sync && git submodule update --init", "repos:pull": "git submodule sync && git submodule update --init --remote", "local-registry": "bun tools/release/local-registry.ts",