ci(cli): add a report-only CLI benchmark for pull requests

.github/scripts/cli-bench/github.ts

@@ -0,0 +1,69 @@
+/** Small GitHub REST helpers shared by the CLI benchmark scripts. */
+
+export function requireEnv(name: string): string {
+  const value = process.env[name];
+  if (!value) throw new Error(`Missing required environment variable: ${name}`);
+  return value;
+}
+
+/** A GitHub request that did not succeed, with the HTTP status and GitHub's own message. */
+export class GithubRequestError extends Error {
+  constructor(
+    readonly status: number,
+    readonly detail: string,
+    url: string,
+  ) {
+    super(`GitHub request failed (${status}) for ${url}`);
+    this.name = "GithubRequestError";
+  }
+}
+
+export async function githubFetch(
+  url: string,
+  token: string,
+  init: RequestInit = {},
+): Promise<Response> {
+  const response = await fetch(url, {
+    ...init,
+    headers: {
+      Authorization: `Bearer ${token}`,
+      Accept: "application/vnd.github+json",
+      "X-GitHub-Api-Version": "2022-11-28",
+      "Content-Type": "application/json",
+    },
+  });
+  if (!response.ok) {
+    const payload = (await response.json().catch(() => ({}))) as { message?: unknown };
+    const detail = typeof payload.message === "string" ? payload.message : response.statusText;
+    throw new GithubRequestError(response.status, detail, url);
+  }
+  return response;
+}
+
+export interface IssueComment {
+  id: number;
+  body: string;
+  authorLogin: string;
+}
+
+/** Every comment on an issue or pull request, oldest first. */
+export async function listIssueComments(
+  token: string,
+  base: string,
+  issue: number,
+): Promise<Array<IssueComment>> {
+  const comments: Array<IssueComment> = [];
+  for (let page = 1; ; page++) {
+    const batch = (await (
+      await githubFetch(`${base}/issues/${issue}/comments?per_page=100&page=${page}`, token)
+    ).json()) as Array<{ id: number; body: string | null; user: { login: string } | null }>;
+    comments.push(
+      ...batch.map((entry) => ({
+        id: entry.id,
+        body: entry.body ?? "",
+        authorLogin: entry.user?.login ?? "",
+      })),
+    );
+    if (batch.length < 100) return comments;
+  }
+}

.github/scripts/cli-bench/history.test.ts

@@ -0,0 +1,114 @@
+import { describe, expect, test } from "bun:test";
+import {
+  decodeHistory,
+  headlineOf,
+  makeEntry,
+  MAX_PREVIOUS_RUNS,
+  mergeHistory,
+  renderHistory,
+  type RunEntry,
+} from "./history.ts";
+
+const entry = (n: number, overrides: Partial<RunEntry> = {}): RunEntry => ({
+  sha: n.toString(16).padStart(7, "0"),
+  at: "2026-10-28T09:05Z",
+  headline: "no changes found",
+  url: `https://github.com/o/r/actions/runs/${n}`,
+  ...overrides,
+});
+
+describe("headlineOf", () => {
+  test("reads the title of the rendered report", () => {
+    expect(headlineOf("<!-- cli-bench:pr -->\n\n## CLI benchmark: 2 things to check\n\nbody")).toBe(
+      "2 things to check",
+    );
+  });
+
+  test("falls back when the report has no title", () => {
+    expect(headlineOf("nothing here")).toBe("unknown");
+  });
+});
+
+describe("makeEntry", () => {
+  test("shortens the commit and rounds the time down to the minute in UTC", () => {
+    const made = makeEntry({
+      sha: "0123456789abcdef0123456789abcdef01234567",
+      now: new Date("2026-10-28T09:05:59Z"),
+      headline: "no changes found",
+      url: "https://github.com/o/r/actions/runs/9/attempts/2",
+    });
+
+    expect(made).toMatchObject({ sha: "0123456", at: "2026-10-28T09:05Z" });
+  });
+
+  test("refuses a run link that is not a workflow run", () => {
+    const made = makeEntry({
+      sha: "abcdef1",
+      now: new Date(),
+      headline: "h",
+      url: "https://evil.test/x",
+    });
+
+    expect(made).toBeUndefined();
+  });
+});
+
+describe("mergeHistory", () => {
+  test("puts the new run first", () => {
+    expect(mergeHistory([entry(1)], entry(2)).map((e) => e.sha)).toEqual([
+      entry(2).sha,
+      entry(1).sha,
+    ]);
+  });
+
+  test("keeps the current run plus at most 20 earlier ones, dropping the oldest", () => {
+    const previous = Array.from({ length: 30 }, (_, i) => entry(i + 1));
+
+    const merged = mergeHistory(previous, entry(100));
+
+    expect(merged).toHaveLength(MAX_PREVIOUS_RUNS + 1);
+    expect(merged[0]).toEqual(entry(100));
+    expect(merged.at(-1)).toEqual(entry(MAX_PREVIOUS_RUNS));
+  });
+});
+
+describe("renderHistory and decodeHistory", () => {
+  test("round-trips the entries through the hidden marker", () => {
+    const entries = [entry(3), entry(2), entry(1)];
+
+    expect(decodeHistory(`body\n${renderHistory(entries, (t) => t)}\n`)).toEqual(entries);
+  });
+
+  test("lists earlier runs newest first and leaves the current run out of the list", () => {
+    const rendered = renderHistory(
+      [entry(3), entry(2, { headline: "1 thing to check" })],
+      (t) => t,
+    );
+
+    expect(rendered).toContain("<summary>Previous runs</summary>");
+    expect(rendered).toContain(
+      "- `0000002` · 2026-10-28 09:05 UTC · 1 thing to check · [run](https://github.com/o/r/actions/runs/2)",
+    );
+    expect(rendered).not.toContain("0000003` ·");
+  });
+
+  test("shows no list for a first run", () => {
+    expect(renderHistory([entry(1)], (t) => t)).not.toContain("Previous runs");
+  });
+
+  test.each([
+    ["no marker", "plain comment"],
+    ["not base64 json", "<!-- cli-bench:history bm90IGpzb24= -->"],
+    ["wrong shape", `<!-- cli-bench:history ${Buffer.from('[{"sha":1}]').toString("base64")} -->`],
+    [
+      "a forged run link",
+      `<!-- cli-bench:history ${Buffer.from(JSON.stringify([entry(1, { url: "https://evil.test/x" })])).toString("base64")} -->`,
+    ],
+    [
+      "too many entries",
+      `<!-- cli-bench:history ${Buffer.from(JSON.stringify(Array.from({ length: 22 }, (_, i) => entry(i + 1)))).toString("base64")} -->`,
+    ],
+  ])("treats %s as empty history", (_name, body) => {
+    expect(decodeHistory(body)).toEqual([]);
+  });
+});

.github/scripts/cli-bench/history.ts

@@ -0,0 +1,106 @@
+/**
+ * Run history kept inside the benchmark comment. The newest run is the comment itself; earlier
+ * runs sit in a collapsed list, backed by a hidden marker so the next update can carry them on.
+ * The marker is read back only from the workflow bot's own comment, and anything that does not
+ * decode cleanly counts as no history.
+ */
+
+/** Earlier runs kept below the current one. */
+export const MAX_PREVIOUS_RUNS = 20;
+const HISTORY_MARKER = /<!-- cli-bench:history ([A-Za-z0-9+/=]+) -->/u;
+const MAX_ENCODED_CHARS = 30_000;
+const MAX_HEADLINE_CHARS = 100;
+const SHA = /^[0-9a-f]{7,40}$/u;
+const AT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z$/u;
+const RUN_URL = /^https:\/\/[^\s)<>]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?$/u;
+
+export interface RunEntry {
+  /** Short commit SHA. */
+  sha: string;
+  /** UTC time to the minute, `YYYY-MM-DDTHH:MMZ`. */
+  at: string;
+  headline: string;
+  url: string;
+}
+
+const isEntry = (value: unknown): value is RunEntry => {
+  if (typeof value !== "object" || value === null) return false;
+  const { sha, at, headline, url } = value as Record<string, unknown>;
+  return (
+    typeof sha === "string" &&
+    SHA.test(sha) &&
+    typeof at === "string" &&
+    AT.test(at) &&
+    typeof headline === "string" &&
+    headline.length <= MAX_HEADLINE_CHARS &&
+    !/[\r\n]/u.test(headline) &&
+    typeof url === "string" &&
+    RUN_URL.test(url)
+  );
+};
+
+/** The title the renderer puts after `## CLI benchmark:`, flattened to one bounded line. */
+export function headlineOf(markdown: string): string {
+  const title = /^## CLI benchmark: (.+)$/mu.exec(markdown)?.[1] ?? "unknown";
+  return title.replaceAll(/\s+/gu, " ").trim().slice(0, MAX_HEADLINE_CHARS);
+}
+
+/** An entry for a run, or undefined when the commit or run link is not in the expected shape. */
+export function makeEntry(input: {
+  sha: string;
+  now: Date;
+  headline: string;
+  url: string;
+}): RunEntry | undefined {
+  const entry = {
+    sha: input.sha.slice(0, 7),
+    at: `${input.now.toISOString().slice(0, 16)}Z`,
+    headline: input.headline,
+    url: input.url,
+  };
+  return isEntry(entry) ? entry : undefined;
+}
+
+/** The runs recorded in a comment body, newest first; empty when the marker is absent or invalid. */
+export function decodeHistory(body: string): Array<RunEntry> {
+  const encoded = HISTORY_MARKER.exec(body)?.[1];
+  if (encoded === undefined || encoded.length > MAX_ENCODED_CHARS) return [];
+  try {
+    const value: unknown = JSON.parse(Buffer.from(encoded, "base64").toString("utf8"));
+    return Array.isArray(value) && value.length <= MAX_PREVIOUS_RUNS + 1 && value.every(isEntry)
+      ? value
+      : [];
+  } catch {
+    return [];
+  }
+}
+
+/** The new run first, then the earlier ones, with at most {@link MAX_PREVIOUS_RUNS} behind it. */
+export function mergeHistory(
+  previous: ReadonlyArray<RunEntry>,
+  current: RunEntry,
+): Array<RunEntry> {
+  return [current, ...previous].slice(0, MAX_PREVIOUS_RUNS + 1);
+}
+
+const day = (at: string) => `${at.slice(0, 10)} ${at.slice(11, 16)} UTC`;
+
+/**
+ * The hidden marker for `entries` (current run first), preceded by a collapsed list of the
+ * earlier ones when there are any. `clean` makes a headline safe to render.
+ */
+export function renderHistory(
+  entries: ReadonlyArray<RunEntry>,
+  clean: (text: string) => string,
+): string {
+  const marker = `<!-- cli-bench:history ${Buffer.from(JSON.stringify(entries)).toString("base64")} -->`;
+  const earlier = entries.slice(1);
+  if (earlier.length === 0) return marker;
+  const lines = earlier.map(
+    (entry) =>
+      `- \`${entry.sha}\` · ${day(entry.at)} · ${clean(entry.headline)} · [run](${entry.url})`,
+  );
+  return ["<details><summary>Previous runs</summary>", "", ...lines, "", "</details>", marker].join(
+    "\n",
+  );
+}