diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6787f36..f68ca43 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -14,8 +14,8 @@ CONTRIBUTING.md explains how commit types decide releases. ## Testing +For visible changes, the screens bot comments with before/after images; when none shows your change, +extend a scene in scenes/. Add a short recording when motion matters: upload it here, don't commit it. --> ## Merging diff --git a/.github/scripts/capture-site.mjs b/.github/scripts/capture-site.mjs new file mode 100644 index 0000000..c79377f --- /dev/null +++ b/.github/scripts/capture-site.mjs @@ -0,0 +1,112 @@ +// Captures the website for the before/after comparison: node capture-site.mjs SITE OUT +// Each page of the SITE folder at desktop and phone size, full page, in light mode to +// OUT/-.png and in dark mode to OUT/--dark.png. Requests are answered from +// the folder, without a server, and nothing else is fetched. So that two captures of one commit +// match, motion is reduced, Math.random is seeded, and the page's clock stands at Thursday 22 October +// 2026, 09:41 UTC, moving only when the capture lets 5 seconds of timers and frames run. Playwright +// is resolved from the working folder; STABLEBAR_CAPTURE_CHANNEL=chrome uses the installed Chrome +// instead of Playwright's Chromium. +import { createRequire } from "node:module"; +import { mkdirSync, realpathSync, statSync } from "node:fs"; +import { join, resolve, sep } from "node:path"; +import { pathToFileURL } from "node:url"; + +export const origin = "http://stablebar.test"; +const pages = ["index", "privacy", "terms", "404"]; +const schemes = { light: "", dark: "-dark" }; +const sizes = { + desktop: { viewport: { width: 1440, height: 900 }, deviceScaleFactor: 2 }, + phone: { viewport: { width: 390, height: 844 }, deviceScaleFactor: 3, isMobile: true, hasTouch: true }, +}; +const fixedTime = new Date("2026-10-22T09:41:00Z"); +/** How long the page's timers and animation frames run, on its stopped clock, before the capture. */ +const settleTime = 5000; + +/** Replaces Math.random with a seeded generator (mulberry32), in every document before its scripts. */ +function seedRandom() { + let seed = 0x5eed; + Math.random = () => { + seed = (seed + 0x6d2b79f5) | 0; + let value = Math.imul(seed ^ (seed >>> 15), 1 | seed); + value = (value + Math.imul(value ^ (value >>> 7), 61 | value)) ^ value; + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +} + +/** + * The real path of the file in the `root` folder that `url` asks for, or null when it asks for none. + * Symbolic links count only when they lead to a file inside the folder. + */ +export function fileFor(root, url) { + const { origin: asked, pathname } = new URL(url); + if (asked !== origin) return null; + try { + const folder = realpathSync(root); + const path = resolve(folder, `.${decodeURIComponent(pathname)}`); + const file = realpathSync(pathname.endsWith("/") ? join(path, "index.html") : path); + return file.startsWith(folder + sep) && statSync(file).isFile() ? file : null; + } catch { + return null; + } +} + +/** + * Waits for the fonts and for every image to load and decode. A broken image is the same in every + * capture, so it only gets a warning, and shows in the comparison. + */ +async function settle(tab) { + await tab.waitForLoadState("networkidle"); + const broken = await tab.evaluate(async () => { + await document.fonts.ready; + const images = [...document.images]; + for (const image of images) image.loading = "eager"; + const results = await Promise.all(images.map((image) => image.decode().then(() => null, () => image.getAttribute("src")))); + return results.filter(Boolean); + }); + if (broken.length > 0) console.log(`::warning::${new URL(tab.url()).pathname}: these images don't load: ${broken.join(", ")}`); +} + +export async function capture(site, out, { channel } = {}) { + const root = resolve(site); + const { chromium } = createRequire(join(process.cwd(), "capture-site.cjs"))("playwright"); + mkdirSync(out, { recursive: true }); + const browser = await chromium.launch(channel ? { channel } : {}); + try { + for (const [size, options] of Object.entries(sizes)) { + for (const [scheme, suffix] of Object.entries(schemes)) { + const context = await browser.newContext({ ...options, colorScheme: scheme, reducedMotion: "reduce", locale: "en-US", timezoneId: "UTC" }); + await context.addInitScript(seedRandom); + await context.route("**/*", (route) => { + const url = route.request().url(); + const file = fileFor(root, url); + if (file) return route.fulfill({ path: file }); + return url.startsWith(`${origin}/`) ? route.fulfill({ status: 404, body: "" }) : route.abort(); + }); + for (const page of pages) { + const tab = await context.newPage(); + await tab.clock.install({ time: fixedTime }); + await tab.clock.pauseAt(fixedTime); + await tab.goto(`${origin}/${page}.html`, { waitUntil: "load" }); + await settle(tab); + await tab.clock.runFor(settleTime); + await settle(tab); + await tab.screenshot({ path: join(out, `${page}-${size}${suffix}.png`), fullPage: true, animations: "disabled" }); + await tab.close(); + } + await context.close(); + } + } + } finally { + await browser.close(); + } +} + +// Node names a module by its real path, so the path it was called by is resolved first. +if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) { + const [site, out] = process.argv.slice(2); + if (!site || !out) { + console.error("usage: capture-site.mjs SITE OUT"); + process.exit(64); + } + await capture(site, out, { channel: process.env.STABLEBAR_CAPTURE_CHANNEL }); +} diff --git a/.github/scripts/capture-site.test.mjs b/.github/scripts/capture-site.test.mjs new file mode 100644 index 0000000..930c433 --- /dev/null +++ b/.github/scripts/capture-site.test.mjs @@ -0,0 +1,29 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, realpathSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileFor, origin } from "./capture-site.mjs"; + +test("fileFor answers requests from the site folder only", () => { + const root = mkdtempSync(join(tmpdir(), "site-")); + mkdirSync(join(root, "site", "images"), { recursive: true }); + const site = join(root, "site"); + writeFileSync(join(site, "index.html"), "

home

"); + writeFileSync(join(site, "images", "a b.webp"), "image"); + writeFileSync(join(root, "secret.txt"), "secret"); + symlinkSync(join(root, "secret.txt"), join(site, "leak.txt")); + symlinkSync(join(site, "index.html"), join(site, "home.html")); + const real = realpathSync(site); + assert.equal(fileFor(site, `${origin}/`), join(real, "index.html")); + assert.equal(fileFor(site, `${origin}/index.html?x=1#top`), join(real, "index.html")); + assert.equal(fileFor(site, `${origin}/images/a%20b.webp`), join(real, "images", "a b.webp")); + assert.equal(fileFor(site, `${origin}/home.html`), join(real, "index.html")); + assert.equal(fileFor(site, `${origin}/leak.txt`), null); + assert.equal(fileFor(site, `${origin}/missing.html`), null); + assert.equal(fileFor(site, `${origin}/images`), null); + assert.equal(fileFor(site, `${origin}/%2e%2e/secret.txt`), null); + assert.equal(fileFor(site, `${origin}/..%2fsecret.txt`), null); + assert.equal(fileFor(site, `${origin}/%E0%A4%A`), null); + assert.equal(fileFor(site, "https://stablebar.app/index.html"), null); +}); diff --git a/.github/scripts/compare-screens.mjs b/.github/scripts/compare-screens.mjs new file mode 100644 index 0000000..db3826f --- /dev/null +++ b/.github/scripts/compare-screens.mjs @@ -0,0 +1,262 @@ +// Compares the screens drawn from a pull request (after) with those drawn from its base (before): +// node compare-screens.mjs BEFORE AFTER OUT --prefix docs [--before-report scenes.json] [--after-report scenes.json] +// It writes cropped before, after and difference images of each changed screen to OUT, and adds the +// screens to OUT/manifest.json: {screens: [{id, status, changedPixels, region, images, afterDigest?, +// truncated?, baseError?}], failed: [{id, error}]}. +// The trusted half of the bot checks that manifest strictly: see readManifest in screens-comment.mjs. +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, readdirSync, realpathSync, writeFileSync, mkdirSync } from "node:fs"; +import { join, basename } from "node:path"; +import { pathToFileURL } from "node:url"; +import { decodePNG, encodePNG } from "./png.mjs"; + +/** A pixel changes when one of its channels differs by more than this. */ +const noise = 1; +/** Context around a changed region, in image pixels: 24 points at 2×, 16 at 3×. */ +const padding = 48; +/** The smallest crop, so a small change still shows what's around it. */ +const smallest = { width: 480, height: 160 }; +/** A crop's most rows: a long change shows its start. */ +const tallest = 1600; +/** The longest message the trusted half takes. */ +const maximumMessage = 1000; + +const cut = (text, limit) => (text.length <= limit ? text : `${text.slice(0, limit - 1)}…`); +/** A digest of an image's pixels: a changed or resized screen carries its whole after image's, since its crop may show only part. */ +const digestOf = ({ width, height, rgba }) => createHash("sha256").update(`${width}x${height}\n`).update(rgba).digest("hex"); +/** Image names the bot can show, since they become file and asset names. */ +const isPlainName = (name) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name) && !name.includes("..") && name.length <= 80; + +/** Which pixels of the top-left `width` by `height` of both images differ. */ +function changedMask(before, after, width, height) { + const mask = new Uint8Array(width * height); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const a = (y * before.width + x) * 4, b = (y * after.width + x) * 4; + for (let channel = 0; channel < 4; channel++) { + if (Math.abs(before.rgba[a + channel] - after.rgba[b + channel]) > noise) { + mask[y * width + x] = 1; + break; + } + } + } + } + return mask; +} + +/** + * How `after` differs from `before`, either of which may be null. A changed pixel with no changed + * neighbour is noise: it neither makes a screen changed nor widens its region. Screens of one width + * but new heights are compared from the top: their region runs from the first change, or from where + * the shorter one ends, to the longer one's end, across the full width. A new width can't be lined up. + */ +export function comparePair(before, after) { + if (!before) return { status: "added", changedPixels: 0, region: null }; + if (!after) return { status: "removed", changedPixels: 0, region: null }; + if (before.width !== after.width) return { status: "resized", changedPixels: 0, region: null }; + const width = before.width, height = Math.min(before.height, after.height); + const { changedPixels, region } = significant(changedMask(before, after, width, height), width, height); + if (before.height === after.height) { + return changedPixels ? { status: "changed", changedPixels, region } : { status: "unchanged", changedPixels: 0, region: null }; + } + const longer = Math.max(before.height, after.height), top = region ? region.y : height; + return { status: "resized", changedPixels: changedPixels + (longer - height) * width, region: { x: 0, y: top, width, height: longer - top } }; +} + +/** The changed pixels of `mask` that have a changed neighbour, and the region around them. */ +function significant(mask, width, height) { + let changedPixels = 0, minX = width, minY = height, maxX = -1, maxY = -1; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + if (!mask[y * width + x]) continue; + let neighbours = 0; + for (let dy = -1; dy <= 1 && !neighbours; dy++) { + for (let dx = -1; dx <= 1; dx++) { + const nx = x + dx, ny = y + dy; + if ((dx || dy) && nx >= 0 && ny >= 0 && nx < width && ny < height && mask[ny * width + nx]) { + neighbours++; + break; + } + } + } + if (!neighbours) continue; + changedPixels++; + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + } + if (!changedPixels) return { changedPixels: 0, region: null }; + return { changedPixels, region: { x: minX, y: minY, width: maxX - minX + 1, height: maxY - minY + 1 } }; +} + +/** `region` with `padding` around it, grown to the smallest crop, moved inside `image` and cut to it and to `tallest`. */ +function padded(region, image) { + const span = (start, length, size, minimum) => { + let from = start - padding, to = start + length + padding; + if (to - from < minimum) { + const grow = minimum - (to - from); + from -= Math.floor(grow / 2); + to += Math.ceil(grow / 2); + } + if (from < 0) { + to -= from; + from = 0; + } + if (to > size) { + from = Math.max(0, from - (to - size)); + to = size; + } + return [from, to]; + }; + const [x0, x1] = span(region.x, region.width, image.width, smallest.width); + const [y0, y1] = span(region.y, region.height, image.height, smallest.height); + return { x: x0, y: y0, width: x1 - x0, height: Math.min(y1, y0 + tallest) - y0, cut: y1 > y0 + tallest }; +} + +/** The crop of `image` for a screen of a new height: full width, from `padding` above the first change, at most `tallest` rows. */ +function resizedCrop(region, image) { + const y = Math.max(0, region.y - padding); + return { x: 0, y, width: image.width, height: Math.min(image.height, y + tallest) - y, cut: image.height > y + tallest }; +} + +/** The part of `image` inside `region`. */ +export function crop(image, region) { + const rgba = new Uint8Array(region.width * region.height * 4); + for (let row = 0; row < region.height; row++) { + const start = ((region.y + row) * image.width + region.x) * 4; + rgba.set(image.rgba.subarray(start, start + region.width * 4), row * region.width * 4); + } + return { width: region.width, height: region.height, rgba }; +} + +/** `after` inside `region`, at 30% brightness, with every changed pixel, and every row `before` doesn't have, red. */ +export function differenceImage(before, after, region) { + const overlap = Math.min(before.height, after.height); + const mask = changedMask(before, after, after.width, overlap); + const rgba = new Uint8Array(region.width * region.height * 4); + for (let row = 0; row < region.height; row++) { + const y = region.y + row; + for (let column = 0; column < region.width; column++) { + const x = region.x + column, source = (y * after.width + x) * 4, target = (row * region.width + column) * 4; + if (y >= overlap || mask[y * after.width + x]) { + rgba.set([255, 59, 48, 255], target); + } else { + rgba.set([after.rgba[source] * 0.3, after.rgba[source + 1] * 0.3, after.rgba[source + 2] * 0.3, 255], target); + } + } + } + return { width: region.width, height: region.height, rgba }; +} + +function read(folder, name) { + const path = join(folder, name); + return existsSync(path) ? decodePNG(readFileSync(path)) : null; +} + +/** The scene name an image of `name-.png` belongs to. */ +function sceneName(image) { + return basename(image, ".png").replace(/-(dark|light)$/, ""); +} + +/** + * Compares every PNG in `beforeDir` and `afterDir`, writes the images of changed, added, removed and + * resized screens to `outDir`, and returns the screens, and as failures the images whose names the + * bot can't show. `beforeErrors` holds, by scene name, why the base couldn't draw a scene, and + * `beforeError` why it drew none. + */ +export function compareDirectories(beforeDir, afterDir, outDir, { prefix, beforeErrors = {}, beforeError }) { + const list = (folder) => (existsSync(folder) ? readdirSync(folder).filter((name) => name.endsWith(".png")) : []); + const all = [...new Set([...list(beforeDir), ...list(afterDir)])].sort(); + const names = all.filter((name) => isPlainName(basename(name, ".png"))); + const failed = all.filter((name) => !names.includes(name)).map((name) => ({ id: cut(`${prefix}/${basename(name, ".png")}`, 200), + error: "Its name may use only letters, digits, dots, dashes and underscores." })); + mkdirSync(outDir, { recursive: true }); + const screens = names.map((name) => { + const before = read(beforeDir, name), after = read(afterDir, name); + const id = `${prefix}/${basename(name, ".png")}`; + const result = comparePair(before, after); + const file = (kind) => `${id.replaceAll("/", "~")}-${kind}.png`; + const images = {}; + let truncated = false; + const save = (kind, image) => { + writeFileSync(join(outDir, file(kind)), encodePNG(image)); + images[kind] = file(kind); + }; + if (result.status === "changed") { + const region = padded(result.region, after); + save("before", crop(before, region)); + save("after", crop(after, region)); + save("diff", differenceImage(before, after, region)); + truncated = region.cut; + } else if (result.status === "resized" && result.region) { + const beforeCrop = resizedCrop(result.region, before), afterCrop = resizedCrop(result.region, after); + save("before", crop(before, beforeCrop)); + save("after", crop(after, afterCrop)); + save("diff", differenceImage(before, after, afterCrop)); + truncated = beforeCrop.cut || afterCrop.cut; + } else if (result.status === "resized") { + save("before", before); + save("after", after); + } else if (result.status === "added") { + save("after", after); + } else if (result.status === "removed") { + save("before", before); + } + const screen = { id, ...result, images }; + if (result.status === "changed" || result.status === "resized") screen.afterDigest = digestOf(after); + if (truncated) screen.truncated = true; + const error = beforeErrors[sceneName(name)] ?? beforeError; + if (result.status === "added" && error) screen.baseError = cut(error, maximumMessage); + return screen; + }); + return { screens, failed }; +} + +/** The scenes of a scenes.json report `--render-scenes` wrote, by name: `{images}` or `{error}`. */ +function scenesOf(report) { + if (!report || !existsSync(report)) return {}; + return JSON.parse(readFileSync(report, "utf8")).scenes ?? {}; +} + +/** Scene name to error, from a scenes.json report. */ +function errors(report) { + return Object.fromEntries(Object.entries(scenesOf(report)).filter(([, entry]) => typeof entry?.error === "string" && entry.error) + .map(([name, entry]) => [name, entry.error])); +} + +/** The images a scenes.json report lists that aren't in `folder`, as failures. */ +function undrawn(report, folder, prefix) { + return Object.entries(scenesOf(report)).flatMap(([name, entry]) => (Array.isArray(entry?.images) ? entry.images : []) + .filter((image) => typeof image !== "string" || !existsSync(join(folder, basename(image)))) + .map((image) => ({ id: cut(`${prefix}/${name}`, 200), error: cut(`scenes.json lists ${image}, but it wasn't drawn.`, maximumMessage) }))); +} + +export function main(argv) { + const [beforeDir, afterDir, outDir, ...options] = argv; + const option = (name) => { + const index = options.indexOf(name); + return index >= 0 ? options[index + 1] : undefined; + }; + const prefix = option("--prefix"); + if (!beforeDir || !afterDir || !outDir || !prefix) { + throw new Error("usage: compare-screens.mjs BEFORE AFTER OUT --prefix NAME [--before-report FILE] [--after-report FILE]"); + } + const beforeReport = option("--before-report"); + const beforeError = beforeReport && !existsSync(beforeReport) ? "The base build wrote no scenes.json, so it may not draw scenes yet." : undefined; + const { screens, failed: unnamed } = compareDirectories(beforeDir, afterDir, outDir, { prefix, beforeErrors: errors(beforeReport), beforeError }); + const afterReport = option("--after-report"); + const failed = [...unnamed, ...Object.entries(errors(afterReport)) + .map(([name, error]) => ({ id: cut(`${prefix}/${name}`, 200), error: cut(error, maximumMessage) })), ...undrawn(afterReport, afterDir, prefix)]; + const manifestPath = join(outDir, "manifest.json"); + const manifest = existsSync(manifestPath) ? JSON.parse(readFileSync(manifestPath, "utf8")) : { screens: [], failed: [] }; + manifest.screens.push(...screens); + manifest.failed.push(...failed); + writeFileSync(manifestPath, JSON.stringify(manifest, null, 2)); + const changed = screens.filter((screen) => screen.status !== "unchanged").length; + console.log(`${prefix}: ${screens.length} screens, ${changed} changed, ${failed.length} failed`); +} + +// Node names a module by its real path, so the path it was called by is resolved first. +if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) main(process.argv.slice(2)); diff --git a/.github/scripts/compare-screens.test.mjs b/.github/scripts/compare-screens.test.mjs new file mode 100644 index 0000000..a6a5525 --- /dev/null +++ b/.github/scripts/compare-screens.test.mjs @@ -0,0 +1,180 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, readdirSync, writeFileSync, mkdirSync, symlinkSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { comparePair, compareDirectories, main } from "./compare-screens.mjs"; +import { decodePNG, encodePNG } from "./png.mjs"; + +function image(width, height, rgba = [40, 40, 40, 255]) { + const pixels = new Uint8Array(width * height * 4); + for (let index = 0; index < width * height; index++) pixels.set(rgba, index * 4); + return { width, height, rgba: pixels }; +} + +function paint(target, x, y, width, height, rgba) { + const copy = { ...target, rgba: Uint8Array.from(target.rgba) }; + for (let row = y; row < y + height; row++) { + for (let column = x; column < x + width; column++) copy.rgba.set(rgba, (row * copy.width + column) * 4); + } + return copy; +} + +test("comparePair finds the changed region and ignores noise", () => { + const base = image(20, 10); + assert.deepEqual(comparePair(base, base), { status: "unchanged", changedPixels: 0, region: null }); + assert.deepEqual(comparePair(base, paint(base, 5, 2, 3, 3, [200, 40, 40, 255])), + { status: "changed", changedPixels: 9, region: { x: 5, y: 2, width: 3, height: 3 } }); + assert.equal(comparePair(base, paint(base, 12, 6, 1, 1, [200, 40, 40, 255])).status, "unchanged"); + assert.equal(comparePair(base, paint(base, 0, 0, 20, 10, [41, 41, 41, 255])).status, "unchanged"); + assert.deepEqual(comparePair(base, image(21, 10)), { status: "resized", changedPixels: 0, region: null }); + assert.equal(comparePair(null, base).status, "added"); + assert.equal(comparePair(base, null).status, "removed"); +}); + +test("comparePair compares screens of new heights from the top, to the first change", () => { + const base = image(20, 10); + // Rows added at the bottom change from where the shorter image ends. + assert.deepEqual(comparePair(base, image(20, 14)), { status: "resized", changedPixels: 80, region: { x: 0, y: 10, width: 20, height: 4 } }); + assert.deepEqual(comparePair(base, image(20, 8)), { status: "resized", changedPixels: 40, region: { x: 0, y: 8, width: 20, height: 2 } }); + // A change in the rows both have starts the region there, across the full width, down to the longer image's end. + const taller = paint(image(20, 14), 5, 2, 3, 3, [200, 40, 40, 255]); + assert.deepEqual(comparePair(base, taller), { status: "resized", changedPixels: 89, region: { x: 0, y: 2, width: 20, height: 12 } }); +}); + +test("compareDirectories writes images for changed screens only, and notes what the base couldn't draw", () => { + const root = mkdtempSync(join(tmpdir(), "screens-")); + const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); + for (const folder of [before, after, out]) mkdirSync(folder, { recursive: true }); + const base = image(60, 30); + writeFileSync(join(before, "same-dark.png"), encodePNG(base)); + writeFileSync(join(after, "same-dark.png"), encodePNG(base)); + writeFileSync(join(before, "moved-dark.png"), encodePNG(base)); + writeFileSync(join(after, "moved-dark.png"), encodePNG(paint(base, 10, 10, 4, 4, [255, 255, 255, 255]))); + writeFileSync(join(after, "new-dark.png"), encodePNG(base)); + writeFileSync(join(after, "odd name-dark.png"), encodePNG(base)); + const { screens, failed } = compareDirectories(before, after, out, + { prefix: "scenes", beforeErrors: { new: `draw: Unknown value ${"y".repeat(1200)}` } }); + assert.deepEqual(screens.map((screen) => [screen.id, screen.status]), + [["scenes/moved-dark", "changed"], ["scenes/new-dark", "added"], ["scenes/same-dark", "unchanged"]]); + const moved = screens[0]; + assert.deepEqual(Object.keys(moved.images).sort(), ["after", "before", "diff"]); + assert.deepEqual(readdirSync(out).sort(), + ["scenes~moved-dark-after.png", "scenes~moved-dark-before.png", "scenes~moved-dark-diff.png", "scenes~new-dark-after.png"]); + // An image smaller than the smallest crop is shown whole. + const crop = decodePNG(readFileSync(join(out, moved.images.after))); + assert.deepEqual([crop.width, crop.height], [60, 30]); + // The trusted half takes messages of at most 1,000 characters. + assert.match(screens[1].baseError, /^draw: Unknown value y+…$/); + assert.equal(screens[1].baseError.length, 1000); + assert.equal(screens[2].images.after, undefined); + assert.deepEqual(failed, [{ id: "scenes/odd name-dark", error: "Its name may use only letters, digits, dots, dashes and underscores." }]); +}); + +test("main notes every new scene when the base wrote no scenes report", () => { + const root = mkdtempSync(join(tmpdir(), "screens-")); + const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); + for (const folder of [before, after]) mkdirSync(folder, { recursive: true }); + writeFileSync(join(after, "bar-dark.png"), encodePNG(image(8, 8))); + writeFileSync(join(after, "scenes.json"), JSON.stringify({ scenes: { bar: { images: ["bar-dark.png"] }, broken: { error: "draw: Unknown value" } } })); + main([before, after, out, "--prefix", "scenes", "--before-report", join(before, "scenes.json"), "--after-report", join(after, "scenes.json")]); + const manifest = JSON.parse(readFileSync(join(out, "manifest.json"), "utf8")); + assert.deepEqual(manifest.screens.map((screen) => [screen.id, screen.status, screen.baseError]), + [["scenes/bar-dark", "added", "The base build wrote no scenes.json, so it may not draw scenes yet."]]); + assert.deepEqual(manifest.failed, [{ id: "scenes/broken", error: "draw: Unknown value" }]); +}); + +test("the command lines run when called through a symbolic link", () => { + // /tmp on macOS is a link to /private/tmp, and Node names a module by its real path. + const link = join(mkdtempSync(join(tmpdir(), "scripts-")), "scripts"); + symlinkSync(dirname(fileURLToPath(import.meta.url)), link); + for (const [script, status] of [["compare-screens.mjs", 1], ["capture-site.mjs", 64]]) { + const run = spawnSync(process.execPath, [join(link, script)], { encoding: "utf8" }); + assert.equal(run.status, status, script); + assert.match(run.stderr, /usage: /, script); + } +}); + +test("crops give a change context: 48 pixels around it, at least 480 by 160, inside the image", () => { + const root = mkdtempSync(join(tmpdir(), "screens-")); + const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); + for (const folder of [before, after]) mkdirSync(folder, { recursive: true }); + const base = image(600, 300); + writeFileSync(join(before, "middle-dark.png"), encodePNG(base)); + writeFileSync(join(after, "middle-dark.png"), encodePNG(paint(base, 300, 150, 4, 4, [255, 255, 255, 255]))); + writeFileSync(join(before, "edge-dark.png"), encodePNG(base)); + writeFileSync(join(after, "edge-dark.png"), encodePNG(paint(base, 2, 150, 4, 4, [255, 255, 255, 255]))); + const { screens } = compareDirectories(before, after, out, { prefix: "scenes" }); + const size = (file) => { const { width, height } = decodePNG(readFileSync(join(out, file))); return [width, height]; }; + for (const screen of screens) { + for (const kind of ["before", "after", "diff"]) assert.deepEqual(size(screen.images[kind]), [480, 160], `${screen.id} ${kind}`); + } + // The edge's crop moves inside the image rather than shrinking: its change is at its left. + const edge = decodePNG(readFileSync(join(out, screens[0].images.after))); + assert.deepEqual([...edge.rgba.subarray((80 * 480 + 2) * 4, (80 * 480 + 2) * 4 + 4)], [255, 255, 255, 255]); +}); + +test("a screen of a new height is cropped from its first change, full width", () => { + const root = mkdtempSync(join(tmpdir(), "screens-")); + const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); + for (const folder of [before, after]) mkdirSync(folder, { recursive: true }); + writeFileSync(join(before, "page-dark.png"), encodePNG(image(100, 50))); + writeFileSync(join(after, "page-dark.png"), encodePNG(image(100, 80))); + writeFileSync(join(before, "tall-dark.png"), encodePNG(image(10, 2000))); + writeFileSync(join(after, "tall-dark.png"), encodePNG(paint(image(10, 2100), 0, 100, 10, 2, [255, 255, 255, 255]))); + writeFileSync(join(before, "wide-dark.png"), encodePNG(image(100, 50))); + writeFileSync(join(after, "wide-dark.png"), encodePNG(image(120, 50))); + const { screens } = compareDirectories(before, after, out, { prefix: "site" }); + const size = (file) => { const { width, height } = decodePNG(readFileSync(join(out, file))); return [width, height]; }; + const [page, tall, wide] = screens; + // Rows added from y = 50: the crops start 48 rows above, and run to each image's end. + assert.deepEqual([page.status, size(page.images.before), size(page.images.after), size(page.images.diff)], + ["resized", [100, 48], [100, 78], [100, 78]]); + const diff = decodePNG(readFileSync(join(out, page.images.diff))); + assert.deepEqual([...diff.rgba.subarray(0, 4)], [12, 12, 12, 255]); + assert.deepEqual([...diff.rgba.subarray((60 * 100) * 4, (60 * 100) * 4 + 4)], [255, 59, 48, 255]); + // A long change shows its first 1,600 rows, and says so. + assert.deepEqual([size(tall.images.before), size(tall.images.after), tall.truncated], [[10, 1600], [10, 1600], true]); + assert.equal(page.truncated, undefined); + // A new width can't be lined up: both images are shown whole. + assert.deepEqual([wide.region, Object.keys(wide.images).sort(), size(wide.images.after)], [null, ["after", "before"], [120, 50]]); +}); + +test("changed and resized screens carry a digest of their whole after image", () => { + const root = mkdtempSync(join(tmpdir(), "screens-")); + const before = join(root, "before"); + mkdirSync(before); + const page = image(10, 2000); + writeFileSync(join(before, "page-dark.png"), encodePNG(page)); + // Both changes span rows 100 to 1901, and the crop shows rows 52 to 1651. + const tall = paint(paint(page, 0, 100, 10, 2, [255, 255, 255, 255]), 0, 1900, 10, 2, [255, 255, 255, 255]); + const runs = [tall, paint(tall, 0, 1800, 10, 2, [255, 255, 255, 255])].map((after) => { + const folder = join(root, `after-${Math.random()}`), out = join(root, `out-${Math.random()}`); + mkdirSync(folder); + writeFileSync(join(folder, "page-dark.png"), encodePNG(after)); + const [screen] = compareDirectories(before, folder, out, { prefix: "site" }).screens; + return { screen, crop: readFileSync(join(out, screen.images.after)) }; + }); + // A change inside the region but past the crop leaves the crop as it was, but not the digest. + assert.deepEqual(runs[0].crop, runs[1].crop); + assert.equal(runs[0].screen.truncated, true); + assert.match(runs[0].screen.afterDigest, /^[0-9a-f]{64}$/); + assert.notEqual(runs[0].screen.afterDigest, runs[1].screen.afterDigest); +}); + +test("main reports images a scenes report lists that weren't drawn", () => { + const root = mkdtempSync(join(tmpdir(), "screens-")); + const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); + for (const folder of [before, after]) mkdirSync(folder, { recursive: true }); + writeFileSync(join(before, "bar-dark.png"), encodePNG(image(8, 8))); + writeFileSync(join(after, "bar-dark.png"), encodePNG(image(8, 8))); + const report = { scenes: { bar: { images: ["bar-dark.png", "bar-light.png"] } } }; + writeFileSync(join(before, "scenes.json"), JSON.stringify(report)); + writeFileSync(join(after, "scenes.json"), JSON.stringify(report)); + main([before, after, out, "--prefix", "scenes", "--before-report", join(before, "scenes.json"), "--after-report", join(after, "scenes.json")]); + const manifest = JSON.parse(readFileSync(join(out, "manifest.json"), "utf8")); + assert.deepEqual(manifest.failed, [{ id: "scenes/bar", error: "scenes.json lists bar-light.png, but it wasn't drawn." }]); +}); diff --git a/.github/scripts/png.mjs b/.github/scripts/png.mjs new file mode 100644 index 0000000..9f39849 --- /dev/null +++ b/.github/scripts/png.mjs @@ -0,0 +1,102 @@ +// Reads and writes the PNG files the before/after screens compare: 8-bit RGB or RGBA without +// interlacing, which is what StableBar's renders and headless Chrome write. No dependencies. +import zlib from "node:zlib"; + +const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); +/** The largest image decodePNG takes, before inflating anything: a long website page is about 2880 by 12000. */ +const limits = { maxWidth: 20000, maxHeight: 100000, maxPixels: 80_000_000 }; + +function paeth(a, b, c) { + const p = a + b - c, pa = Math.abs(p - a), pb = Math.abs(p - b), pc = Math.abs(p - c); + return pa <= pb && pa <= pc ? a : pb <= pc ? b : c; +} + +function chunk(type, data) { + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const body = Buffer.concat([Buffer.from(type, "latin1"), data]); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(zlib.crc32(body)); + return Buffer.concat([length, body, crc]); +} + +/** + * The pixels of a PNG as `{width, height, rgba}`, every pixel four bytes. It throws for an image larger + * than its limits, a chunk cut short, or image data that doesn't inflate to exactly the image's size. + */ +export function decodePNG(input, { maxWidth = limits.maxWidth, maxHeight = limits.maxHeight, maxPixels = limits.maxPixels } = {}) { + const buffer = Buffer.isBuffer(input) ? input : Buffer.from(input); + if (buffer.length < 8 || !buffer.subarray(0, 8).equals(signature)) throw new Error("Unsupported PNG: no PNG signature."); + let offset = 8, width = 0, height = 0, bitDepth = 0, colorType = -1, interlace = -1; + const data = []; + while (offset + 8 <= buffer.length) { + const length = buffer.readUInt32BE(offset); + if (offset + 12 + length > buffer.length) throw new Error("Unsupported PNG: a chunk is cut short."); + const type = buffer.toString("latin1", offset + 4, offset + 8); + const body = buffer.subarray(offset + 8, offset + 8 + length); + if (type === "IHDR") { + width = body.readUInt32BE(0); + height = body.readUInt32BE(4); + bitDepth = body[8]; + colorType = body[9]; + interlace = body[12]; + } else if (type === "IDAT") { + data.push(body); + } else if (type === "IEND") { + break; + } + offset += 12 + length; + } + if (!width || !height || bitDepth !== 8 || (colorType !== 2 && colorType !== 6) || interlace !== 0) { + throw new Error("Unsupported PNG: only 8-bit RGB or RGBA without interlacing."); + } + if (width > maxWidth || height > maxHeight || width * height > maxPixels) { + throw new Error(`Unsupported PNG: ${width}×${height} is larger than it takes.`); + } + const channels = colorType === 6 ? 4 : 3; + const stride = width * channels; + let raw; + try { + raw = zlib.inflateSync(Buffer.concat(data), { maxOutputLength: height * (stride + 1) }); + } catch (error) { + throw new Error(`Unsupported PNG: its image data doesn't inflate to its size (${error.code ?? error.message}).`); + } + if (raw.length < height * (stride + 1)) throw new Error("Unsupported PNG: the image data is cut short."); + const pixels = new Uint8Array(height * stride); + for (let y = 0; y < height; y++) { + const filter = raw[y * (stride + 1)]; + if (filter > 4) throw new Error(`Unsupported PNG: unknown filter ${filter}.`); + const line = y * (stride + 1) + 1; + for (let x = 0; x < stride; x++) { + const a = x >= channels ? pixels[y * stride + x - channels] : 0; + const b = y > 0 ? pixels[(y - 1) * stride + x] : 0; + const c = x >= channels && y > 0 ? pixels[(y - 1) * stride + x - channels] : 0; + const predicted = filter === 0 ? 0 : filter === 1 ? a : filter === 2 ? b : filter === 3 ? (a + b) >> 1 : paeth(a, b, c); + pixels[y * stride + x] = (raw[line + x] + predicted) & 255; + } + } + if (channels === 4) return { width, height, rgba: pixels }; + const rgba = new Uint8Array(width * height * 4); + for (let index = 0; index < width * height; index++) { + rgba[index * 4] = pixels[index * 3]; + rgba[index * 4 + 1] = pixels[index * 3 + 1]; + rgba[index * 4 + 2] = pixels[index * 3 + 2]; + rgba[index * 4 + 3] = 255; + } + return { width, height, rgba }; +} + +/** An RGBA PNG of `{width, height, rgba}`. */ +export function encodePNG({ width, height, rgba }) { + const stride = width * 4; + const raw = Buffer.alloc(height * (stride + 1)); + for (let y = 0; y < height; y++) { + Buffer.from(rgba.buffer, rgba.byteOffset + y * stride, stride).copy(raw, y * (stride + 1) + 1); + } + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; + header[9] = 6; + return Buffer.concat([signature, chunk("IHDR", header), chunk("IDAT", zlib.deflateSync(raw)), chunk("IEND", Buffer.alloc(0))]); +} diff --git a/.github/scripts/png.test.mjs b/.github/scripts/png.test.mjs new file mode 100644 index 0000000..2e18905 --- /dev/null +++ b/.github/scripts/png.test.mjs @@ -0,0 +1,85 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import zlib from "node:zlib"; +import { decodePNG, encodePNG } from "./png.mjs"; + +function chunk(type, data) { + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const body = Buffer.concat([Buffer.from(type, "latin1"), data]); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(zlib.crc32(body)); + return Buffer.concat([length, body, crc]); +} + +function paeth(a, b, c) { + const p = a + b - c, pa = Math.abs(p - a), pb = Math.abs(p - b), pc = Math.abs(p - c); + return pa <= pb && pa <= pc ? a : pb <= pc ? b : c; +} + +/** An 8-bit RGB PNG whose every scanline uses `filter`. */ +function rgbPNG(width, height, rgb, filter) { + const stride = width * 3; + const raw = []; + for (let y = 0; y < height; y++) { + raw.push(filter); + for (let x = 0; x < stride; x++) { + const a = x >= 3 ? rgb[y * stride + x - 3] : 0; + const b = y > 0 ? rgb[(y - 1) * stride + x] : 0; + const c = x >= 3 && y > 0 ? rgb[(y - 1) * stride + x - 3] : 0; + const predicted = [0, a, b, (a + b) >> 1, paeth(a, b, c)][filter]; + raw.push((rgb[y * stride + x] - predicted) & 255); + } + } + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; + header[9] = 2; + return Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), chunk("IHDR", header), + chunk("IDAT", zlib.deflateSync(Buffer.from(raw))), chunk("IEND", Buffer.alloc(0))]); +} + +test("encodePNG and decodePNG round-trip RGBA pixels", () => { + const rgba = Uint8Array.from([255, 0, 0, 255, 0, 255, 0, 128, 0, 0, 255, 0, 9, 8, 7, 6, 1, 2, 3, 4, 250, 251, 252, 253]); + const decoded = decodePNG(encodePNG({ width: 3, height: 2, rgba })); + assert.equal(decoded.width, 3); + assert.equal(decoded.height, 2); + assert.deepEqual([...decoded.rgba], [...rgba]); +}); + +test("decodePNG reads RGB scanlines with every filter as opaque RGBA", () => { + const width = 4, height = 3; + const rgb = Array.from({ length: width * height * 3 }, (_, index) => (index * 37 + 11) % 256); + const expected = []; + for (let pixel = 0; pixel < width * height; pixel++) expected.push(...rgb.slice(pixel * 3, pixel * 3 + 3), 255); + for (const filter of [0, 1, 2, 3, 4]) { + const decoded = decodePNG(rgbPNG(width, height, rgb, filter)); + assert.deepEqual([...decoded.rgba], expected, `filter ${filter}`); + } +}); + +test("decodePNG rejects what isn't an 8-bit RGB or RGBA PNG", () => { + assert.throws(() => decodePNG(Buffer.from("not a png")), /Unsupported PNG/); +}); + +test("decodePNG refuses images larger than its limits, cut short, or inflating past their size", () => { + const header = (width, height) => { + const data = Buffer.alloc(13); + data.writeUInt32BE(width, 0); + data.writeUInt32BE(height, 4); + data[8] = 8; + data[9] = 6; + return chunk("IHDR", data); + }; + const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); + const png = (width, height, raw) => Buffer.concat([signature, header(width, height), chunk("IDAT", zlib.deflateSync(raw)), chunk("IEND", Buffer.alloc(0))]); + // Its header alone says too much: nothing is inflated. + assert.throws(() => decodePNG(png(100000, 100000, Buffer.alloc(10))), /larger than/); + assert.throws(() => decodePNG(png(4, 4, Buffer.alloc(4 * 17)), { maxPixels: 15 }), /larger than/); + // A chunk that claims more bytes than the file has. + const whole = encodePNG({ width: 2, height: 2, rgba: new Uint8Array(16) }); + assert.throws(() => decodePNG(whole.subarray(0, whole.length - 20)), /cut short/); + // Image data that inflates to far more than a 2 by 2 image holds. + assert.throws(() => decodePNG(png(2, 2, Buffer.alloc(50 * 1024 * 1024))), /Unsupported PNG/); +}); diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs new file mode 100644 index 0000000..80d652a --- /dev/null +++ b/.github/scripts/screens-bot.mjs @@ -0,0 +1,428 @@ +// The GitHub side of the before/after screens, for screens.yml, which runs the default branch's copy. +// Every event, a CI run finishing or a pull request labelled, retargeted, closed or reopened, runs one +// reconcile of its pull request, one at a time: plan() reads where the pull request stands and which CI +// run's screens to publish, the workflow downloads that run's artifacts, and apply() makes the comment, +// the images and the Visual review status match. GitHub may drop an event waiting behind another; the +// next one still reconciles everything, since nothing depends on which events ran. visual-ok approves +// the screens the comment shows when it was added after they appeared. The artifacts come from the +// pull request's own CI jobs, so they are read as untrusted data. +import { closeSync, existsSync, lstatSync, openSync, readFileSync, readSync, readdirSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { join } from "node:path"; +import { validateArtifact, readManifest, fingerprint, summary, reviewState, commentBody, withReview, noScreensBody, readData, + findScreensComment, assetName, releaseTag, staleAssets, pullRequestAssets } from "./screens-comment.mjs"; +import { decodePNG, encodePNG } from "./png.mjs"; + +const statusContext = "Visual review"; +const approval = "visual-ok"; +const workflowPath = ".github/workflows/ci.yml"; +/** The CI jobs the screens depend on, by the names they show. */ +const jobNames = { current: "Check revision", build: "Build release", app: "Draw screens", site: "Draw website screens" }; +/** Each kind of screens: its artifact, the ids it may hold, the screens it always has, and the jobs it needs. */ +const kinds = { + app: { artifact: "screens-app", prefixes: ["docs", "scenes"], always: { prefix: "docs", name: "docs images" }, needs: ["current", "build"] }, + site: { artifact: "screens-site", prefixes: ["site"], always: { prefix: "site", name: "website screens" }, needs: ["current"] }, +}; +const broken = ["failure", "cancelled", "timed_out", "missing"]; +/** + * The files whose changes call for each kind of screens: the patterns CI's Check revision job greps + * for its screens and site outputs. A test keeps them the same. The bot reads the pull request's + * files from GitHub, so a run whose job graph skips or drops a screens job can't pass as having none. + */ +export const changePatterns = { + app: /^(Sources\/|Resources\/|Package\.(swift|resolved)$|scripts\/|scenes\/|examples\/|\.github\/workflows\/ci\.yml$|\.github\/scripts\/(png|compare-screens)\.mjs$)/, + site: /^(site\/|\.github\/workflows\/ci\.yml$|\.github\/scripts\/(png|compare-screens|capture-site)\.mjs$)/, +}; + +/** The open pull request a pull_request run of CI is for: listed on the run, or for a fork, found by its head branch. */ +async function pullRequestOfRun({ github, owner, repo, run }) { + if (!run || run.event !== "pull_request" || run.path !== workflowPath || !run.head_repository) return null; + if (run.pull_requests.length > 0) return run.pull_requests[0].number; + const pulls = await github.paginate(github.rest.pulls.list, + { owner, repo, state: "open", head: `${run.head_repository.owner.login}:${run.head_branch}`, per_page: 100 }); + return (pulls.find((pr) => pr.head.sha === run.head_sha) ?? pulls[0])?.number ?? null; +} + +/** + * The pull request an event is about: in a label, edit or close event's payload, or for a CI run, the + * open one it was for. The workflow also asks for it when plan() fails, to mark that head errored. + */ +export async function pullRequestNumber({ github, context }) { + const { owner, repo } = context.repo; + return context.payload.pull_request?.number ?? await pullRequestOfRun({ github, owner, repo, run: context.payload.workflow_run }); +} + +/** + * The newest finished CI run for the pull request's head, from its own head repository and branch: a + * pull request from another fork at the same commit has runs of its own, and they don't count. + */ +async function latestRun({ github, owner, repo, pr }) { + const runs = (await github.paginate(github.rest.actions.listWorkflowRuns, + { owner, repo, workflow_id: "ci.yml", head_sha: pr.head.sha, event: "pull_request", status: "completed", per_page: 100 })) + .filter((run) => run.path === workflowPath && run.head_branch === pr.head.ref + && pr.head.repo?.full_name !== undefined && run.head_repository?.full_name === pr.head.repo.full_name); + return runs.sort((a, b) => b.run_number - a.run_number)[0] ?? null; +} + +/** + * What a reconcile of the event's pull request does, decided before the workflow downloads anything: + * "none" for a run that isn't CI's for an open pull request, "close" for a closed pull request, + * "ambiguous" when another pull request from its branch is at its commit, "publish" with the newest + * finished CI run for its head when the comment doesn't show that run yet, and "review" otherwise. A publication carries what the run's jobs did, which artifacts it uploaded + * and which screens the pull request's files call for. A job the run doesn't have is "missing". + */ +export async function plan({ github, context }) { + const { owner, repo } = context.repo; + const number = await pullRequestNumber({ github, context }); + if (!number) return { action: "none" }; + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: number }); + if (pr.state !== "open") return { action: "close", number }; + // Another pull request from this branch at this commit, open or closed, has CI runs with the same head, branch and + // commit, and GitHub lists every pull request of a branch on each run, so their runs can't be told apart. + const siblings = pr.head.repo ? await github.paginate(github.rest.pulls.list, + { owner, repo, state: "all", head: `${pr.head.repo.owner.login}:${pr.head.ref}`, per_page: 100 }) : []; + if (siblings.some((other) => other.number !== number && other.head.sha === pr.head.sha)) return { action: "ambiguous", number }; + const run = await latestRun({ github, owner, repo, pr }); + const existing = findScreensComment(await comments(github, owner, repo, number)); + const data = existing ? readData(existing.body) : null; + if (!run || (data?.runId === run.id && data?.runAttempt === run.run_attempt)) return { action: "review", number }; + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { owner, repo, run_id: run.id, filter: "latest", per_page: 100 }); + const uploaded = (await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { owner, repo, run_id: run.id, per_page: 100 })) + .map((artifact) => artifact.name); + const files = (await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number: number, per_page: 100 })) + .flatMap((file) => [file.filename, file.previous_filename].filter(Boolean)); + return { + action: "publish", number, + run: { + id: run.id, attempt: run.run_attempt, head: pr.head.sha, createdAt: run.created_at, + jobs: Object.fromEntries(Object.entries(jobNames).map(([key, name]) => [key, jobs.find((job) => job.name === name)?.conclusion ?? "missing"])), + artifacts: Object.fromEntries(Object.entries(kinds).map(([kind, { artifact }]) => [kind, uploaded.includes(artifact)])), + expected: Object.fromEntries(Object.keys(kinds).map((kind) => [kind, files.some((file) => changePatterns[kind].test(file))])), + }, + }; +} + +/** A digest of an image's pixels, the same for any encoding of them. */ +const pixelDigest = ({ width, height, rgba }) => createHash("sha256").update(`${width}x${height}\n`).update(rgba).digest("hex"); + +/** + * The screens, failures and images of a downloaded artifact folder, checked as untrusted data. Every + * image its screens name is decoded within limits and encoded again, so only the bot's own PNGs are + * uploaded: `files` maps each name to `{data, digest}`, the digest being of the pixels. + */ +export function readArtifact(folder, prefixes, always) { + if (!existsSync(folder)) return { ok: false, problems: ["It couldn't be downloaded."] }; + const files = readdirSync(folder).map((name) => { + const path = join(folder, name); + const stat = lstatSync(path); + const head = Buffer.alloc(8); + if (stat.isFile()) { + const descriptor = openSync(path, "r"); + try { + readSync(descriptor, head, 0, 8, 0); + } finally { + closeSync(descriptor); + } + } + return { name, path, size: stat.size, head, regular: stat.isFile() }; + }); + const checked = validateArtifact(files); + if (!checked.ok) return checked; + const manifest = files.find((file) => file.name === "manifest.json"); + if (!manifest) return { ok: false, problems: ["It has no manifest.json."] }; + const read = readManifest(readFileSync(manifest.path, "utf8"), files.map((file) => file.name), prefixes); + if (!read.ok) return read; + // A job that drew nothing, or not what it always draws, failed even if it says it didn't. + if (read.screens.length === 0 && read.failed.length === 0) return { ok: false, problems: ["It holds no screens."] }; + if (always && !read.screens.some((screen) => screen.id.startsWith(`${always.prefix}/`))) return { ok: false, problems: [`It holds no ${always.name}.`] }; + const images = new Map(); + for (const name of new Set(read.screens.flatMap((screen) => Object.values(screen.images)))) { + let image; + try { + image = decodePNG(readFileSync(join(folder, name))); + } catch (error) { + return { ok: false, problems: [`\`${name}\` isn't a PNG the bot can read: ${error.message}`] }; + } + images.set(name, { data: encodePNG(image), digest: pixelDigest(image) }); + } + return { ...read, files: images }; +} + +const comments = (github, owner, repo, number) => + github.paginate(github.rest.issues.listComments, { owner, repo, issue_number: number, per_page: 100 }); + +/** When visual-ok was last added to the pull request, and when its base last changed, as GitHub's ISO times, or null. */ +async function history({ github, owner, repo, number }) { + const events = await github.paginate(github.rest.issues.listEvents, { owner, repo, issue_number: number, per_page: 100 }); + const last = (matches) => events.filter(matches).map((event) => event.created_at).sort().at(-1) ?? null; + return { + labeledAt: last((event) => event.event === "labeled" && event.label?.name === approval), + baseChangedAt: last((event) => event.event === "base_ref_changed"), + }; +} + +/** Whether the pull request's base changed after `time`, an ISO time. */ +const changedSince = (baseChangedAt, time) => baseChangedAt !== null && time !== null && Date.parse(baseChangedAt) > Date.parse(time); + +async function removeApproval({ github, owner, repo, number }) { + try { + await github.rest.issues.removeLabel({ owner, repo, issue_number: number, name: approval }); + } catch (error) { + if (error.status !== 404) throw error; + } +} + +async function setStatus({ github, owner, repo, sha, review, url }) { + await github.rest.repos.createCommitStatus({ owner, repo, sha, state: review.state, context: statusContext, description: review.description, + ...(url ? { target_url: url } : {}) }); +} + +/** A release holds at most this many assets. */ +const capacity = 1000; + +/** + * The first of this month's pre-releases for the images with room for `needed` more, `pr-previews-YYYY-MM`, + * then `-2`, `-3` and on, created when missing, never as the latest release. + */ +async function previewRelease({ github, owner, repo, target, needed }) { + const month = releaseTag(new Date()); + const find = async (tag) => { + try { + return (await github.rest.repos.getReleaseByTag({ owner, repo, tag })).data; + } catch (error) { + if (error.status === 404) return null; + throw error; + } + }; + for (let shard = 1; shard <= 50; shard++) { + const tag = shard === 1 ? month : `${month}-${shard}`; + let release = await find(tag); + if (!release) { + try { + release = (await github.rest.repos.createRelease({ owner, repo, tag_name: tag, target_commitish: target, name: `Pull request screens, ${tag.slice(12)}`, + prerelease: true, make_latest: "false", + body: "Images for the before and after screens the bot posts on pull requests. An unmerged pull request's images are deleted when it closes." })).data; + } catch (error) { + if (error.status !== 422) throw error; + release = await find(tag); + } + } + const assets = await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id: release.id, per_page: 100 }); + if (assets.length + needed <= capacity) return release; + } + throw new Error(`No pull request screens release of ${month} has room for ${needed} images.`); +} + +/** + * Uploads `images`, [file, PNG data] pairs, and returns each file's URL as GitHub answers it. The head's + * images from an earlier run of it are deleted first, wherever they are. + */ +async function upload({ github, owner, repo, number, head, target, images }) { + const urls = new Map(); + if (images.length === 0) return urls; + await deleteAssets({ github, owner, repo, pick: (assets) => pullRequestAssets(assets, number).filter((asset) => asset.name.startsWith(assetName(number, head, ""))) }); + const release = await previewRelease({ github, owner, repo, target, needed: images.length }); + for (const [file, data] of images) { + const { data: asset } = await github.rest.repos.uploadReleaseAsset({ owner, repo, release_id: release.id, name: assetName(number, head, file), data, + headers: { "content-type": "image/png", "content-length": data.length } }); + urls.set(file, asset.browser_download_url); + } + return urls; +} + +/** Deletes the images `pick` chooses from every pull request screens pre-release, and counts them. */ +async function deleteAssets({ github, owner, repo, pick }) { + const releases = (await github.paginate(github.rest.repos.listReleases, { owner, repo, per_page: 100 })) + .filter((release) => release.tag_name.startsWith("pr-previews-")); + let deleted = 0; + for (const release of releases) { + const assets = await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id: release.id, per_page: 100 }); + for (const asset of pick(assets)) { + await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id }); + deleted++; + } + } + return deleted; +} + +/** What the run's jobs and artifacts give: the screens, the failures, the image files and whether any screens were drawn. */ +function gather(run, folders) { + const screens = [], failed = [], files = new Map(); + let ranScreens = false; + for (const [kind, { artifact, prefixes, always, needs }] of Object.entries(kinds)) { + const conclusion = run.jobs[kind]; + if (conclusion === "missing") { + ranScreens = true; + failed.push({ id: kind, error: `${jobNames[kind]} is missing from this CI run.` }); + continue; + } + if (conclusion === "skipped") { + const blocker = needs.find((job) => broken.includes(run.jobs[job])); + if (blocker) { + ranScreens = true; + failed.push({ id: kind, error: `${jobNames[blocker]} ${run.jobs[blocker] === "missing" ? "is missing" : `ended with ${run.jobs[blocker]}`}, so ${jobNames[kind]} didn't run.` }); + } else if (run.expected[kind]) { + ranScreens = true; + failed.push({ id: kind, error: `${jobNames[kind]} was skipped, but this pull request changes what it draws.` }); + } + continue; + } + ranScreens = true; + if (conclusion !== "success") { + failed.push({ id: kind, error: `${jobNames[kind]} ended with ${conclusion}.` }); + continue; + } + const read = run.artifacts[kind] ? readArtifact(folders[kind] ?? "", prefixes, always) : { ok: false, problems: ["The job uploaded none."] }; + if (!read.ok) { + failed.push({ id: kind, error: `The ${artifact} artifact can't be used: ${read.problems.join(" ")}` }); + continue; + } + screens.push(...read.screens); + failed.push(...read.failed); + for (const [name, image] of read.files) files.set(name, image); + } + return { screens, failed, files, ranScreens }; +} + +/** + * Publishes a run from plan(): uploads the images of its changed screens, writes or rewrites the + * comment, sets the status on the head, takes visual-ok off when it came before these screens, and then + * deletes the images of the pull request's other heads. Screens the comment already showed keep the + * time they first appeared, and so their approval, unless the base changed since. New screens appear + * when the comment is written: that write's time becomes `shownAt`, so a label added during the uploads + * approves nothing. A run that started before the base last changed compared the old base: its review + * waits for a new run. + */ +async function publish({ github, owner, repo, pr, existing, data: previous, labels, labeledAt, baseChangedAt, run, folders, target }) { + const number = pr.number, head = pr.head.sha; + const { screens, failed, files, ranScreens } = gather(run, folders); + const drawn = { head, base: pr.base.sha, baseRef: pr.base.ref, runId: run.id, runAttempt: run.attempt, runCreatedAt: run.createdAt ?? null }; + const baseChanged = changedSince(baseChangedAt, drawn.runCreatedAt); + let review, body, data, shownLater = false; + if (!ranScreens) { + review = reviewState({ ranScreens: false }); + data = { ...drawn, ranScreens: false, fingerprint: null, shownAt: null, total: 0, changed: 0, failed: 0, failure: null }; + body = noScreensBody({ reason: `This pull request at ${head.slice(0, 7)} changes neither the app nor the website, so there are no screens to compare.`, + review, data }); + } else { + const shown = screens.filter((screen) => screen.status !== "unchanged").flatMap((screen) => Object.values(screen.images)); + const urls = await upload({ github, owner, repo, number, head, target, images: shown.map((file) => [file, files.get(file).data]) }); + const print = fingerprint({ screens, failed }, (file) => files.get(file).digest); + const carried = previous?.fingerprint === print && previous.shownAt !== null && !baseChanged && !changedSince(baseChangedAt, previous.shownAt); + const counts = summary(screens, failed); + data = { ...drawn, ranScreens: true, fingerprint: print, shownAt: carried ? previous.shownAt : null, ...counts }; + shownLater = !carried; + review = reviewState({ ...data, labels, labeledAt, baseChanged }); + if (review.removeLabel) await removeApproval({ github, owner, repo, number }); + body = commentBody({ baseRef: pr.base.ref, base: pr.base.sha, head, screens, failed, imageURL: (file) => urls.get(file), review, data }); + } + let comment = existing; + if (existing) { + comment = (await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body })).data; + } else if (ranScreens) { + comment = (await github.rest.issues.createComment({ owner, repo, issue_number: number, body })).data; + } + if (shownLater && comment) { + comment = (await github.rest.issues.updateComment({ owner, repo, comment_id: comment.id, + body: withReview(body, review, { ...data, shownAt: comment.updated_at }) })).data; + } + await setStatus({ github, owner, repo, sha: head, review, url: comment?.html_url }); + await deleteAssets({ github, owner, repo, pick: (assets) => staleAssets(assets, number, head) }); + return review; +} + +/** + * Brings the review up to date without drawing anything: the labels, when visual-ok was added, and + * whether the pull request still has the base its screens were compared with. Sets the status when + * the comment is for the current head; a newer head's run sets its own. + */ +async function review({ github, owner, repo, pr, existing, data: stored, labels, labeledAt, baseChangedAt }) { + if (!stored) return null; + // A publication cut short before it recorded when its screens appeared: the comment's last write is no earlier. + const data = stored.ranScreens && stored.shownAt === null ? { ...stored, shownAt: existing.updated_at } : stored; + const baseChanged = changedSince(baseChangedAt, data.runCreatedAt); + const result = reviewState({ ...data, labels, labeledAt, baseChanged }); + if (result.removeLabel) await removeApproval({ github, owner, repo, number: pr.number }); + const body = withReview(existing.body, result, data); + if (body !== existing.body) await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + if (data.head === pr.head.sha) await setStatus({ github, owner, repo, sha: data.head, review: result, url: existing.html_url }); + return result; +} + +const sharedCommit = "Another pull request is at this branch's commit: close it if it's open, then push a commit."; + +/** Fails the review of a pull request that another pull request from its branch shares a commit with, and says why. */ +async function ambiguous({ github, owner, repo, pr, existing }) { + const data = { head: pr.head.sha, base: pr.base.sha, baseRef: pr.base.ref, runId: null, runAttempt: null, runCreatedAt: null, + ranScreens: true, fingerprint: null, shownAt: null, total: 0, changed: 0, failed: 1, failure: sharedCommit }; + const result = reviewState(data); + const body = noScreensBody({ review: result, data, reason: "Another pull request from this branch is at this commit, open or closed, so the bot " + + "can't tell their CI runs apart and compares no screens. Close it if it's open, then push a commit." }); + const comment = existing + ? (await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body })).data + : (await github.rest.issues.createComment({ owner, repo, issue_number: pr.number, body })).data; + await setStatus({ github, owner, repo, sha: pr.head.sha, review: result, url: comment.html_url }); + return result; +} + +/** + * Deletes an unmerged, closed pull request's images from every month, says so in its comment, and counts + * them. The comment forgets which run it showed, so a reopened pull request shows its screens again. + */ +async function closeUnmerged({ github, owner, repo, number }) { + const deleted = await deleteAssets({ github, owner, repo, pick: (assets) => pullRequestAssets(assets, number) }); + const existing = findScreensComment(await comments(github, owner, repo, number)); + const stored = existing ? readData(existing.body) : null; + if (stored) { + const data = { ...stored, runId: null, runAttempt: null }; + const review = { state: "success", description: "Nothing to review: the pull request closed.", removeLabel: false }; + const body = noScreensBody({ reason: "This pull request closed without merging, so the images of its screens were deleted.", review, data }); + if (body !== existing.body) await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + } + return deleted; +} + +/** + * Makes the pull request's comment, images and Visual review status match where it stands now, + * following plan(): `folders` hold the artifacts the workflow downloaded for a publication. A closed + * pull request loses its images unless it merged; a head that moved since the plan waits for its own + * run. Returns what it did: `{action, review}`, or `{action: "close", deleted}`. + */ +export async function apply({ github, context, plan: chosen, folders = {} }) { + if (chosen.action === "none") return { action: "none" }; + const { owner, repo } = context.repo; + const number = chosen.number; + const response = await github.rest.pulls.get({ owner, repo, pull_number: number }); + const pr = response.data; + if (pr.state !== "open") return { action: "close", deleted: pr.merged ? 0 : await closeUnmerged({ github, owner, repo, number }) }; + const existing = findScreensComment(await comments(github, owner, repo, number)); + const state = { + github, owner, repo, pr, existing, data: existing ? readData(existing.body) : null, + labels: pr.labels.map((label) => label.name), ...await history({ github, owner, repo, number }), + }; + if (chosen.action === "ambiguous") return { action: "ambiguous", review: await ambiguous(state) }; + if (chosen.action === "publish" && chosen.run.head === pr.head.sha) { + return { action: "publish", review: await publish({ ...state, run: chosen.run, folders, target: context.payload.repository?.default_branch }) }; + } + return { action: "review", review: await review(state) }; +} + +/** + * Marks an open pull request's head as errored when the bot itself fails, so a required Visual review doesn't + * wait forever. When GitHub can't say where the pull request stands, a label, edit or close event's own pull + * request does: a base change whose reconcile failed must not keep the old comparison's status. + */ +export async function failRun({ github, context, number }) { + if (!number) return; + const { owner, repo } = context.repo; + let pr; + try { + pr = (await github.rest.pulls.get({ owner, repo, pull_number: number })).data; + } catch (error) { + if (context.payload.pull_request?.number !== number) throw error; + pr = context.payload.pull_request; + } + if (pr.state !== "open") return; + await github.rest.repos.createCommitStatus({ owner, repo, sha: pr.head.sha, state: "error", context: statusContext, + description: "The screens bot failed: see the Screens workflow run.", target_url: `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}` }); +} diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs new file mode 100644 index 0000000..c195c36 --- /dev/null +++ b/.github/scripts/screens-bot.test.mjs @@ -0,0 +1,522 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import zlib from "node:zlib"; +import { decodePNG, encodePNG } from "./png.mjs"; +import { main as compare } from "./compare-screens.mjs"; +import { readData, releaseTag } from "./screens-comment.mjs"; +import { apply, changePatterns, failRun, plan, pullRequestNumber } from "./screens-bot.mjs"; + +const head = "a".repeat(40), oldHead = "c".repeat(40), base = "b".repeat(40); +const repo = { owner: "Seb-GRAF", repo: "StableBar" }; +const bot = { login: "github-actions[bot]", type: "Bot" }; +const notFound = () => Object.assign(new Error("Not Found"), { status: 404 }); +const conclusions = { "Check revision": "success", "Build release": "success", "Draw screens": "success", "Draw website screens": "skipped" }; +const jobsOf = (entries) => Object.entries(entries).map(([name, conclusion]) => ({ name, conclusion })); + +function image(color) { + const rgba = new Uint8Array(40 * 20 * 4); + for (let index = 0; index < 40 * 20; index++) rgba.set([40, 40, 40, 255], index * 4); + for (let y = 5; y < 10; y++) for (let x = 10; x < 15; x++) rgba.set(color, (y * 40 + x) * 4); + return { width: 40, height: 20, rgba }; +} + +/** An artifact folder as the Draw screens job uploads it: docs/hero-dark drawn in `color` after, docs/same-dark unchanged. */ +function artifact(color = [255, 255, 255, 255]) { + const root = mkdtempSync(join(tmpdir(), "artifact-")); + const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); + mkdirSync(before); + mkdirSync(after); + for (const folder of [before, after]) writeFileSync(join(folder, "same-dark.png"), encodePNG(image([40, 40, 40, 255]))); + writeFileSync(join(before, "hero-dark.png"), encodePNG(image([40, 40, 40, 255]))); + writeFileSync(join(after, "hero-dark.png"), encodePNG(image(color))); + const log = console.log; + console.log = () => {}; + try { + compare([before, after, out, "--prefix", "docs"]); + } finally { + console.log = log; + } + return out; +} + +/** + * The parts of Octokit the bot uses, over a little state with its own clock, which answers in the Date + * header. Uploads get GitHub's renaming of special characters. + */ +function fakeGitHub({ pull = {}, comments = [], labels = [], releases = [], files = ["Sources/StableBar/BarView.swift"] } = {}) { + const state = { + now: new Date("2026-10-31T10:00:00Z"), comments, labels, releases, files, events: [], statuses: [], uploads: [], deleted: [], next: 1000, + pr: { number: 7, state: "open", merged: false, head: { sha: head, ref: "feat/x", repo: { owner: { login: "someone" }, full_name: "someone/StableBar" } }, + base: { sha: base, ref: "main" }, ...pull }, + runs: [{ id: 99, run_number: 50, run_attempt: 1, head_sha: head, path: ".github/workflows/ci.yml", pull_requests: [], created_at: "2026-10-31T09:50:00Z", + head_branch: "feat/x", head_repository: { full_name: "someone/StableBar" } }], + duringUpload: null, others: [], + jobs: { 99: jobsOf(conclusions) }, artifacts: { 99: [{ name: "screens-app" }] }, + }; + const at = () => state.now.toISOString(); + const rest = { + pulls: { + get: async ({ pull_number }) => { + if (pull_number !== state.pr.number) throw notFound(); + return { data: { ...state.pr, labels: state.labels.map((name) => ({ name })) }, headers: { date: state.now.toUTCString() } }; + }, + list: async ({ head: filter, state: wanted }) => + ({ data: [state.pr, ...state.others].filter((pr) => (wanted === "all" || pr.state === wanted) && `${pr.head.repo.owner.login}:${pr.head.ref}` === filter) }), + listFiles: async () => ({ data: state.files.map((filename) => ({ filename })) }), + }, + issues: { + listComments: async () => ({ data: state.comments }), + createComment: async ({ body }) => { + const comment = { id: state.next++, user: bot, body, html_url: `https://example.test/comment/${state.next}`, updated_at: at() }; + state.comments.push(comment); + return { data: comment }; + }, + updateComment: async ({ comment_id, body }) => { + const comment = state.comments.find((entry) => entry.id === comment_id); + comment.body = body; + comment.updated_at = at(); + return { data: comment }; + }, + listEvents: async () => ({ data: state.events }), + removeLabel: async ({ name }) => { + if (!state.labels.includes(name)) throw notFound(); + state.labels = state.labels.filter((entry) => entry !== name); + state.events.push({ event: "unlabeled", label: { name }, created_at: at() }); + return { data: [] }; + }, + }, + repos: { + createCommitStatus: async (status) => { + state.statuses.push(status); + return { data: status }; + }, + getReleaseByTag: async ({ tag }) => { + const release = state.releases.find((entry) => entry.tag_name === tag); + if (!release) throw notFound(); + return { data: release }; + }, + createRelease: async (params) => { + const release = { id: state.next++, ...params, assets: [] }; + state.releases.push(release); + return { data: release }; + }, + listReleases: async () => ({ data: state.releases }), + listReleaseAssets: async ({ release_id }) => ({ data: state.releases.find((release) => release.id === release_id).assets }), + uploadReleaseAsset: async ({ release_id, name, data, headers }) => { + // Uploads take a while: time passes, and a maintainer may act meanwhile. + state.now = new Date(state.now.getTime() + 10_000); + if (state.duringUpload) { + const act = state.duringUpload; + state.duringUpload = null; + act(); + } + const renamed = name.replaceAll("~", "."); + const asset = { id: state.next++, name: renamed, browser_download_url: `https://example.test/download/${renamed}` }; + state.releases.find((release) => release.id === release_id).assets.push(asset); + state.uploads.push({ name, size: data.length, type: headers["content-type"], data: Buffer.from(data) }); + return { data: asset }; + }, + deleteReleaseAsset: async ({ asset_id }) => { + for (const release of state.releases) { + const index = release.assets.findIndex((asset) => asset.id === asset_id); + if (index >= 0) state.deleted.push(...release.assets.splice(index, 1).map((asset) => asset.name)); + } + }, + }, + actions: { + listWorkflowRuns: async ({ head_sha }) => ({ data: { workflow_runs: state.runs.filter((run) => run.head_sha === head_sha) } }), + listJobsForWorkflowRun: async ({ run_id }) => ({ data: { jobs: state.jobs[run_id] ?? [] } }), + listWorkflowRunArtifacts: async ({ run_id }) => ({ data: { artifacts: state.artifacts[run_id] ?? [] } }), + }, + }; + const paginate = async (method, params) => { + const { data } = await method(params); + return Array.isArray(data) ? data : data.jobs ?? data.artifacts ?? data.workflow_runs; + }; + /** Time passes; then a maintainer adds a label. */ + const label = (name = "visual-ok") => { + state.now = new Date(state.now.getTime() + 60_000); + state.labels.push(name); + state.events.push({ event: "labeled", label: { name }, created_at: at() }); + }; + /** A push or a rerun: a new finished CI run for `sha`, with these jobs and artifacts, from the pull request's own fork unless `from` says otherwise. */ + const run = ({ sha = head, jobs = conclusions, artifacts = ["screens-app"], from = "someone/StableBar", created = null } = {}) => { + state.now = new Date(state.now.getTime() + 60_000); + const id = state.next++; + state.runs.push({ id, run_number: id, run_attempt: 1, head_sha: sha, path: ".github/workflows/ci.yml", pull_requests: [], + created_at: created ?? at(), head_branch: "feat/x", head_repository: { full_name: from } }); + state.jobs[id] = jobsOf(jobs); + state.artifacts[id] = artifacts.map((name) => ({ name })); + return id; + }; + /** Time passes; then the pull request's base changes. */ + const retarget = (ref = "develop") => { + state.now = new Date(state.now.getTime() + 60_000); + state.pr.base = { sha: "d".repeat(40), ref }; + state.events.push({ event: "base_ref_changed", created_at: at() }); + }; + return { github: { rest, paginate }, state, label, run, retarget }; +} + +const runEvent = (extra = {}) => ({ repo, serverUrl: "https://github.com", runId: 5, payload: { + repository: { default_branch: "main" }, + workflow_run: { id: 99, event: "pull_request", path: ".github/workflows/ci.yml", head_sha: head, head_branch: "feat/x", + head_repository: { owner: { login: "someone" } }, pull_requests: [], ...extra } } }); +const prEvent = (action, extra = {}) => ({ repo, serverUrl: "https://github.com", runId: 6, payload: { + repository: { default_branch: "main" }, action, label: { name: "visual-ok" }, pull_request: { number: 7 }, ...extra } }); +/** What screens.yml does for one event: plan, download the run's artifacts, apply. */ +const reconcile = async (github, context, folders = {}) => apply({ github, context, plan: await plan({ github, context }), folders }); +const comment = (state) => state.comments.find((entry) => entry.user.login === bot.login); + +test("plan finds a fork's pull request, and the newest finished CI run for its head", async () => { + const { github, state, run } = fakeGitHub({ files: ["Sources/StableBar/BarView.swift", "README.md"] }); + assert.deepEqual(await plan({ github, context: runEvent() }), { action: "publish", number: 7, run: { id: 99, attempt: 1, head, createdAt: "2026-10-31T09:50:00Z", + jobs: { current: "success", build: "success", app: "success", site: "skipped" }, artifacts: { app: true, site: false }, expected: { app: true, site: false } } }); + // A newer run for the head, a rerun or a run another event started, is the one to publish. + const newer = run({ jobs: { "Check revision": "success" }, artifacts: [] }); + const chosen = await plan({ github, context: prEvent("labeled") }); + assert.deepEqual([chosen.run.id, chosen.run.jobs], [newer, { current: "success", build: "missing", app: "missing", site: "missing" }]); + assert.equal(state.uploads.length, 0); +}); + +test("plan takes only CI's pull request runs, and leaves a run already shown to review", async () => { + const { github } = fakeGitHub(); + assert.deepEqual(await plan({ github, context: runEvent({ path: ".github/workflows/other.yml" }) }), { action: "none" }); + assert.deepEqual(await plan({ github, context: runEvent({ event: "push" }) }), { action: "none" }); + await reconcile(github, runEvent(), { app: artifact() }); + assert.deepEqual(await plan({ github, context: runEvent() }), { action: "review", number: 7 }); +}); + +test("a publication uploads the changed screens, comments, sets the status and drops older heads' images", async () => { + const old = { id: 1, name: `pr7-${oldHead.slice(0, 7)}-docs.hero-dark-after.png` }, other = { id: 2, name: "pr8-1234567-docs.a-after.png" }; + const { github, state } = fakeGitHub({ releases: [{ id: 3, tag_name: "pr-previews-2026-09", assets: [old, other] }] }); + const { review } = await reconcile(github, runEvent(), { app: artifact() }); + assert.deepEqual(review, { state: "pending", description: "1 screen changed: a maintainer adds visual-ok after looking.", removeLabel: false }); + const release = state.releases.find((entry) => entry.tag_name === releaseTag(new Date())); + assert.deepEqual([release.prerelease, release.make_latest, release.target_commitish], [true, "false", "main"]); + assert.deepEqual(state.uploads.map((upload) => [upload.name, upload.type]).sort(), + ["after", "before", "diff"].map((kind) => [`pr7-aaaaaaa-docs~hero-dark-${kind}.png`, "image/png"])); + assert.match(comment(state).body, /src="https:\/\/example\.test\/download\/pr7-aaaaaaa-docs\.hero-dark-after\.png"/); + const data = readData(comment(state).body); + // The screens appeared when the comment was written, after the uploads. + assert.deepEqual([data.head, data.base, data.baseRef, data.runId, data.runAttempt, data.runCreatedAt, data.total, data.changed, data.shownAt], + [head, base, "main", 99, 1, "2026-10-31T09:50:00Z", 2, 1, "2026-10-31T10:00:30.000Z"]); + assert.deepEqual(state.statuses, [{ ...repo, sha: head, state: "pending", context: "Visual review", description: review.description, + target_url: comment(state).html_url }]); + assert.deepEqual(state.deleted, [old.name]); + // Reconciling again uploads nothing new. + await reconcile(github, runEvent(), { app: artifact() }); + assert.equal(state.uploads.length, 3); +}); + +test("visual-ok approves the screens shown before it, survives a rerun of them, and comes off for others", async () => { + const { github, state, label, run } = fakeGitHub(); + await reconcile(github, runEvent(), { app: artifact() }); + label(); + assert.deepEqual((await reconcile(github, prEvent("labeled"))).review, { state: "success", description: "1 changed screen approved.", removeLabel: false }); + assert.equal(state.statuses.at(-1).state, "success"); + run(); + assert.deepEqual([(await reconcile(github, runEvent(), { app: artifact() })).review.state, state.labels], ["success", ["visual-ok"]]); + run(); + const changed = (await reconcile(github, runEvent(), { app: artifact([255, 0, 0, 255]) })).review; + assert.deepEqual([changed.state, state.labels], ["pending", []]); + assert.match(comment(state).body, /`visual-ok` was taken off/); + assert.equal(state.comments.length, 1); + assert.equal(state.statuses.at(-1).state, "pending"); +}); + +test("an approval holds even when the event of its label never ran", async () => { + const { github, state, label, run } = fakeGitHub(); + await reconcile(github, runEvent(), { app: artifact() }); + label(); + // GitHub dropped the label's reconcile; the next run's reconcile still sees the label came after the screens. + run(); + assert.equal((await reconcile(github, runEvent(), { app: artifact() })).review.state, "success"); + assert.deepEqual(state.labels, ["visual-ok"]); +}); + +test("visual-ok added before the screens appeared comes off, and removing it sends the review back to pending", async () => { + const { github, state, label } = fakeGitHub(); + label(); + assert.equal((await reconcile(github, runEvent(), { app: artifact() })).review.state, "pending"); + assert.deepEqual(state.labels, []); + label(); + assert.equal((await reconcile(github, prEvent("labeled"))).review.state, "success"); + state.labels = []; + assert.equal((await reconcile(github, prEvent("unlabeled"))).review.state, "pending"); +}); + +test("a label added while a newer head is drawn leaves the status to that head's run", async () => { + const { github, state, label } = fakeGitHub(); + await reconcile(github, runEvent(), { app: artifact() }); + state.pr.head = { ...state.pr.head, sha: oldHead }; + label(); + const statuses = state.statuses.length; + assert.equal((await reconcile(github, prEvent("labeled"))).action, "review"); + assert.equal(state.statuses.length, statuses); +}); + +test("a changed base sends the review back to pending until a run started after it", async () => { + const { github, state, label, run, retarget } = fakeGitHub(); + await reconcile(github, runEvent(), { app: artifact() }); + label(); + await reconcile(github, prEvent("labeled")); + const before = state.now.toISOString(); + retarget(); + const pending = { state: "pending", description: "The base changed: push a commit to compare the screens with it.", removeLabel: false }; + const edited = await reconcile(github, prEvent("edited", { changes: { base: { ref: { from: "main" } } } })); + assert.deepEqual([edited.action, edited.review], ["review", pending]); + assert.match(comment(state).body, /\*\*Visual review:\*\* The base changed/); + // A run that started before the change, and finished after it, compared the old base: still pending. + const early = run({ created: before }); + const stale = await reconcile(github, runEvent(), { app: artifact() }); + assert.deepEqual([stale.action, readData(comment(state).body).runId, stale.review], ["publish", early, pending]); + // A run started after the change publishes, and the approval from before it doesn't count. + run(); + assert.deepEqual([(await reconcile(github, runEvent(), { app: artifact() })).review.state, state.labels], ["pending", []]); +}); + +test("a publication cut short before it recorded when its screens appeared is repaired by the next reconcile", async () => { + const { github, state, label } = fakeGitHub(); + const update = github.rest.issues.updateComment; + github.rest.issues.updateComment = async () => { + throw Object.assign(new Error("Server Error"), { status: 502 }); + }; + await assert.rejects(reconcile(github, runEvent(), { app: artifact() }), /Server Error/); + github.rest.issues.updateComment = update; + assert.equal(readData(comment(state).body).shownAt, null); + label(); + assert.equal((await reconcile(github, prEvent("labeled"))).review.state, "success"); +}); + +test("only the pull request's own head repository and branch publish screens", async () => { + const { github, state, run } = fakeGitHub(); + // Someone else's fork, at the same commit, with a newer run. + run({ from: "somebody-else/StableBar" }); + const chosen = await plan({ github, context: runEvent() }); + assert.equal(chosen.run.id, 99); + state.runs = state.runs.filter((entry) => entry.id !== 99); + assert.deepEqual(await plan({ github, context: runEvent() }), { action: "review", number: 7 }); +}); + +test("another pull request at the same branch and commit, open or closed, stops the comparison until a new commit", async () => { + const { github, state, run } = fakeGitHub(); + // The same branch, opened again against another base: its CI runs look like this pull request's. + state.others.push({ ...state.pr, number: 8, base: { sha: "d".repeat(40), ref: "develop" } }); + const { review } = await reconcile(github, runEvent(), { app: artifact() }); + const reason = "Another pull request is at this branch's commit: close it if it's open, then push a commit."; + assert.deepEqual(review, { state: "failure", description: `Drawing the screens failed: ${reason}`, removeLabel: false }); + assert.ok(comment(state).body.includes("can't tell their CI runs apart")); + assert.equal(state.uploads.length, 0); + // Closed, it still has runs at this commit. + state.others[0].state = "closed"; + assert.equal((await reconcile(github, runEvent(), { app: artifact() })).action, "ambiguous"); + // A new commit is this pull request's alone. + const pushed = "e".repeat(40); + state.pr.head = { ...state.pr.head, sha: pushed }; + run({ sha: pushed }); + assert.equal((await reconcile(github, runEvent({ head_sha: pushed }), { app: artifact() })).action, "publish"); + assert.equal(state.uploads.length, 3); +}); + +test("a label added while the screens upload doesn't approve them", async () => { + const { github, state, label } = fakeGitHub(); + state.duringUpload = () => label(); + assert.equal((await reconcile(github, runEvent(), { app: artifact() })).review.state, "pending"); + // The label's own reconcile follows: it came before the screens appeared, so it comes off. + assert.deepEqual([(await reconcile(github, prEvent("labeled"))).review.state, state.labels], ["pending", []]); + label(); + assert.equal((await reconcile(github, prEvent("labeled"))).review.state, "success"); +}); + +test("lookalike comments are ignored", async () => { + const lookalike = { id: 1, user: { login: "someone" }, body: `\n` }; + const { github, state, label } = fakeGitHub({ comments: [lookalike] }); + label(); + assert.equal((await reconcile(github, prEvent("labeled"))).action, "publish"); + assert.equal(state.comments.length, 2); + assert.equal(state.statuses.at(-1).state, "failure"); +}); + +test("a run that drew no screens passes without a comment, or rewrites the one an earlier head left", async () => { + const { github, state, run } = fakeGitHub(); + state.jobs[99] = jobsOf({ "Check revision": "success", "Build release": "skipped", "Draw screens": "skipped", "Draw website screens": "skipped" }); + state.artifacts[99] = []; + state.files = ["docs/README.md"]; + assert.deepEqual((await reconcile(github, runEvent())).review, { state: "success", description: "No screens to compare.", removeLabel: false }); + assert.deepEqual([state.comments.length, state.statuses[0].state], [0, "success"]); + state.files = ["Sources/StableBar/BarView.swift"]; + run(); + await reconcile(github, runEvent(), { app: artifact() }); + run({ jobs: { "Check revision": "success", "Build release": "skipped", "Draw screens": "skipped", "Draw website screens": "skipped" }, artifacts: [] }); + state.files = ["docs/README.md"]; + await reconcile(github, runEvent()); + assert.match(comment(state).body, /changes neither the app nor the website/); + assert.equal(readData(comment(state).body).ranScreens, false); +}); + +test("failed and missing jobs, and screens the changed files call for, fail the review", async () => { + const cases = [ + [{ "Check revision": "success", "Build release": "failure", "Draw screens": "skipped", "Draw website screens": "skipped" }, [], + "Build release ended with failure, so Draw screens didn't run."], + [{ "Check revision": "success", "Build release": "success", "Draw website screens": "skipped" }, [], "Draw screens is missing from this CI run."], + [{ "Check revision": "success", "Build release": "success", "Draw screens": "skipped", "Draw website screens": "skipped" }, [], + "Draw screens was skipped, but this pull request changes what it draws."], + [{ ...conclusions, "Draw screens": "cancelled" }, [], "Draw screens ended with cancelled."], + ]; + for (const [jobs, artifacts, error] of cases) { + const { github, state } = fakeGitHub(); + state.jobs[99] = jobsOf(jobs); + state.artifacts[99] = artifacts; + assert.equal((await reconcile(github, runEvent())).review.state, "failure", error); + assert.ok(comment(state).body.includes(`- \`app\`: \`${error}\``), error); + } +}); + +test("unusable artifacts are refused whole", async () => { + const refused = async (folder, expected, kind = "app") => { + const { github, state } = fakeGitHub(); + if (kind === "site") { + state.jobs[99] = jobsOf({ "Check revision": "success", "Build release": "skipped", "Draw screens": "skipped", "Draw website screens": "success" }); + state.artifacts[99] = [{ name: "screens-site" }]; + state.files = ["site/index.html"]; + } + assert.equal((await reconcile(github, runEvent(), { [kind]: folder })).review.state, "failure"); + assert.match(comment(state).body, expected); + assert.equal(state.uploads.length, 0); + }; + const linked = artifact(); + symlinkSync("/etc/hosts", join(linked, "hosts.png")); + await refused(linked, /The screens-app artifact can't be used: 'hosts\.png': not a regular file\./); + await refused(artifact(), /its id isn't site\/ followed by a plain name/, "site"); + const unreadable = artifact(); + writeFileSync(join(unreadable, "docs~hero-dark-diff.png"), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])); + await refused(unreadable, /'docs~hero-dark-diff\.png' isn't a PNG the bot can read/); + const empty = mkdtempSync(join(tmpdir(), "artifact-")); + writeFileSync(join(empty, "manifest.json"), JSON.stringify({ screens: [], failed: [] })); + await refused(empty, /The screens-app artifact can't be used: It holds no screens\./); + const scenes = mkdtempSync(join(tmpdir(), "artifact-")); + writeFileSync(join(scenes, "manifest.json"), JSON.stringify({ screens: [{ id: "scenes/bar-dark", status: "unchanged", changedPixels: 0, + region: null, images: {} }], failed: [] })); + await refused(scenes, /It holds no docs images\./); +}); + +/** `file` with a text chunk after its header: other bytes, the same pixels. */ +function withText(file) { + const png = readFileSync(file); + const body = Buffer.concat([Buffer.from("tEXt", "latin1"), Buffer.from("Comment\0from a pull request", "latin1")]); + const length = Buffer.alloc(4); + length.writeUInt32BE(body.length - 4); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(zlib.crc32(body)); + writeFileSync(file, Buffer.concat([png.subarray(0, 33), length, body, crc, png.subarray(33)])); +} + +test("a publication uploads its own encoding of the pixels, and fingerprints the pixels", async () => { + const { github, state, run } = fakeGitHub(); + const plain = artifact(); + await reconcile(github, runEvent(), { app: plain }); + const first = readData(comment(state).body).fingerprint; + const uploaded = state.uploads.find((upload) => upload.name.endsWith("after.png")); + assert.deepEqual(uploaded.data, encodePNG(decodePNG(readFileSync(join(plain, "docs~hero-dark-after.png"))))); + const texted = artifact(); + for (const kind of ["before", "after", "diff"]) withText(join(texted, `docs~hero-dark-${kind}.png`)); + run(); + await reconcile(github, runEvent(), { app: texted }); + assert.equal(readData(comment(state).body).fingerprint, first); + assert.ok(!state.uploads.at(-1).data.includes("from a pull request")); +}); + +test("a publication moves on to the month's next release when one is full", async () => { + const month = releaseTag(new Date()); + const full = { id: 3, tag_name: month, assets: Array.from({ length: 999 }, (_, index) => ({ id: 10000 + index, name: `pr1-1234567-${index}.png` })) }; + const { github, state } = fakeGitHub({ releases: [full] }); + await reconcile(github, runEvent(), { app: artifact() }); + const next = state.releases.find((release) => release.tag_name === `${month}-2`); + assert.deepEqual([full.assets.length, next?.assets.length, next?.prerelease], [999, 3, true]); + assert.match(comment(state).body, /download\/pr7-aaaaaaa-docs\.hero-dark-after\.png/); +}); + +test("closing unmerged deletes the images from every month; merging keeps them; a closed pull request publishes nothing", async () => { + const { github, state } = fakeGitHub({ releases: [ + { id: 1, tag_name: "pr-previews-2026-09", assets: [{ id: 11, name: "pr8-aaaaaaa-a.png" }] }, + { id: 2, tag_name: "pr-previews-2026-10", assets: [{ id: 21, name: "pr7-bbbbbbb-b.png" }] }, + { id: 3, tag_name: "v1.0.0", assets: [{ id: 31, name: "pr7-ccccccc-StableBar.dmg" }] }] }); + await reconcile(github, runEvent(), { app: artifact() }); + state.pr.state = "closed"; + assert.deepEqual(await reconcile(github, prEvent("closed")), { action: "close", deleted: 3 }); + assert.deepEqual(state.releases.slice(0, 3).map((release) => [release.tag_name, release.assets.map((asset) => asset.name)]), + [["pr-previews-2026-09", ["pr8-aaaaaaa-a.png"]], ["pr-previews-2026-10", []], ["v1.0.0", ["pr7-ccccccc-StableBar.dmg"]]]); + assert.match(comment(state).body, /closed without merging, so the images of its screens were deleted/); + assert.deepEqual(await reconcile(github, runEvent(), { app: artifact() }), { action: "none" }); + assert.equal(state.uploads.length, 3); + // Reopened, it shows its last run's screens again at once. + state.pr.state = "open"; + assert.equal((await reconcile(github, prEvent("reopened"), { app: artifact() })).action, "publish"); + assert.equal(state.uploads.length, 6); + assert.doesNotMatch(comment(state).body, /closed without merging/); + const merged = fakeGitHub(); + await reconcile(merged.github, runEvent(), { app: artifact() }); + Object.assign(merged.state.pr, { state: "closed", merged: true }); + assert.deepEqual(await reconcile(merged.github, prEvent("closed")), { action: "close", deleted: 0 }); + assert.equal(merged.state.deleted.length, 0); +}); + +test("a reconcile that can't plan still finds its pull request, so its head is marked errored", async () => { + const { github, state } = fakeGitHub(); + github.rest.actions.listWorkflowRuns = async () => { + throw Object.assign(new Error("Server Error"), { status: 502 }); + }; + // As the Plan step does: a fork's CI run lists no pull request, so the number comes from its branch. + await assert.rejects(plan({ github, context: runEvent() }), /Server Error/); + const number = await pullRequestNumber({ github, context: runEvent() }); + assert.equal(number, 7); + await failRun({ github, context: runEvent(), number }); + assert.deepEqual(state.statuses.map((status) => [status.sha, status.state]), [[head, "error"]]); + assert.equal(await pullRequestNumber({ github, context: prEvent("labeled") }), 7); +}); + +test("failRun falls back on the event's own pull request when GitHub can't answer", async () => { + const { github, state } = fakeGitHub(); + github.rest.pulls.get = async () => { + throw Object.assign(new Error("Server Error"), { status: 502 }); + }; + // A base change whose plan failed on reading the pull request: the event carries its head. + const edited = prEvent("edited", { pull_request: { number: 7, state: "open", head: { sha: head } } }); + await failRun({ github, context: edited, number: 7 }); + assert.deepEqual(state.statuses.map((status) => [status.sha, status.state]), [[head, "error"]]); + // A CI run carries no pull request to fall back on. + await assert.rejects(failRun({ github, context: runEvent(), number: 7 }), /Server Error/); +}); + +test("failRun marks an open pull request's head when the bot itself fails", async () => { + const { github, state } = fakeGitHub(); + await failRun({ github, context: runEvent(), number: 7 }); + assert.deepEqual(state.statuses, [{ ...repo, sha: head, state: "error", context: "Visual review", + description: "The screens bot failed: see the Screens workflow run.", target_url: "https://github.com/Seb-GRAF/StableBar/actions/runs/5" }]); + await failRun({ github, context: runEvent(), number: undefined }); + assert.equal(state.statuses.length, 1); +}); + +test("changePatterns call for screens when the files the scenes read change", () => { + for (const file of ["Sources/StableBar/BarView.swift", "Resources/config.json", "scenes/starter-bar.json", "examples/essentials.json"]) { + assert.equal(changePatterns.app.test(file), true, file); + } + assert.equal(changePatterns.app.test("docs/README.md"), false); + assert.equal(changePatterns.site.test("site/index.html"), true); +}); + +test("changePatterns match the files CI's Check revision job draws screens for", () => { + const workflow = readFileSync(new URL("../workflows/ci.yml", import.meta.url), "utf8"); + const found = Object.fromEntries([...workflow.matchAll(/grep -qE '([^']+)' <<< "\$changed"; then\n\s+echo "(\w+)=true"/g)] + .map(([, pattern, output]) => [output, pattern])); + // A JavaScript pattern's source writes "/" as "\/". + const grep = (pattern) => pattern.source.replaceAll("\\/", "/"); + assert.deepEqual({ app: grep(changePatterns.app), site: grep(changePatterns.site) }, { app: found.screens, site: found.site }); +}); diff --git a/.github/scripts/screens-comment.mjs b/.github/scripts/screens-comment.mjs new file mode 100644 index 0000000..f9d9959 --- /dev/null +++ b/.github/scripts/screens-comment.mjs @@ -0,0 +1,326 @@ +// The logic of the before/after screens comment and its Visual review status, for screens.yml. The +// workflow runs the default branch's copy and reads the pull request's artifact only as data: the +// pull request's CI job writes it, so every name and message in it is untrusted. Such text reaches +// the comment only inside code spans, and the comment's data is always its last line. +import { createHash } from "node:crypto"; + +export const botLogin = "github-actions[bot]"; +const marker = ""; +const reviewHeading = "**Visual review:**"; +const dataPattern = /\r?\n\s*$/; +const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); +const maximumFileSize = 10 * 1024 * 1024; +const maximumManifestSize = 1024 * 1024; +const maximumFiles = 300; +const maximumScreens = 300; +const maximumFailures = 50; +const maximumMessage = 1000; +const maximumProblems = 20; +/** GitHub takes 65,536 characters in a comment. */ +const maximumComment = 60000; +const maximumRows = 100; +const maximumListed = 20; +const statuses = ["unchanged", "changed", "resized", "added", "removed"]; +/** The images a screen comes with: a resized screen lined up from the top has a region and a difference too. */ +const imagesOf = (status, region) => ({ unchanged: [], changed: ["after", "before", "diff"], added: ["after"], removed: ["before"], + resized: region ? ["after", "before", "diff"] : ["after", "before"] })[status]; +const isRegion = (region) => hasOnly(region, ["x", "y", "width", "height"]) && isCount(region.x) && isCount(region.y) + && isCount(region.width, 1) && isCount(region.height, 1); + +const isObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value); +const hasOnly = (value, keys) => isObject(value) && Object.keys(value).every((key) => keys.includes(key)); +const isCount = (value, minimum = 0) => Number.isSafeInteger(value) && value >= minimum; +const isMessage = (value, limit = maximumMessage) => typeof value === "string" && value.length > 0 && value.length <= limit; +const isHex = (value, length) => typeof value === "string" && value.length === length && /^[0-9a-f]+$/.test(value); +const isImageName = (name) => /^[A-Za-z0-9][A-Za-z0-9._~-]*\.png$/.test(name) && !name.includes("..") && name.length <= 200; +const isScreenId = (id, prefixes) => typeof id === "string" && prefixes.includes(id.split("/")[0]) + && /^[a-z]+\/[A-Za-z0-9][A-Za-z0-9._-]*$/.test(id) && !id.includes("..") && id.length <= 100; +/** The name compare-screens.mjs gives a screen's image of one kind. */ +const imageOf = (id, kind) => `${id.replaceAll("/", "~")}-${kind}.png`; + +/** `text` on one line and at most `limit` characters long. */ +function cut(text, limit) { + const characters = Array.from(String(text ?? "").replace(/\s+/g, " ").trim()); + return characters.length <= limit ? characters.join("") : `${characters.slice(0, limit - 1).join("")}…`; +} + +/** Untrusted text as an inline code span: markdown, HTML, links and mentions in it stay plain text. */ +const code = (text, limit = 200) => `\`${cut(text, limit).replaceAll("`", "'") || " "}\``; +const entities = { "&": "&", '"': """, "<": "<", ">": ">" }; +const attribute = (value) => String(value).replace(/[&"<>]/g, (character) => entities[character]); +/** The data's JSON, which then holds no "<" or ">" and so can't close its HTML comment early. */ +const encodeData = (data) => JSON.stringify(data).replace(/[<>&]/g, (character) => `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`); +const plural = (count, word) => `${count} ${word}${count === 1 ? "" : "s"}`; +const list = (ids, limit) => [...ids.slice(0, limit).map((id) => code(id)), ...(ids.length > limit ? [`and ${ids.length - limit} more`] : [])].join(", "); + +/** Checks an artifact's files before anything is read or uploaded: regular files only, PNG images and a small manifest. */ +export function validateArtifact(files) { + const problems = []; + if (files.length > maximumFiles) problems.push(`${files.length} files, more than ${maximumFiles}.`); + for (const file of files) { + const name = code(file.name, 120); + if (!file.regular) { + problems.push(`${name}: not a regular file.`); + } else if (file.name === "manifest.json") { + if (file.size > maximumManifestSize) problems.push(`${name}: larger than 1 MB.`); + } else if (!isImageName(file.name)) { + problems.push(`${name}: not an image name this bot writes.`); + } else if (!Buffer.from(file.head ?? []).subarray(0, 8).equals(signature)) { + problems.push(`${name}: not a PNG.`); + } else if (file.size > maximumFileSize) { + problems.push(`${name}: larger than 10 MB.`); + } + } + return { ok: problems.length === 0, problems: problems.slice(0, maximumProblems) }; +} + +function readScreen(screen, where, files, ids, prefixes, problems) { + if (!hasOnly(screen, ["id", "status", "changedPixels", "region", "images", "afterDigest", "truncated", "baseError"])) { + problems.push(`${where} isn't a screen as compare-screens.mjs writes it.`); + return null; + } + if (!isScreenId(screen.id, prefixes)) problems.push(`${where}: its id isn't ${prefixes.map((prefix) => `${prefix}/`).join(" or ")} followed by a plain name.`); + else if (ids.has(screen.id)) problems.push(`${where}: ${code(screen.id)} is listed twice.`); + ids.add(screen.id); + if (!statuses.includes(screen.status)) { + problems.push(`${where}: its status isn't one of ${statuses.join(", ")}.`); + return null; + } + if (!isCount(screen.changedPixels)) problems.push(`${where}: changedPixels isn't a count.`); + const { region } = screen; + const regionFits = screen.status === "changed" ? isRegion(region) : screen.status === "resized" ? region === null || isRegion(region) : region === null; + if (!regionFits) problems.push(`${where}: its region doesn't fit its status.`); + const kinds = imagesOf(screen.status, regionFits && region !== null); + if (!hasOnly(screen.images, kinds) || Object.keys(screen.images).length !== kinds.length) { + problems.push(`${where}: a screen that is ${screen.status} comes with ${kinds.length ? kinds.join(", ") : "no images"}.`); + } else { + for (const kind of kinds) { + const file = screen.images[kind]; + if (typeof screen.id !== "string" || file !== imageOf(screen.id, kind)) problems.push(`${where}: its ${kind} image isn't named after it.`); + else if (!isImageName(file) || !files.has(file)) problems.push(`${where}: its ${kind} image isn't in the artifact.`); + } + } + if (screen.baseError !== undefined && !isMessage(screen.baseError)) problems.push(`${where}: baseError isn't a message of at most ${maximumMessage} characters.`); + const cropped = screen.status === "changed" || screen.status === "resized"; + if (cropped ? !isHex(screen.afterDigest, 64) : screen.afterDigest !== undefined) problems.push(`${where}: afterDigest doesn't fit its status.`); + if (screen.truncated !== undefined && typeof screen.truncated !== "boolean") problems.push(`${where}: truncated isn't true or false.`); + return { + id: screen.id, status: screen.status, changedPixels: screen.changedPixels, + region: regionFits && region !== null ? { x: region.x, y: region.y, width: region.width, height: region.height } : null, + images: Object.fromEntries(kinds.map((kind) => [kind, screen.images?.[kind]])), + ...(cropped ? { afterDigest: screen.afterDigest } : {}), + ...(screen.truncated === true ? { truncated: true } : {}), + ...(screen.baseError === undefined ? {} : { baseError: screen.baseError }), + }; +} + +/** + * The screens and failures of an artifact's manifest.json, or why it can't be used. `names` are the + * artifact's files, and `prefixes` the kinds of screens it may hold. Only known fields come back, and + * every image a screen names is one of those files, named after the screen. + */ +export function readManifest(text, names, prefixes = ["docs", "scenes", "site"]) { + const invalid = (problems) => ({ ok: false, problems: problems.slice(0, maximumProblems), screens: [], failed: [] }); + let manifest = null; + try { + manifest = JSON.parse(text); + } catch { + return invalid(["manifest.json isn't JSON."]); + } + if (!hasOnly(manifest, ["screens", "failed"]) || !Array.isArray(manifest.screens) || !Array.isArray(manifest.failed)) { + return invalid(["manifest.json isn't a list of screens and failures."]); + } + if (manifest.screens.length > maximumScreens) return invalid([`manifest.json lists more than ${maximumScreens} screens.`]); + if (manifest.failed.length > maximumFailures) return invalid([`manifest.json lists more than ${maximumFailures} failures.`]); + const problems = [], files = new Set(names), ids = new Set(); + const screens = manifest.screens.map((screen, index) => readScreen(screen, `Screen ${index + 1}`, files, ids, prefixes, problems)); + const failed = manifest.failed.map((entry, index) => { + if (hasOnly(entry, ["id", "error"]) && isMessage(entry.id, 300) && isMessage(entry.error)) return { id: entry.id, error: entry.error }; + problems.push(`Failure ${index + 1} isn't an id and a message of at most ${maximumMessage} characters.`); + return null; + }); + return problems.length > 0 ? invalid(problems) : { ok: true, problems: [], screens, failed }; +} + +/** + * What a maintainer approves with visual-ok: each changed screen with its status, the image the + * comment shows of it, its after image or, once removed, its before image, the digest of its whole + * after image, since a crop may show only part of a change, and every failure. `digest(file)` is the + * bot's own digest of an image it uploads; the whole image's digest comes from the pull request's CI, + * which decides its screens anyway. Order doesn't matter. + */ +export function fingerprint({ screens = [], failed = [] }, digest) { + const entries = [ + ...screens.filter((screen) => screen.status !== "unchanged") + .map((screen) => ["screen", screen.id, screen.status, digest(screen.images.after ?? screen.images.before), screen.afterDigest ?? null]), + ...failed.map((entry) => ["failed", entry.id, entry.error]), + ].map((entry) => JSON.stringify(entry)).sort(); + return createHash("sha256").update(JSON.stringify(entries)).digest("hex"); +} + +/** What the status needs to know of a run's screens, kept in the comment's data for label events. */ +export function summary(screens, failed) { + return { total: screens.length, changed: screens.filter((screen) => screen.status !== "unchanged").length, failed: failed.length, + failure: failed.length > 0 ? cut(`${failed[0].id}: ${failed[0].error}`, 140) : null }; +} + +/** + * The Visual review status, from a run's summary: success when nothing needs a look or visual-ok was + * added after the comment began showing these screens (`labeledAt` after `shownAt`, both ISO times + * from GitHub), pending while changed screens wait for it, failure when drawing failed. Screens + * compared with a base the pull request no longer has wait for a new run. `removeLabel` asks to take + * visual-ok off when it came before these screens. + */ +export function reviewState({ ranScreens, total = 0, changed = 0, failed = 0, failure = null, labels = [], labeledAt = null, shownAt = null, + baseChanged = false }) { + if (!ranScreens) return { state: "success", description: "No screens to compare.", removeLabel: false }; + if (baseChanged) return { state: "pending", description: "The base changed: push a commit to compare the screens with it.", removeLabel: false }; + const labelled = labels.includes("visual-ok"); + const valid = labelled && labeledAt !== null && shownAt !== null && Date.parse(labeledAt) > Date.parse(shownAt); + if (failed > 0) { + if (valid) return { state: "success", description: `Approved with visual-ok although ${plural(failed, "screen")} failed to draw.`, removeLabel: false }; + return { state: "failure", description: cut(`Drawing the screens failed: ${failure ?? "see the comment."}`, 140), removeLabel: labelled }; + } + if (changed === 0) return { state: "success", description: `${plural(total, "screen")}, none changed.`, removeLabel: false }; + if (valid) return { state: "success", description: `${plural(changed, "changed screen")} approved.`, removeLabel: false }; + return { state: "pending", description: `${plural(changed, "screen")} changed: a maintainer adds visual-ok after looking.`, removeLabel: labelled }; +} + +function describe(screen) { + switch (screen.status) { + case "changed": { + const { x, y, width, height } = screen.region; + return [`${screen.changedPixels.toLocaleString("en-US")} pixels`, `${x},${y} · ${width}×${height}`]; + } + case "added": return ["new", "the whole screen"]; + case "removed": return ["removed", "the whole screen"]; + case "resized": { + if (!screen.region) return ["new size", "the whole screen"]; + const { x, y, width, height } = screen.region; + return [`${screen.changedPixels.toLocaleString("en-US")} pixels, new size`, `${x},${y} · ${width}×${height}`]; + } + default: return ["", ""]; + } +} + +function section(screen, imageURL) { + const image = (file, label) => `${label}`; + const lines = [`#### ${code(screen.id)}`]; + if (screen.baseError) lines.push("", `The base couldn't draw this scene: ${code(screen.baseError, 300)}`); + if (screen.truncated) lines.push("", "The images show the first 1,600 rows of this change."); + const { before, after, diff } = screen.images; + if (before && after) { + lines.push("", "| Before | After |", "| --- | --- |", `| ${image(before, "Before")} | ${image(after, "After")} |`); + } else if (after) { + lines.push("", image(after, "After")); + } else if (before) { + lines.push("", image(before, "Before")); + } + if (diff) lines.push("", "Difference, changed pixels in red:", "", image(diff, "Difference")); + lines.push(""); + return lines.join("\n"); +} + +/** The review line, which shows no untrusted text, and the data. */ +function reviewSection(review, data) { + const lines = [`${reviewHeading} ${review.state === "failure" ? "Drawing the screens failed, see above." : review.description}`]; + if (review.removeLabel) lines.push("", "`visual-ok` was taken off: it was added before these screens appeared."); + lines.push("", ``); + return lines.join("\n"); +} + +/** + * The comment: changed screens with their images, unchanged ones folded, failures, the review line and + * the data. When it would be too long for GitHub, the last changed screens lose their images. + */ +export function commentBody({ baseRef = "main", base, head, screens, failed = [], imageURL, review, data }) { + const changed = screens.filter((screen) => screen.status !== "unchanged"); + const unchanged = screens.filter((screen) => screen.status === "unchanged"); + const sections = changed.map((screen) => section(screen, imageURL)); + const assemble = (shown) => { + const lines = [marker, "### Before and after", "", `${code(baseRef, 100)} at ${base.slice(0, 7)} → this pull request at ${head.slice(0, 7)}` + + ` · ${plural(screens.length, "screen")} drawn in one CI job · ${changed.length} changed`, ""]; + if (changed.length > 0) { + lines.push("| Screen | Change | Where (image pixels) |", "| --- | --- | --- |"); + for (const screen of changed.slice(0, maximumRows)) lines.push(`| ${code(screen.id)} | ${describe(screen).join(" | ")} |`); + if (changed.length > maximumRows) lines.push("", `${changed.length - maximumRows} more changed screens aren't in the table.`); + lines.push("", ...sections.slice(0, shown)); + if (shown < changed.length) { + lines.push(`Not shown, to keep this comment under GitHub's size limit: ${list(changed.slice(shown).map((screen) => screen.id), maximumRows)}.`, ""); + } + } + if (unchanged.length > 0) { + lines.push(`
${plural(unchanged.length, "unchanged screen")}`, "", + list(unchanged.map((screen) => screen.id), 2 * maximumRows), "", "
", ""); + } + if (failed.length > 0) { + lines.push("**Couldn't draw:**", "", ...failed.slice(0, maximumListed).map((entry) => `- ${code(entry.id)}: ${code(entry.error, 300)}`)); + if (failed.length > maximumListed) lines.push(`- and ${failed.length - maximumListed} more`); + lines.push(""); + } + lines.push(reviewSection(review, data)); + return lines.join("\n"); + }; + let shown = sections.length; + let body = assemble(shown); + while (body.length > maximumComment && shown > 0) body = assemble(--shown); + return body; +} + +/** A comment without screens: why there are none, which is the bot's own text, the review line and the data. */ +export function noScreensBody({ reason, review, data }) { + return [marker, "### Before and after", "", reason, "", reviewSection(review, data)].join("\n"); +} + +/** `body` with a new review line and data, for label events, which don't draw anything. */ +export function withReview(body, review, data) { + const start = body.lastIndexOf(`\n${reviewHeading} `); + return `${start >= 0 ? body.slice(0, start + 1) : `${body.trimEnd()}\n\n`}${reviewSection(review, data)}`; +} + +const isTime = (value) => typeof value === "string" && value.length <= 40 && !Number.isNaN(Date.parse(value)); + +/** The comment's data: what was drawn for which head and CI run, when that run started, and since when it's shown. */ +function isData(data) { + return hasOnly(data, ["head", "base", "baseRef", "runId", "runAttempt", "runCreatedAt", "ranScreens", "fingerprint", "shownAt", "total", "changed", + "failed", "failure"]) + && isHex(data.head, 40) && (data.base === null || isHex(data.base, 40)) + && (data.baseRef === null || (typeof data.baseRef === "string" && data.baseRef.length > 0 && data.baseRef.length <= 255)) + && (data.runId === null || isCount(data.runId, 1)) && (data.runAttempt === null || isCount(data.runAttempt, 1)) + && (data.runCreatedAt === null || isTime(data.runCreatedAt)) + && typeof data.ranScreens === "boolean" + && (data.fingerprint === null || isHex(data.fingerprint, 64)) && (data.shownAt === null || isTime(data.shownAt)) + && isCount(data.total) && isCount(data.changed) && isCount(data.failed) + && (data.failure === null || (typeof data.failure === "string" && data.failure.length <= 300)); +} + +/** The data on the last line of a comment from commentBody, or null. */ +export function readData(body) { + const match = dataPattern.exec(body ?? ""); + if (!match) return null; + try { + const data = JSON.parse(match[1]); + return isData(data) ? data : null; + } catch { + return null; + } +} + +export const isScreensComment = (body) => (body ?? "").startsWith(marker); + +/** The bot's screens comment among a pull request's comments. Anyone can post the marker; only the bot's comment counts. */ +export function findScreensComment(comments) { + return comments.find((comment) => comment.user?.login === botLogin && isScreensComment(comment.body)) ?? null; +} + +/** The name of an uploaded image: the pull request and head commit, then the file. */ +export const assetName = (pr, sha, file) => `pr${pr}-${sha.slice(0, 7)}-${file}`; + +/** The pre-release that holds a month's images. */ +export const releaseTag = (date) => `pr-previews-${date.toISOString().slice(0, 7)}`; + +/** A pull request's images. GitHub may rename the rest of an asset's name, but not this prefix. */ +export const pullRequestAssets = (assets, pr) => assets.filter((asset) => asset.name.startsWith(`pr${pr}-`)); + +/** A pull request's images of other heads than `sha`. */ +export const staleAssets = (assets, pr, sha) => pullRequestAssets(assets, pr).filter((asset) => !asset.name.startsWith(assetName(pr, sha, ""))); diff --git a/.github/scripts/screens-comment.test.mjs b/.github/scripts/screens-comment.test.mjs new file mode 100644 index 0000000..01e7057 --- /dev/null +++ b/.github/scripts/screens-comment.test.mjs @@ -0,0 +1,269 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { validateArtifact, readManifest, fingerprint, summary, reviewState, commentBody, withReview, noScreensBody, readData, + findScreensComment, assetName, releaseTag, staleAssets, pullRequestAssets } from "./screens-comment.mjs"; + +const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); +const sha = (character) => character.repeat(40); +const digest = (character) => character.repeat(64); +const screen = (id, status, images = {}, extra = {}) => ({ id, status, changedPixels: status === "changed" ? 120 : 0, + region: status === "changed" ? { x: 10, y: 20, width: 30, height: 40 } : null, images, + ...(status === "changed" || status === "resized" ? { afterDigest: "e".repeat(64) } : {}), ...extra }); +const data = (extra = {}) => ({ head: sha("a"), base: sha("b"), baseRef: "main", runId: 41, runAttempt: 1, runCreatedAt: "2026-10-31T09:50:00Z", ranScreens: true, + fingerprint: digest("f"), shownAt: "2026-10-31T10:00:00.000Z", + total: 2, changed: 1, failed: 0, failure: null, ...extra }); +const urls = (file) => `https://example.test/${file}`; +// What a pull request's CI job could put in a name or a message. +const hostile = `@octocat \`x\` | #1 https://evil.test **bold**\nsecond line`; + +test("validateArtifact accepts only regular PNG files and a small manifest", () => { + const good = [{ name: "manifest.json", size: 100, head: Buffer.from("{"), regular: true }, + { name: "docs~hero-dark-after.png", size: 2048, head: png, regular: true }]; + assert.deepEqual(validateArtifact(good), { ok: true, problems: [] }); + for (const bad of [{ name: "../x.png", size: 10, head: png, regular: true }, { name: "a.txt", size: 10, head: png, regular: true }, + { name: "fake.png", size: 10, head: Buffer.from("GIF89a.."), regular: true }, + { name: "big.png", size: 11 * 1024 * 1024, head: png, regular: true }, { name: "link.png", size: 10, head: png, regular: false }, + { name: "manifest.json", size: 2 * 1024 * 1024, head: Buffer.from("{"), regular: true }]) { + assert.equal(validateArtifact([...good.filter((file) => file.name !== bad.name), bad]).ok, false, bad.name); + } + const many = Array.from({ length: 301 }, (_, index) => ({ name: `s${index}.png`, size: 10, head: png, regular: true })); + assert.equal(validateArtifact(many).ok, false); +}); + +test("readManifest keeps a manifest compare-screens wrote", () => { + const names = ["docs~hero-dark-before.png", "docs~hero-dark-after.png", "docs~hero-dark-diff.png", "scenes~new-dark-after.png"]; + const manifest = { screens: [screen("docs/hero-dark", "changed", { before: names[0], after: names[1], diff: names[2] }), + screen("scenes/new-dark", "added", { after: names[3] }, { baseError: "draw: Unknown value" }), screen("docs/settings-dark", "unchanged")], + failed: [{ id: "scenes//runner/work/scenes/a b.json", error: "Another scene is also named a b." }] }; + assert.deepEqual(readManifest(JSON.stringify(manifest), names), { ok: true, problems: [], screens: manifest.screens, failed: manifest.failed }); +}); + +test("readManifest rejects what compare-screens doesn't write", () => { + const names = ["docs~a-after.png", "docs~a-before.png", "docs~a-diff.png"]; + const good = screen("docs/a", "changed", { before: names[1], after: names[0], diff: names[2] }); + const manifest = (screens, failed = []) => JSON.stringify({ screens, failed }); + const cases = { + "not JSON": "{", + "no lists": JSON.stringify({ screens: {} }), + "unknown top-level field": '{"screens":[],"failed":[],"__proto__":{}}', + "unknown field": manifest([{ ...good, extra: 1 }]), + "id with markup": manifest([{ ...good, id: "docs/a`/); + assert.match(body, /`main` at 1111111 → this pull request at aaaaaaa/); + assert.match(body, /\| Screen \| Change \| Where \(image pixels\) \|/); + assert.doesNotMatch(body, /first 1,600 rows/); + const long = commentBody({ base: sha("1"), head: sha("a"), failed: [], imageURL: urls, review, data: data(), + screens: [screen("site/page-dark", "changed", { before: "b.png", after: "a.png", diff: "d.png" }, { truncated: true })] }); + assert.match(long, /The images show the first 1,600 rows of this change\./); + assert.match(body, /https:\/\/example\.test\/b\.png/); + assert.match(body, /https:\/\/example\.test\/d\.png/); + assert.match(body, /The base couldn't draw this scene: `draw: Unknown value`/); + assert.match(body, /
1 unchanged screen<\/summary>/); + assert.doesNotMatch(body, /docs\/settings-dark.*example\.test/); + assert.match(body, /\n\*\*Visual review:\*\* 2 screens changed: a maintainer adds visual-ok after looking\.\n/); + assert.deepEqual(readData(body), data()); +}); + +test("commentBody keeps untrusted text in code spans and its data its own", () => { + const screens = [screen("scenes/new-dark", "added", { after: "n.png" }, { baseError: hostile })]; + const failed = [{ id: "scenes/x ", error: hostile }]; + const review = reviewState({ ranScreens: true, ...summary(screens, failed), labels: [] }); + const body = commentBody({ baseRef: "main`", base: sha("1"), head: sha("a"), screens, failed, + imageURL: (file) => `https://example.test/${file}?a="b"`, review, data: data({ failed: 1, failure: hostile }) }); + assert.deepEqual(readData(body), data({ failed: 1, failure: hostile })); + // The HTML comment holding the data ends where its line does, so none of it shows. + const dataLine = body.split("\n").at(-1); + assert.deepEqual([dataLine.indexOf("-->"), dataLine.lastIndexOf("<")], [dataLine.length - 3, 0]); + const shown = body.split("\n").filter((line) => !line.startsWith("\n### Before and after\n\nThis pull request changes neither/); + assert.match(body, /\*\*Visual review:\*\* No screens to compare\./); + assert.deepEqual(readData(body), data({ ranScreens: false })); + assert.equal(withReview(body, review, data()), noScreensBody({ reason: "This pull request changes neither the app nor the website.", review, data: data() })); +}); + +test("readData reads only the data on the comment's last line", () => { + const line = (value) => ``; + assert.deepEqual(readData(`\ntext\n${line(data())}`), data()); + assert.deepEqual(readData(`\r\ntext\r\n${line(data())}\r\n`), data()); + assert.deepEqual(readData(`\n${line(data({ runId: 42 }))}\n${line(data())}`), data()); + assert.equal(readData(`\n${line(data())}\nmore text`), null); + assert.equal(readData(`x\n${line({ ...data(), shownAt: "yesterday" })}`), null); + assert.equal(readData(`x\n${line({ ...data(), approved: null })}`), null); + assert.deepEqual(readData(`x\n${line(data({ ranScreens: false, fingerprint: null, shownAt: null, runId: null, runAttempt: null }))}`), + data({ ranScreens: false, fingerprint: null, shownAt: null, runId: null, runAttempt: null })); + assert.equal(readData(`x\n${line({ ...data(), runCreatedAt: "soon" })}`), null); + assert.equal(readData(`x\n${line({ ...data(), extra: 1 })}`), null); + assert.equal(readData("no data here"), null); +}); + +test("findScreensComment trusts only the bot's comment", () => { + const body = `\n`; + const impostor = { id: 1, user: { login: "someone", type: "User" }, body }; + const other = { id: 2, user: { login: "github-actions[bot]", type: "Bot" }, body: "Something else" }; + const bot = { id: 3, user: { login: "github-actions[bot]", type: "Bot" }, body }; + assert.equal(findScreensComment([impostor, other, bot]), bot); + assert.equal(findScreensComment([impostor, other]), null); +}); + +test("assetName and releaseTag name the uploaded images", () => { + assert.equal(assetName(412, "abcdef1234", "docs~hero-dark-after.png"), "pr412-abcdef1-docs~hero-dark-after.png"); + assert.equal(releaseTag(new Date("2026-11-03T10:00:00Z")), "pr-previews-2026-11"); +}); + +test("staleAssets and pullRequestAssets pick a pull request's images by name", () => { + const assets = ["pr7-abcdef1-docs.a-after.png", "pr7-1234567-docs.a-after.png", "pr70-1234567-docs.a-after.png", "pr8-abcdef1-x.png", "other.png"] + .map((name, id) => ({ id, name })); + assert.deepEqual(staleAssets(assets, 7, "abcdef1234").map((asset) => asset.name), ["pr7-1234567-docs.a-after.png"]); + assert.deepEqual(pullRequestAssets(assets, 7).map((asset) => asset.name), ["pr7-abcdef1-docs.a-after.png", "pr7-1234567-docs.a-after.png"]); +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fabe24..1efe461 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,8 @@ jobs: build: ${{ steps.revision.outputs.build }} app: ${{ steps.changes.outputs.app }} packaging: ${{ steps.changes.outputs.packaging }} + screens: ${{ steps.changes.outputs.screens }} + site: ${{ steps.changes.outputs.site }} steps: - name: Skip superseded pushes id: revision @@ -51,7 +53,10 @@ jobs: persist-credentials: false # A pull request compiles the release and draws the docs images only when it changes what they # build from: the files scripts/unchanged-app.sh compares, or this workflow. Packaging reads - # only the scripts and resources. HEAD merges the pull request into its base, HEAD^. + # only the scripts and resources. The before/after screens are drawn when the app, the scenes, + # the examples they read or the comparison change, and the website's when the website or its + # capture change. + # HEAD merges the pull request into its base, HEAD^. - name: Find what the pull request changes id: changes if: github.event_name == 'pull_request' @@ -63,6 +68,12 @@ jobs: if grep -qE '^(Resources/|scripts/|\.github/workflows/ci\.yml$)' <<< "$changed"; then echo "packaging=true" >> "$GITHUB_OUTPUT" fi + if grep -qE '^(Sources/|Resources/|Package\.(swift|resolved)$|scripts/|scenes/|examples/|\.github/workflows/ci\.yml$|\.github/scripts/(png|compare-screens)\.mjs$)' <<< "$changed"; then + echo "screens=true" >> "$GITHUB_OUTPUT" + fi + if grep -qE '^(site/|\.github/workflows/ci\.yml$|\.github/scripts/(png|compare-screens|capture-site)\.mjs$)' <<< "$changed"; then + echo "site=true" >> "$GITHUB_OUTPUT" + fi test: name: Test @@ -99,11 +110,11 @@ jobs: run: swift test --skip-build # Runs on main, and on pull requests that change the app, so a release-only failure shows up - # before it merges and leaves main without a beta. + # before it merges and leaves main without a beta. Pull requests that draw screens use it too. build: name: Build release needs: current - if: needs.current.outputs.build == 'true' && (github.ref == 'refs/heads/main' || needs.current.outputs.app == 'true') + if: needs.current.outputs.build == 'true' && (github.ref == 'refs/heads/main' || needs.current.outputs.app == 'true' || needs.current.outputs.screens == 'true') runs-on: macos-26 timeout-minutes: 20 steps: @@ -144,7 +155,8 @@ jobs: name: Docs images env: STABLEBAR_PREBUILT_DIR: release-build - needs: build + needs: [current, build] + if: github.ref == 'refs/heads/main' || needs.current.outputs.app == 'true' runs-on: macos-26 timeout-minutes: 15 steps: @@ -234,8 +246,9 @@ jobs: TITLE: ${{ github.event.pull_request.title }} run: printf '%s\n' "$TITLE" | npx commitlint --verbose - # Tests the logic of the Linked issue and Description checks. Those checks run from the default - # branch (pull-request-checks.yml), so a change to them is tested here before it merges. + # Tests the scripts in .github/scripts: the Linked issue and Description checks, and the + # before/after screens. The checks and the screens comment run from the default branch + # (pull-request-checks.yml, screens.yml), so a change to them is tested here before it merges. pull-request-checks: name: Test pull request checks if: github.event_name == 'pull_request' @@ -250,6 +263,132 @@ jobs: node-version: 24 - run: node --test .github/scripts/*.test.mjs + # Draws the docs images and the scenes with the pull request's app and with its base's, and + # compares them into the screens-app artifact. Both draw the pull request's scenes, each beside + # its own resources and examples. This job runs the pull request's code with a read-only token: + # screens.yml, which runs from the default branch, posts the comparison and the Visual review + # status. + screens: + name: Draw screens + env: + STABLEBAR_PREBUILT_DIR: release-build + needs: [current, build] + if: github.event_name == 'pull_request' && needs.current.outputs.screens == 'true' + runs-on: macos-26 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 2 + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: 24 + - uses: actions/download-artifact@v8 + with: + name: release-build + # The scenes and the comparison scripts are kept aside before the checkout moves to the base. + # A scene that fails to draw is reported in scenes.json, and later in the comparison. + - name: Draw the pull request's screens + timeout-minutes: 10 + run: | + mkdir release-build + tar -xf release-build.tar -C release-build + ./scripts/build.sh "$RUNNER_TEMP/head.app" + app="$RUNNER_TEMP/head.app/Contents/MacOS/StableBar" + "$app" --render-docs-images "$RUNNER_TEMP/after/docs" + "$app" --render-scenes "$RUNNER_TEMP/after/scenes" scenes || [[ -f "$RUNNER_TEMP/after/scenes/scenes.json" ]] + cp -R scenes "$RUNNER_TEMP/scenes" + cp -R .github/scripts "$RUNNER_TEMP/scripts" + - name: Check out the base + run: | + git checkout --quiet --detach HEAD^ + rm -rf scenes + cp -R "$RUNNER_TEMP/scenes" scenes + - name: Identify Swift toolchain + if: needs.current.outputs.app == 'true' + id: swift + run: echo "version=$({ swift --version; xcodebuild -version; xcrun --show-sdk-version; } | shasum -a 256 | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT" + - uses: actions/cache/restore@v6 + if: needs.current.outputs.app == 'true' + with: + path: .build + key: ${{ runner.os }}-${{ runner.arch }}-release-${{ steps.swift.outputs.version }}-${{ hashFiles('Package.swift', 'Package.resolved') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-release-${{ steps.swift.outputs.version }}-${{ hashFiles('Package.swift', 'Package.resolved') }}- + # When the app didn't change, the pull request's app draws the base too, and the docs images + # are the same. A base that can't draw scenes leaves no scenes.json; the comparison says so. + - name: Draw the base's screens + timeout-minutes: 15 + env: + APP: ${{ needs.current.outputs.app }} + run: | + mkdir -p "$RUNNER_TEMP/before" + if [[ "$APP" == true ]]; then + STABLEBAR_PREBUILT_DIR= ./scripts/build.sh "$RUNNER_TEMP/base.app" + app="$RUNNER_TEMP/base.app/Contents/MacOS/StableBar" + "$app" --render-docs-images "$RUNNER_TEMP/before/docs" + else + app="$RUNNER_TEMP/head.app/Contents/MacOS/StableBar" + cp -R "$RUNNER_TEMP/after/docs" "$RUNNER_TEMP/before/docs" + fi + "$app" --render-scenes "$RUNNER_TEMP/before/scenes" scenes || true + - name: Compare + run: | + compare="$RUNNER_TEMP/scripts/compare-screens.mjs" + node "$compare" "$RUNNER_TEMP/before/docs" "$RUNNER_TEMP/after/docs" "$RUNNER_TEMP/out" --prefix docs + node "$compare" "$RUNNER_TEMP/before/scenes" "$RUNNER_TEMP/after/scenes" "$RUNNER_TEMP/out" --prefix scenes \ + --before-report "$RUNNER_TEMP/before/scenes/scenes.json" --after-report "$RUNNER_TEMP/after/scenes/scenes.json" + - uses: actions/upload-artifact@v7 + with: + name: screens-app + path: ${{ runner.temp }}/out/ + retention-days: 7 + if-no-files-found: error + + # Captures the website's pages from the pull request and from its base, and compares them into + # the screens-site artifact, for screens.yml as above. + site-screens: + name: Draw website screens + needs: current + if: github.event_name == 'pull_request' && needs.current.outputs.site == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 2 + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: 24 + - name: Install Playwright + working-directory: ${{ runner.temp }} + run: | + mkdir playwright + cd playwright + npm init --yes > /dev/null + # One exact version: a newer Chromium can draw text a pixel differently, which a rerun would + # report as a change. Raise it on purpose. + npm install --no-audit --no-fund playwright@1.64.1 + npx playwright install --with-deps chromium + - name: Capture the website + run: | + mkdir "$RUNNER_TEMP/base" + git archive HEAD^ site | tar -x -C "$RUNNER_TEMP/base" + capture="$GITHUB_WORKSPACE/.github/scripts/capture-site.mjs" + cd "$RUNNER_TEMP/playwright" + node "$capture" "$RUNNER_TEMP/base/site" "$RUNNER_TEMP/before" + node "$capture" "$GITHUB_WORKSPACE/site" "$RUNNER_TEMP/after" + - name: Compare + run: node .github/scripts/compare-screens.mjs "$RUNNER_TEMP/before" "$RUNNER_TEMP/after" "$RUNNER_TEMP/out" --prefix site + - uses: actions/upload-artifact@v7 + with: + name: screens-site + path: ${{ runner.temp }}/out/ + retention-days: 7 + if-no-files-found: error + # Every push to main with releasable commits publishes the next version as a beta prerelease, # unless it would ship the same app as the beta before it. It commits nothing. Both jobs build # with github.run_number, which only goes up, so a stable build always numbers higher than the diff --git a/.github/workflows/screens.yml b/.github/workflows/screens.yml new file mode 100644 index 0000000..0aece2c --- /dev/null +++ b/.github/workflows/screens.yml @@ -0,0 +1,99 @@ +name: Screens + +# Posts the before/after screens of a pull request and sets its Visual review status. CI's Draw +# screens and Draw website screens jobs run the pull request's code and upload what they drew. This +# workflow runs the default branch's copy of itself and of .github/scripts, reads those artifacts +# only as data, and holds the write token. Each event reconciles its pull request, one at a time: +# it publishes the newest finished CI run's screens to a monthly pre-release and the comment, sets +# the status from the visual-ok label's history, and deletes an unmerged pull request's images when +# it closes. Running it again for a pull request only brings everything up to date. The logic and +# its tests are in .github/scripts/screens-bot.mjs and screens-comment.mjs. +on: + workflow_run: + workflows: [CI] + types: [completed] + pull_request_target: + types: [labeled, unlabeled, edited, closed, reopened] + +permissions: + actions: read + contents: write + issues: write + pull-requests: write + statuses: write + +jobs: + reconcile: + name: Reconcile screens + if: >- + (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'pull_request') + || (github.event_name == 'pull_request_target' && ( + github.event.action == 'closed' || github.event.action == 'reopened' + || (github.event.action == 'edited' && github.event.changes.base) + || github.event.label.name == 'visual-ok')) + # One reconcile of a pull request at a time. GitHub keeps only the newest event waiting, but + # each reconcile reads everything afresh, so a dropped one loses nothing. + concurrency: + group: screens-${{ github.event.workflow_run.head_repository.full_name || github.event.pull_request.head.repo.full_name }}-${{ github.event.workflow_run.head_branch || github.event.pull_request.head.ref }} + cancel-in-progress: false + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.repository.default_branch }} + sparse-checkout: .github/scripts + persist-credentials: false + - name: Plan + id: plan + uses: actions/github-script@v8 + with: + # A plan that fails marks the head errored, as Apply does, so a required Visual review doesn't wait forever. + script: | + const bot = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); + let chosen; + try { + chosen = await bot.plan({ github, context }); + } catch (error) { + const number = await bot.pullRequestNumber({ github, context }).catch(() => null); + await bot.failRun({ github, context, number }).catch((failure) => core.warning(failure.message)); + throw error; + } + core.info(`Plan: ${chosen.action}${chosen.number ? ` for #${chosen.number}` : ""}${chosen.run ? ` from run ${chosen.run.id}` : ""}`); + core.setOutput("plan", JSON.stringify(chosen)); + core.setOutput("run-id", chosen.run?.id ?? ""); + core.setOutput("app", chosen.run?.artifacts.app ? "true" : ""); + core.setOutput("site", chosen.run?.artifacts.site ? "true" : ""); + # A download that fails leaves its folder missing, which Apply reports. + - uses: actions/download-artifact@v8 + if: steps.plan.outputs.app + continue-on-error: true + with: + name: screens-app + path: ${{ runner.temp }}/screens-app + run-id: ${{ steps.plan.outputs.run-id }} + github-token: ${{ github.token }} + - uses: actions/download-artifact@v8 + if: steps.plan.outputs.site + continue-on-error: true + with: + name: screens-site + path: ${{ runner.temp }}/screens-site + run-id: ${{ steps.plan.outputs.run-id }} + github-token: ${{ github.token }} + - name: Apply + uses: actions/github-script@v8 + env: + PLAN: ${{ steps.plan.outputs.plan }} + with: + script: | + const bot = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); + const chosen = JSON.parse(process.env.PLAN); + const folders = { app: `${process.env.RUNNER_TEMP}/screens-app`, site: `${process.env.RUNNER_TEMP}/screens-site` }; + try { + const result = await bot.apply({ github, context, plan: chosen, folders }); + core.info(result.review ? `Visual review: ${result.review.state}, ${result.review.description}` : JSON.stringify(result)); + } catch (error) { + await bot.failRun({ github, context, number: chosen.number }).catch((failure) => core.warning(failure.message)); + throw error; + } diff --git a/AGENTS.md b/AGENTS.md index a191220..871732b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ The developer's own StableBar is usually running while you work. - Run the tests for what you changed with `swift test --filter `. CI runs the whole suite on macOS for every pull request. - `node --test .github/scripts/*.test.mjs` tests the pull request checks in `.github/scripts`. - `StableBar --validate PATH` checks a configuration without starting its scripts. -- Visible changes (the bar, popups, setup, Settings) need before/after screenshots in the pull request. Ask before launching the app or taking screenshots: a second bar appears on the developer's screen. `StableBar --render-docs-images DIR` draws the bar, popups and Settings in offscreen windows instead, so nothing appears on screen. For a before/after of your change, add or extend a scene in `scenes/` and draw it with `StableBar --render-scenes DIR scenes`, from a build of `main` and of your branch: it runs nothing and draws offscreen (see [Scenes](CONTRIBUTING.md#scenes)). +- Visible changes (the bar, popups, setup, Settings, the website) need before/after images in the pull request, and CI draws them: the screens bot comments with the docs images, the scenes in `scenes/` and the website's pages, drawn from `main` and from your branch (see [Before and after screens](CONTRIBUTING.md#before-and-after-screens)). When no screen shows your change, add or extend a scene so the bot draws it, instead of uploading screenshots. To look before pushing, `StableBar --render-scenes DIR scenes` draws the scenes offscreen from a build, running nothing. Ask before launching the app or taking screenshots of it: a second bar appears on the developer's screen. Add a recording when motion matters. ## Hit every surface diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f6ffe64..4cb1077 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,9 +77,9 @@ There is no web view, custom drawing loop or private window-server API. Styling ## How we work 1. **Start from an issue** for every `fix`, `feat` or `perf` change, using the issue forms, and link it from the pull request. The [triage skill](#agent-skills) checks whether a report is already tracked, fixed or supported. Other changes, such as CI, tests or docs, can go straight to a pull request. -2. **Branch from `main` and open a pull request.** Nobody pushes to `main` directly. Fill in the template: the problem, the changes, how you tested them and how to merge, and end with the model and harness that did the work, or with "No agent". Upload screenshots of visible changes to the pull request instead of committing them. +2. **Branch from `main` and open a pull request.** Nobody pushes to `main` directly. Fill in the template: the problem, the changes, how you tested them and how to merge, and end with the model and harness that did the work, or with "No agent". For visible changes, the screens bot posts before and after images ([Before and after screens](#before-and-after-screens)); upload a recording when motion matters, instead of committing it. 3. **Keep one problem per pull request.** Unrelated fixes and cleanups get their own. -4. **Wait for CI.** `Test` and `Commit messages` must pass. `Linked issue` and `Description` check the pull request itself and rerun when you edit its description: a `fix`, `feat` or `perf` pull request closes an issue ("Fixes #N" in the description), and Problem, Changes and Testing are filled in. Depending on what a pull request changes, `Build release`, `Docs images` and `Package` run too ([Commits and releases](#commits-and-releases) lists when). Fix them before merging: the same failure on `main` stops the next beta. +4. **Wait for CI.** `Test` and `Commit messages` must pass. `Linked issue` and `Description` check the pull request itself and rerun when you edit its description: a `fix`, `feat` or `perf` pull request closes an issue ("Fixes #N" in the description), and Problem, Changes and Testing are filled in. Depending on what a pull request changes, `Build release`, `Docs images`, `Package`, `Draw screens` and `Draw website screens` run too ([Commits and releases](#commits-and-releases) lists when). Fix them before merging: the same failure on `main` stops the next beta. When screens changed, `Visual review` waits until a maintainer has looked at them and added the `visual-ok` label. 5. **Merge** with a merge commit or a rebase merge, so semantic-release sees each commit. Squash only when the title alone should reach the changelog. Reviews are requested automatically from the people listed in [CODEOWNERS](.github/CODEOWNERS), but they are not required. Ask for one when a change is risky or outside your usual area. @@ -88,9 +88,9 @@ Reviews are requested automatically from the people listed in [CODEOWNERS](.gith ## Commits and releases -Pull requests run `swift test` on macOS and check that every commit message, and the pull request title, follows [Conventional Commits](https://www.conventionalcommits.org). Squash merges use the title as the commit message. The tests load every configuration, module and package in `examples`, and the **Test** job parses each script in `scripts` and `examples` with the shell its first line names. The **Test pull request checks** job runs `node --test` on the logic of `Linked issue` and `Description`, in `.github/scripts`. +Pull requests run `swift test` on macOS and check that every commit message, and the pull request title, follows [Conventional Commits](https://www.conventionalcommits.org). Squash merges use the title as the commit message. The tests load every configuration, module and package in `examples`, and the **Test** job parses each script in `scripts` and `examples` with the shell its first line names. The **Test pull request checks** job runs `node --test` on the scripts in `.github/scripts`: the logic of `Linked issue`, `Description` and the [before and after screens](#before-and-after-screens). -A pull request that changes `Sources`, `Resources`, `Package.swift`, `Package.resolved`, `scripts` or the CI workflow also compiles the release and draws the [docs images](#docs-images), as `main` does. If it changes `Resources`, `scripts` or the workflow, the **Package** job builds the DMG as a beta would, ad-hoc signed because pull requests get no Developer ID, and checks its signature, version and starter configuration. +A pull request that changes `Sources`, `Resources`, `Package.swift`, `Package.resolved`, `scripts` or the CI workflow also compiles the release and draws the [docs images](#docs-images), as `main` does. If it changes `Resources`, `scripts` or the workflow, the **Package** job builds the DMG as a beta would, ad-hoc signed because pull requests get no Developer ID, and checks its signature, version and starter configuration. **Draw screens** runs when the app, `scenes`, the `examples` scenes read or the comparison scripts change, and **Draw website screens** when `site` changes. [semantic-release](https://github.com/semantic-release/semantic-release) reads the commits since the last stable release and decides the next version: @@ -139,7 +139,35 @@ StableBar --render-scenes /tmp/scenes scenes It writes `-dark.png` and `-light.png` for each scene, and `scenes.json` with each scene's images or the error that stopped it. It runs nothing (no command, provider, window manager or network request), and every clock shows Thursday 22 October 2026, 09:41 UTC, so two runs of one commit on one Mac draw the same pixels. Text about the Mac itself, such as a display's name or the window managers installed, still comes from the Mac that draws. -A change you can see extends the scene that shows its area, or adds one when none does. Scenes can be edited, merged or deleted. Keep the format additive: a build reports a scene it can't read as an error and draws the others. +A change you can see extends the scene that shows its area, or adds one when none does. Scenes can be edited, merged or deleted. Keep the format additive: a build reports a scene it can't read as an error and draws the others. Name scene files with letters, digits, dots, dashes and underscores, so the screens bot can show them. + +## Before and after screens + +On a pull request, CI draws the screens twice, from the pull request and from its base, and a bot posts what changed. + +- **Draw screens** draws the [docs images](#docs-images) and the [scenes](#scenes) when the app, the scenes, the examples they read or the comparison scripts change. Both sides draw the pull request's scene files, each beside its own resources and examples, so a new scene shows how `main` draws it too. When the app didn't change, the pull request's app draws both sides. +- **Draw website screens** captures the four pages of `site` at desktop (1440×900) and phone (390×844) size, in light and dark mode, when the website changes. The capture answers requests from the folder and fetches nothing else, seeds `Math.random` and stops the page's clock at the scenes' date, so two captures of one commit match. + +Both sides are drawn in one job on one machine. A pixel counts as changed when a color channel moves by more than 1 and a pixel next to it changed too. A screen whose height changed, such as a website page that grew, is compared from the top: its crop starts at the first change and runs across the full width. A crop shows at most 1,600 rows of a change, and the comment says so when it stops short; the approval still covers the whole image. + +These jobs run the pull request's code with a read-only token. The **Screens** workflow runs from `main` when CI finishes, and when a pull request gets or loses `visual-ok`, changes its base, closes or reopens. Each time it reads where the pull request stands, one pull request at a time, so running it again only brings everything up to date. It reads what the jobs uploaded only as data, and: + +- comments once on the pull request and rewrites that comment for each new CI run: a table of the changed screens, their before, after and difference images cropped to the change, the unchanged screens folded, and what couldn't be drawn; +- keeps the images as assets of a monthly prerelease, `pr-previews-YYYY-MM`, then `-2` and on when one is full, and deletes a head's images once a newer head replaces them. Closing a pull request without merging deletes all of its images; a merged one keeps its last; +- sets the **Visual review** status on the pull request's head. + +| Visual review | When | +| --- | --- | +| Success: "No screens to compare." | The pull request changes neither the app nor the website. | +| Success: "N screens, none changed." | Nothing looks different. | +| Pending | Screens changed. A maintainer looks at them, then adds the `visual-ok` label. | +| Success: "M changed screens approved." | `visual-ok` was added after the comment began showing these screens. | +| Pending: "The base changed…" | The pull request's base changed since its screens were drawn. Push a commit to draw them again. | +| Failure | Drawing failed; the comment says what. A `visual-ok` added while it showed the failure still approves it. | + +`visual-ok` approves the screens the comment showed when the label was added, so reload the pull request first if a push just landed. Any maintainer can add it, the author too. A later run that shows the same screens against the same base keeps the approval. One that shows other screens, or other failures, takes the label off and the review goes back to pending: look again, and add it again. Removing the label also sends the review back to pending. + +Pull requests from forks get the same comment and status: the drawing runs in the fork's pull request, and **Screens** finds the pull request by its branch and head. Two pull requests from one branch at one commit, open or closed, share CI runs the bot can't tell apart, so it compares no screens and fails the review: close the other one if it's open, then push a commit. The comparison runs in the pull request's own CI, so a pull request that changes the CI workflow or the drawing scripts decides what its comparison shows. **Screens** reads the pull request's changed files from GitHub, so screens they call for that weren't drawn fail the review, but review such a change itself before you trust its screens. ## Agent skills