From 6cb66ca5239087b4f3240e3fa8da87953594bc31 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 15:39:17 +0100 Subject: [PATCH 01/25] ci: compare screens drawn from a pull request and its base compare-screens.mjs compares every PNG of two folders, the base's and the pull request's. A pixel changes when a channel moves by more than one step, and a changed pixel with no changed neighbour is noise. For each changed screen it writes the before, after and difference, cropped to the changed region, and adds the screens to manifest.json, with a note for a scene the base couldn't draw. png.mjs reads and writes 8-bit RGB and RGBA PNGs with no dependencies. --- .github/scripts/compare-screens.mjs | 174 +++++++++++++++++++++++ .github/scripts/compare-screens.test.mjs | 57 ++++++++ .github/scripts/png.mjs | 88 ++++++++++++ .github/scripts/png.test.mjs | 64 +++++++++ 4 files changed, 383 insertions(+) create mode 100644 .github/scripts/compare-screens.mjs create mode 100644 .github/scripts/compare-screens.test.mjs create mode 100644 .github/scripts/png.mjs create mode 100644 .github/scripts/png.test.mjs diff --git a/.github/scripts/compare-screens.mjs b/.github/scripts/compare-screens.mjs new file mode 100644 index 0000000..ff4b7f7 --- /dev/null +++ b/.github/scripts/compare-screens.mjs @@ -0,0 +1,174 @@ +// Compares the screens drawn from a pull request (after) with those drawn from its base (before): +// node compare-screens.mjs BEFORE AFTER OUT --prefix docs [--before-report scenes.json] [--after-report scenes.json] +// It writes cropped before, after and difference images of each changed screen to OUT, and adds the +// screens to OUT/manifest.json: {screens: [{id, status, changedPixels, region, images, note}], failed: [{id, error}]}. +import { existsSync, readFileSync, readdirSync, writeFileSync, mkdirSync } from "node:fs"; +import { join, basename } from "node:path"; +import { pathToFileURL } from "node:url"; +import { decodePNG, encodePNG } from "./png.mjs"; + +/** A pixel changes when one of its channels differs by more than this. */ +const noise = 1; +/** Points of context around a changed region, at the renders' 2× scale. */ +const padding = 24; + +function changedMask(before, after) { + const mask = new Uint8Array(before.width * before.height); + for (let index = 0; index < mask.length; index++) { + for (let channel = 0; channel < 4; channel++) { + if (Math.abs(before.rgba[index * 4 + channel] - after.rgba[index * 4 + channel]) > noise) { + mask[index] = 1; + break; + } + } + } + return mask; +} + +/** + * How `after` differs from `before`, either of which may be null. A changed pixel with no changed + * neighbour is noise: it neither makes a screen changed nor widens its region. + */ +export function comparePair(before, after) { + if (!before) return { status: "added", changedPixels: 0, region: null }; + if (!after) return { status: "removed", changedPixels: 0, region: null }; + if (before.width !== after.width || before.height !== after.height) return { status: "resized", changedPixels: 0, region: null }; + const { width, height } = before; + const mask = changedMask(before, after); + let changedPixels = 0, minX = width, minY = height, maxX = -1, maxY = -1; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + if (!mask[y * width + x]) continue; + let neighbours = 0; + for (let dy = -1; dy <= 1 && !neighbours; dy++) { + for (let dx = -1; dx <= 1; dx++) { + const nx = x + dx, ny = y + dy; + if ((dx || dy) && nx >= 0 && ny >= 0 && nx < width && ny < height && mask[ny * width + nx]) { + neighbours++; + break; + } + } + } + if (!neighbours) continue; + changedPixels++; + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + } + if (!changedPixels) return { status: "unchanged", changedPixels: 0, region: null }; + return { status: "changed", changedPixels, region: { x: minX, y: minY, width: maxX - minX + 1, height: maxY - minY + 1 } }; +} + +function padded(region, image) { + const x = Math.max(0, region.x - padding), y = Math.max(0, region.y - padding); + return { + x, y, + width: Math.min(image.width, region.x + region.width + padding) - x, + height: Math.min(image.height, region.y + region.height + padding) - y, + }; +} + +/** The part of `image` inside `region`. */ +export function crop(image, region) { + const rgba = new Uint8Array(region.width * region.height * 4); + for (let row = 0; row < region.height; row++) { + const start = ((region.y + row) * image.width + region.x) * 4; + rgba.set(image.rgba.subarray(start, start + region.width * 4), row * region.width * 4); + } + return { width: region.width, height: region.height, rgba }; +} + +/** `after` at 30% brightness with every changed pixel red, inside `region`. */ +export function differenceImage(before, after, region) { + const mask = changedMask(before, after); + const rgba = new Uint8Array(after.rgba.length); + for (let index = 0; index < mask.length; index++) { + if (mask[index]) { + rgba.set([255, 59, 48, 255], index * 4); + } else { + rgba.set([after.rgba[index * 4] * 0.3, after.rgba[index * 4 + 1] * 0.3, after.rgba[index * 4 + 2] * 0.3, 255], index * 4); + } + } + return crop({ width: after.width, height: after.height, rgba }, region); +} + +function read(folder, name) { + const path = join(folder, name); + return existsSync(path) ? decodePNG(readFileSync(path)) : null; +} + +/** The scene name an image of `name-.png` belongs to. */ +function sceneName(image) { + return basename(image, ".png").replace(/-(dark|light)$/, ""); +} + +/** + * Compares every PNG in `beforeDir` and `afterDir`, writes the images of changed, added, removed and + * resized screens to `outDir`, and returns the screens. `beforeErrors` holds, by scene name, why + * the base couldn't draw a scene. + */ +export function compareDirectories(beforeDir, afterDir, outDir, { prefix, beforeErrors = {} }) { + const list = (folder) => (existsSync(folder) ? readdirSync(folder).filter((name) => name.endsWith(".png")) : []); + const names = [...new Set([...list(beforeDir), ...list(afterDir)])].sort(); + mkdirSync(outDir, { recursive: true }); + return names.map((name) => { + const before = read(beforeDir, name), after = read(afterDir, name); + const id = `${prefix}/${basename(name, ".png")}`; + const result = comparePair(before, after); + const file = (kind) => `${id.replaceAll("/", "~")}-${kind}.png`; + const images = {}; + const save = (kind, image) => { + writeFileSync(join(outDir, file(kind)), encodePNG(image)); + images[kind] = file(kind); + }; + if (result.status === "changed") { + const region = padded(result.region, after); + save("before", crop(before, region)); + save("after", crop(after, region)); + save("diff", differenceImage(before, after, region)); + } else if (result.status === "resized") { + save("before", before); + save("after", after); + } else if (result.status === "added") { + save("after", after); + } else if (result.status === "removed") { + save("before", before); + } + const screen = { id, ...result, images }; + const error = beforeErrors[sceneName(name)]; + if (result.status === "added" && error) screen.note = `The base couldn't draw this scene: ${error}`; + return screen; + }); +} + +/** Scene name to error, from a scenes.json report `--render-scenes` wrote. */ +function errors(report) { + if (!report || !existsSync(report)) return {}; + const scenes = JSON.parse(readFileSync(report, "utf8")).scenes ?? {}; + return Object.fromEntries(Object.entries(scenes).filter(([, entry]) => entry.error).map(([name, entry]) => [name, entry.error])); +} + +export function main(argv) { + const [beforeDir, afterDir, outDir, ...options] = argv; + const option = (name) => { + const index = options.indexOf(name); + return index >= 0 ? options[index + 1] : undefined; + }; + const prefix = option("--prefix"); + if (!beforeDir || !afterDir || !outDir || !prefix) { + throw new Error("usage: compare-screens.mjs BEFORE AFTER OUT --prefix NAME [--before-report FILE] [--after-report FILE]"); + } + const screens = compareDirectories(beforeDir, afterDir, outDir, { prefix, beforeErrors: errors(option("--before-report")) }); + const failed = Object.entries(errors(option("--after-report"))).map(([name, error]) => ({ id: `${prefix}/${name}`, error })); + const manifestPath = join(outDir, "manifest.json"); + const manifest = existsSync(manifestPath) ? JSON.parse(readFileSync(manifestPath, "utf8")) : { screens: [], failed: [] }; + manifest.screens.push(...screens); + manifest.failed.push(...failed); + writeFileSync(manifestPath, JSON.stringify(manifest, null, 2)); + const changed = screens.filter((screen) => screen.status !== "unchanged").length; + console.log(`${prefix}: ${screens.length} screens, ${changed} changed, ${failed.length} failed`); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) main(process.argv.slice(2)); diff --git a/.github/scripts/compare-screens.test.mjs b/.github/scripts/compare-screens.test.mjs new file mode 100644 index 0000000..137caa8 --- /dev/null +++ b/.github/scripts/compare-screens.test.mjs @@ -0,0 +1,57 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, readdirSync, writeFileSync, mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { comparePair, compareDirectories } from "./compare-screens.mjs"; +import { decodePNG, encodePNG } from "./png.mjs"; + +function image(width, height, rgba = [40, 40, 40, 255]) { + const pixels = new Uint8Array(width * height * 4); + for (let index = 0; index < width * height; index++) pixels.set(rgba, index * 4); + return { width, height, rgba: pixels }; +} + +function paint(target, x, y, width, height, rgba) { + const copy = { ...target, rgba: Uint8Array.from(target.rgba) }; + for (let row = y; row < y + height; row++) { + for (let column = x; column < x + width; column++) copy.rgba.set(rgba, (row * copy.width + column) * 4); + } + return copy; +} + +test("comparePair finds the changed region and ignores noise", () => { + const base = image(20, 10); + assert.deepEqual(comparePair(base, base), { status: "unchanged", changedPixels: 0, region: null }); + assert.deepEqual(comparePair(base, paint(base, 5, 2, 3, 3, [200, 40, 40, 255])), + { status: "changed", changedPixels: 9, region: { x: 5, y: 2, width: 3, height: 3 } }); + assert.equal(comparePair(base, paint(base, 12, 6, 1, 1, [200, 40, 40, 255])).status, "unchanged"); + assert.equal(comparePair(base, paint(base, 0, 0, 20, 10, [41, 41, 41, 255])).status, "unchanged"); + assert.equal(comparePair(base, image(21, 10)).status, "resized"); + assert.equal(comparePair(null, base).status, "added"); + assert.equal(comparePair(base, null).status, "removed"); +}); + +test("compareDirectories writes images for changed screens only, and notes what the base couldn't draw", () => { + const root = mkdtempSync(join(tmpdir(), "screens-")); + const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); + for (const folder of [before, after, out]) mkdirSync(folder, { recursive: true }); + const base = image(60, 30); + writeFileSync(join(before, "same-dark.png"), encodePNG(base)); + writeFileSync(join(after, "same-dark.png"), encodePNG(base)); + writeFileSync(join(before, "moved-dark.png"), encodePNG(base)); + writeFileSync(join(after, "moved-dark.png"), encodePNG(paint(base, 10, 10, 4, 4, [255, 255, 255, 255]))); + writeFileSync(join(after, "new-dark.png"), encodePNG(base)); + const screens = compareDirectories(before, after, out, { prefix: "scenes", beforeErrors: { new: "draw: Unknown value" } }); + assert.deepEqual(screens.map((screen) => [screen.id, screen.status]), + [["scenes/moved-dark", "changed"], ["scenes/new-dark", "added"], ["scenes/same-dark", "unchanged"]]); + const moved = screens[0]; + assert.deepEqual(Object.keys(moved.images).sort(), ["after", "before", "diff"]); + assert.deepEqual(readdirSync(out).sort(), + ["scenes~moved-dark-after.png", "scenes~moved-dark-before.png", "scenes~moved-dark-diff.png", "scenes~new-dark-after.png"]); + const crop = decodePNG(readFileSync(join(out, moved.images.after))); + // The 4-pixel square at x = 10 gets 24 pixels of padding, cut at the left edge: from 0 to 10 + 4 + 24. + assert.equal(crop.width, 38); + assert.match(screens[1].note, /Unknown value/); + assert.equal(screens[2].images.after, undefined); +}); diff --git a/.github/scripts/png.mjs b/.github/scripts/png.mjs new file mode 100644 index 0000000..8d332e6 --- /dev/null +++ b/.github/scripts/png.mjs @@ -0,0 +1,88 @@ +// Reads and writes the PNG files the before/after screens compare: 8-bit RGB or RGBA without +// interlacing, which is what StableBar's renders and headless Chrome write. No dependencies. +import zlib from "node:zlib"; + +const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); + +function paeth(a, b, c) { + const p = a + b - c, pa = Math.abs(p - a), pb = Math.abs(p - b), pc = Math.abs(p - c); + return pa <= pb && pa <= pc ? a : pb <= pc ? b : c; +} + +function chunk(type, data) { + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const body = Buffer.concat([Buffer.from(type, "latin1"), data]); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(zlib.crc32(body)); + return Buffer.concat([length, body, crc]); +} + +/** The pixels of a PNG as `{width, height, rgba}`, every pixel four bytes. */ +export function decodePNG(input) { + const buffer = Buffer.isBuffer(input) ? input : Buffer.from(input); + if (buffer.length < 8 || !buffer.subarray(0, 8).equals(signature)) throw new Error("Unsupported PNG: no PNG signature."); + let offset = 8, width = 0, height = 0, bitDepth = 0, colorType = -1, interlace = -1; + const data = []; + while (offset + 8 <= buffer.length) { + const length = buffer.readUInt32BE(offset); + const type = buffer.toString("latin1", offset + 4, offset + 8); + const body = buffer.subarray(offset + 8, offset + 8 + length); + if (type === "IHDR") { + width = body.readUInt32BE(0); + height = body.readUInt32BE(4); + bitDepth = body[8]; + colorType = body[9]; + interlace = body[12]; + } else if (type === "IDAT") { + data.push(body); + } else if (type === "IEND") { + break; + } + offset += 12 + length; + } + if (!width || !height || bitDepth !== 8 || (colorType !== 2 && colorType !== 6) || interlace !== 0) { + throw new Error("Unsupported PNG: only 8-bit RGB or RGBA without interlacing."); + } + const channels = colorType === 6 ? 4 : 3; + const stride = width * channels; + const raw = zlib.inflateSync(Buffer.concat(data)); + if (raw.length < height * (stride + 1)) throw new Error("Unsupported PNG: the image data is cut short."); + const pixels = new Uint8Array(height * stride); + for (let y = 0; y < height; y++) { + const filter = raw[y * (stride + 1)]; + if (filter > 4) throw new Error(`Unsupported PNG: unknown filter ${filter}.`); + const line = y * (stride + 1) + 1; + for (let x = 0; x < stride; x++) { + const a = x >= channels ? pixels[y * stride + x - channels] : 0; + const b = y > 0 ? pixels[(y - 1) * stride + x] : 0; + const c = x >= channels && y > 0 ? pixels[(y - 1) * stride + x - channels] : 0; + const predicted = filter === 0 ? 0 : filter === 1 ? a : filter === 2 ? b : filter === 3 ? (a + b) >> 1 : paeth(a, b, c); + pixels[y * stride + x] = (raw[line + x] + predicted) & 255; + } + } + if (channels === 4) return { width, height, rgba: pixels }; + const rgba = new Uint8Array(width * height * 4); + for (let index = 0; index < width * height; index++) { + rgba[index * 4] = pixels[index * 3]; + rgba[index * 4 + 1] = pixels[index * 3 + 1]; + rgba[index * 4 + 2] = pixels[index * 3 + 2]; + rgba[index * 4 + 3] = 255; + } + return { width, height, rgba }; +} + +/** An RGBA PNG of `{width, height, rgba}`. */ +export function encodePNG({ width, height, rgba }) { + const stride = width * 4; + const raw = Buffer.alloc(height * (stride + 1)); + for (let y = 0; y < height; y++) { + Buffer.from(rgba.buffer, rgba.byteOffset + y * stride, stride).copy(raw, y * (stride + 1) + 1); + } + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; + header[9] = 6; + return Buffer.concat([signature, chunk("IHDR", header), chunk("IDAT", zlib.deflateSync(raw)), chunk("IEND", Buffer.alloc(0))]); +} diff --git a/.github/scripts/png.test.mjs b/.github/scripts/png.test.mjs new file mode 100644 index 0000000..c8cacf3 --- /dev/null +++ b/.github/scripts/png.test.mjs @@ -0,0 +1,64 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import zlib from "node:zlib"; +import { decodePNG, encodePNG } from "./png.mjs"; + +function chunk(type, data) { + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const body = Buffer.concat([Buffer.from(type, "latin1"), data]); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(zlib.crc32(body)); + return Buffer.concat([length, body, crc]); +} + +function paeth(a, b, c) { + const p = a + b - c, pa = Math.abs(p - a), pb = Math.abs(p - b), pc = Math.abs(p - c); + return pa <= pb && pa <= pc ? a : pb <= pc ? b : c; +} + +/** An 8-bit RGB PNG whose every scanline uses `filter`. */ +function rgbPNG(width, height, rgb, filter) { + const stride = width * 3; + const raw = []; + for (let y = 0; y < height; y++) { + raw.push(filter); + for (let x = 0; x < stride; x++) { + const a = x >= 3 ? rgb[y * stride + x - 3] : 0; + const b = y > 0 ? rgb[(y - 1) * stride + x] : 0; + const c = x >= 3 && y > 0 ? rgb[(y - 1) * stride + x - 3] : 0; + const predicted = [0, a, b, (a + b) >> 1, paeth(a, b, c)][filter]; + raw.push((rgb[y * stride + x] - predicted) & 255); + } + } + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; + header[9] = 2; + return Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), chunk("IHDR", header), + chunk("IDAT", zlib.deflateSync(Buffer.from(raw))), chunk("IEND", Buffer.alloc(0))]); +} + +test("encodePNG and decodePNG round-trip RGBA pixels", () => { + const rgba = Uint8Array.from([255, 0, 0, 255, 0, 255, 0, 128, 0, 0, 255, 0, 9, 8, 7, 6, 1, 2, 3, 4, 250, 251, 252, 253]); + const decoded = decodePNG(encodePNG({ width: 3, height: 2, rgba })); + assert.equal(decoded.width, 3); + assert.equal(decoded.height, 2); + assert.deepEqual([...decoded.rgba], [...rgba]); +}); + +test("decodePNG reads RGB scanlines with every filter as opaque RGBA", () => { + const width = 4, height = 3; + const rgb = Array.from({ length: width * height * 3 }, (_, index) => (index * 37 + 11) % 256); + const expected = []; + for (let pixel = 0; pixel < width * height; pixel++) expected.push(...rgb.slice(pixel * 3, pixel * 3 + 3), 255); + for (const filter of [0, 1, 2, 3, 4]) { + const decoded = decodePNG(rgbPNG(width, height, rgb, filter)); + assert.deepEqual([...decoded.rgba], expected, `filter ${filter}`); + } +}); + +test("decodePNG rejects what isn't an 8-bit RGB or RGBA PNG", () => { + assert.throws(() => decodePNG(Buffer.from("not a png")), /Unsupported PNG/); +}); From f5695c7f1cb8ab4ec25c1bfddc0a6cf57dda53fc Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 15:40:40 +0100 Subject: [PATCH 02/25] ci: write the screens comment and decide the visual review screens-comment.mjs checks an artifact's files (only PNG images and the manifest, each under 10 MB, at most 300), fingerprints the changed screens from their after images, writes the comment with the changed screens and the unchanged ones folded, and decides the Visual review status: success when nothing needs a look or visual-ok approved these exact screens, pending while changed screens wait, failure when drawing failed. Descriptions fit GitHub's 140 characters. --- .github/scripts/screens-comment.mjs | 125 +++++++++++++++++++++++ .github/scripts/screens-comment.test.mjs | 68 ++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 .github/scripts/screens-comment.mjs create mode 100644 .github/scripts/screens-comment.test.mjs diff --git a/.github/scripts/screens-comment.mjs b/.github/scripts/screens-comment.mjs new file mode 100644 index 0000000..bcc44ca --- /dev/null +++ b/.github/scripts/screens-comment.mjs @@ -0,0 +1,125 @@ +// The logic of the before/after screens comment and its Visual review status, for screens.yml. The +// workflow runs the default branch's copy and reads the pull request's artifact only as data. +import { createHash } from "node:crypto"; + +const marker = ""; +const dataPattern = //s; +const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); +const maximumFileSize = 10 * 1024 * 1024; +const maximumFiles = 300; + +/** Checks an artifact's files before anything is uploaded: only the manifest and PNG images, small enough. */ +export function validateArtifact(files) { + const problems = []; + if (files.length > maximumFiles) problems.push(`${files.length} files, more than ${maximumFiles}.`); + for (const file of files) { + if (file.name === "manifest.json") continue; + if (!/^[A-Za-z0-9][A-Za-z0-9._~-]*\.png$/.test(file.name) || file.name.includes("..")) { + problems.push(`${file.name}: not an image name this bot writes.`); + } else if (!Buffer.from(file.head ?? []).subarray(0, 8).equals(signature)) { + problems.push(`${file.name}: not a PNG.`); + } else if (file.size > maximumFileSize) { + problems.push(`${file.name}: larger than 10 MB.`); + } + } + return { ok: problems.length === 0, problems }; +} + +/** One value for the changed screens and their after images, whatever their order. */ +export function fingerprint(changed) { + const lines = changed.map(({ id, afterSha256 }) => `${id} ${afterSha256}`).sort(); + return createHash("sha256").update(lines.join("\n")).digest("hex"); +} + +const plural = (count, word) => `${count} ${word}${count === 1 ? "" : "s"}`; +/** A commit status description holds at most 140 characters. */ +const fit = (text) => (text.length <= 140 ? text : `${text.slice(0, 139)}…`); + +/** + * The Visual review status: success when nothing needs a look or a maintainer approved these exact + * screens with visual-ok, pending while changed screens wait for one, failure when drawing failed. + * `removeLabel` asks to take visual-ok off when it approved other screens than these. + */ +export function reviewState({ ranScreens, screens = [], failed = [], labels = [], approved, fingerprint: current }) { + if (!ranScreens) return { state: "success", description: "No screens to compare.", removeLabel: false }; + const changed = screens.filter((screen) => screen.status !== "unchanged"); + const labelled = labels.includes("visual-ok"); + const valid = labelled && approved != null && approved === current; + if (failed.length > 0) { + if (valid) return { state: "success", description: fit(`Approved with visual-ok although ${plural(failed.length, "screen")} failed to draw.`), removeLabel: false }; + return { state: "failure", description: fit(`Drawing the screens failed: ${failed[0].id}: ${failed[0].error}`), removeLabel: labelled }; + } + if (changed.length === 0) return { state: "success", description: `${plural(screens.length, "screen")}, none changed.`, removeLabel: false }; + if (valid) return { state: "success", description: `${plural(changed.length, "changed screen")} approved.`, removeLabel: false }; + return { state: "pending", description: `${plural(changed.length, "screen")} changed: a maintainer adds visual-ok after looking.`, + removeLabel: labelled }; +} + +function describe(screen) { + switch (screen.status) { + case "changed": { + const { x, y, width, height } = screen.region; + return [`${screen.changedPixels.toLocaleString("en-US")} pixels`, `${x},${y} · ${width}×${height}`]; + } + case "added": return ["new", "the whole screen"]; + case "removed": return ["removed", "the whole screen"]; + case "resized": return ["new size", "the whole screen"]; + default: return ["", ""]; + } +} + +/** The comment: changed screens with their images, unchanged ones folded, failures, the status and its data. */ +export function commentBody({ base, head, screens, failed = [], imageURL, review, data }) { + const changed = screens.filter((screen) => screen.status !== "unchanged"); + const unchanged = screens.filter((screen) => screen.status === "unchanged"); + const image = (file, label) => `${label}`; + const lines = [marker, "### Before and after", "", + `\`main\` at ${base.slice(0, 7)} → this pull request at ${head.slice(0, 7)} · ${plural(screens.length, "screen")} drawn in one CI job · ${changed.length} changed`, ""]; + if (changed.length > 0) { + lines.push("| Screen | Change | Where (pixels at 2×) |", "| --- | --- | --- |"); + for (const screen of changed) lines.push(`| \`${screen.id}\` | ${describe(screen).join(" | ")} |`); + lines.push(""); + for (const screen of changed) { + lines.push(`#### \`${screen.id}\``); + if (screen.note) lines.push("", screen.note); + const { before, after, diff } = screen.images; + if (before && after) { + lines.push("", "| Before | After |", "| --- | --- |", `| ${image(before, "Before")} | ${image(after, "After")} |`); + } else if (after) { + lines.push("", image(after, "After")); + } else if (before) { + lines.push("", image(before, "Before")); + } + if (diff) lines.push("", `Difference, changed pixels in red:`, "", image(diff, "Difference")); + lines.push(""); + } + } + if (unchanged.length > 0) { + lines.push(`
${plural(unchanged.length, "unchanged screen")}`, "", + unchanged.map((screen) => `\`${screen.id}\``).join(", "), "", "
", ""); + } + if (failed.length > 0) { + lines.push("**Couldn't draw:**", "", ...failed.map((entry) => `- \`${entry.id}\`: ${entry.error}`), ""); + } + lines.push(`**Visual review:** ${review.description}`, "", ``); + return lines.join("\n"); +} + +/** The data a comment from commentBody carries, or null. */ +export function readData(body) { + const match = dataPattern.exec(body ?? ""); + if (!match) return null; + try { + return JSON.parse(match[1]); + } catch { + return null; + } +} + +export const isScreensComment = (body) => (body ?? "").startsWith(marker); + +/** The name of an uploaded image: the pull request and head commit, then the file. */ +export const assetName = (pr, sha, file) => `pr${pr}-${sha.slice(0, 7)}-${file}`; + +/** The pre-release that holds a month's images. */ +export const releaseTag = (date) => `pr-previews-${date.toISOString().slice(0, 7)}`; diff --git a/.github/scripts/screens-comment.test.mjs b/.github/scripts/screens-comment.test.mjs new file mode 100644 index 0000000..3f466b2 --- /dev/null +++ b/.github/scripts/screens-comment.test.mjs @@ -0,0 +1,68 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { validateArtifact, fingerprint, reviewState, commentBody, readData, assetName, releaseTag } from "./screens-comment.mjs"; + +const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); +const screen = (id, status, extra = {}) => ({ id, status, changedPixels: status === "changed" ? 120 : 0, + region: status === "changed" ? { x: 10, y: 20, width: 30, height: 40 } : null, images: {}, ...extra }); + +test("validateArtifact accepts only PNG images and the manifest", () => { + const good = [{ name: "manifest.json", size: 100, head: Buffer.from("{") }, { name: "docs~hero-dark-after.png", size: 2048, head: png }]; + assert.deepEqual(validateArtifact(good), { ok: true, problems: [] }); + for (const bad of [{ name: "../x.png", size: 10, head: png }, { name: "a.txt", size: 10, head: png }, + { name: "fake.png", size: 10, head: Buffer.from("GIF89a..") }, { name: "big.png", size: 11 * 1024 * 1024, head: png }]) { + assert.equal(validateArtifact([...good, bad]).ok, false, bad.name); + } + const many = Array.from({ length: 301 }, (_, index) => ({ name: `s${index}.png`, size: 10, head: png })); + assert.equal(validateArtifact(many).ok, false); +}); + +test("fingerprint ignores order and follows the after images", () => { + const first = fingerprint([{ id: "docs/a", afterSha256: "11" }, { id: "docs/b", afterSha256: "22" }]); + assert.equal(first, fingerprint([{ id: "docs/b", afterSha256: "22" }, { id: "docs/a", afterSha256: "11" }])); + assert.notEqual(first, fingerprint([{ id: "docs/a", afterSha256: "11" }, { id: "docs/b", afterSha256: "23" }])); +}); + +test("reviewState follows the visual review rules", () => { + const changed = [screen("docs/a", "changed"), screen("docs/b", "unchanged")]; + assert.deepEqual(reviewState({ ranScreens: false }), { state: "success", description: "No screens to compare.", removeLabel: false }); + assert.deepEqual(reviewState({ ranScreens: true, screens: [screen("docs/a", "unchanged"), screen("docs/b", "unchanged")], failed: [], labels: [] }), + { state: "success", description: "2 screens, none changed.", removeLabel: false }); + const waiting = reviewState({ ranScreens: true, screens: changed, failed: [], labels: [], fingerprint: "f1" }); + assert.equal(waiting.state, "pending"); + assert.equal(waiting.description, "1 screen changed: a maintainer adds visual-ok after looking."); + assert.deepEqual(reviewState({ ranScreens: true, screens: changed, failed: [], labels: ["visual-ok"], approved: "f1", fingerprint: "f1" }), + { state: "success", description: "1 changed screen approved.", removeLabel: false }); + const stale = reviewState({ ranScreens: true, screens: changed, failed: [], labels: ["visual-ok"], approved: "f0", fingerprint: "f1" }); + assert.equal(stale.state, "pending"); + assert.equal(stale.removeLabel, true); + const failed = [{ id: "scenes/x", error: "draw: Unknown value " + "y".repeat(200) }]; + const broken = reviewState({ ranScreens: true, screens: changed, failed, labels: [], fingerprint: "f1" }); + assert.equal(broken.state, "failure"); + assert.match(broken.description, /^Drawing the screens failed: scenes\/x/); + assert.ok(broken.description.length <= 140); + assert.equal(reviewState({ ranScreens: true, screens: changed, failed, labels: ["visual-ok"], approved: "f1", fingerprint: "f1" }).state, "success"); +}); + +test("commentBody shows changed screens, folds the rest and keeps its data", () => { + const screens = [screen("docs/hero-dark", "changed", { images: { before: "b.png", after: "a.png", diff: "d.png" } }), + screen("scenes/new-dark", "added", { images: { after: "n.png" }, note: "The base couldn't draw this scene: draw: Unknown value" }), + screen("docs/settings-dark", "unchanged")]; + const data = { head: "abcdef1234", fingerprint: "f1", approved: null }; + const body = commentBody({ base: "1234567890", head: "abcdef1234", screens, failed: [], imageURL: (file) => `https://example.test/${file}`, + review: { state: "pending", description: "2 screens changed: a maintainer adds visual-ok after looking." }, data }); + assert.match(body, /^/); + assert.match(body, /`main` at 1234567 → this pull request at abcdef1/); + assert.match(body, /https:\/\/example\.test\/b\.png/); + assert.match(body, /https:\/\/example\.test\/d\.png/); + assert.match(body, /couldn't draw this scene/); + assert.match(body, /
1 unchanged screen<\/summary>/); + assert.doesNotMatch(body, /docs\/settings-dark.*example\.test/); + assert.deepEqual(readData(body), data); + assert.equal(readData("no data here"), null); +}); + +test("assetName and releaseTag name the uploaded images", () => { + assert.equal(assetName(412, "abcdef1234", "docs~hero-dark-after.png"), "pr412-abcdef1-docs~hero-dark-after.png"); + assert.equal(releaseTag(new Date("2026-11-03T10:00:00Z")), "pr-previews-2026-11"); +}); From 4af35b0090831a4e5cb1c53ed8abb7869ebf2044 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 15:55:32 +0100 Subject: [PATCH 03/25] ci: bind visual-ok to the screens shown and keep artifact text inert The pull request's own CI job writes the screens artifact, so its names and messages are untrusted. A message could add a second data block that approved its own screens, and a new failure left visual-ok in place. - readManifest checks the manifest field by field, and every image it names must be in the artifact. validateArtifact takes only regular files and a manifest under 1 MB. - Names and messages reach the comment only in code spans, so they can't mention, link or hide anything, and the review line shows none of them. - The data is the comment's last line, its JSON can't end the HTML comment early, and only the bot's own comment is read. - The fingerprint covers each screen's status and every failure, so a new failure clears visual-ok. Label events recompute the status from the data alone. - A comment too long for GitHub drops the images of its last screens. - compare-screens reports names the bot can't show as failures, and carries the base's error as data. --- .github/scripts/compare-screens.mjs | 31 ++- .github/scripts/compare-screens.test.mjs | 9 +- .github/scripts/screens-comment.mjs | 281 ++++++++++++++++++----- .github/scripts/screens-comment.test.mjs | 220 +++++++++++++++--- 4 files changed, 433 insertions(+), 108 deletions(-) diff --git a/.github/scripts/compare-screens.mjs b/.github/scripts/compare-screens.mjs index ff4b7f7..313acf8 100644 --- a/.github/scripts/compare-screens.mjs +++ b/.github/scripts/compare-screens.mjs @@ -1,7 +1,8 @@ // Compares the screens drawn from a pull request (after) with those drawn from its base (before): // node compare-screens.mjs BEFORE AFTER OUT --prefix docs [--before-report scenes.json] [--after-report scenes.json] // It writes cropped before, after and difference images of each changed screen to OUT, and adds the -// screens to OUT/manifest.json: {screens: [{id, status, changedPixels, region, images, note}], failed: [{id, error}]}. +// screens to OUT/manifest.json: {screens: [{id, status, changedPixels, region, images, baseError?}], failed: [{id, error}]}. +// The trusted half of the bot checks that manifest strictly: see readManifest in screens-comment.mjs. import { existsSync, readFileSync, readdirSync, writeFileSync, mkdirSync } from "node:fs"; import { join, basename } from "node:path"; import { pathToFileURL } from "node:url"; @@ -11,6 +12,12 @@ import { decodePNG, encodePNG } from "./png.mjs"; const noise = 1; /** Points of context around a changed region, at the renders' 2× scale. */ const padding = 24; +/** The longest message the trusted half takes. */ +const maximumMessage = 1000; + +const cut = (text, limit) => (text.length <= limit ? text : `${text.slice(0, limit - 1)}…`); +/** Image names the bot can show, since they become file and asset names. */ +const isPlainName = (name) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name) && !name.includes("..") && name.length <= 80; function changedMask(before, after) { const mask = new Uint8Array(before.width * before.height); @@ -106,14 +113,17 @@ function sceneName(image) { /** * Compares every PNG in `beforeDir` and `afterDir`, writes the images of changed, added, removed and - * resized screens to `outDir`, and returns the screens. `beforeErrors` holds, by scene name, why - * the base couldn't draw a scene. + * resized screens to `outDir`, and returns the screens, and as failures the images whose names the + * bot can't show. `beforeErrors` holds, by scene name, why the base couldn't draw a scene. */ export function compareDirectories(beforeDir, afterDir, outDir, { prefix, beforeErrors = {} }) { const list = (folder) => (existsSync(folder) ? readdirSync(folder).filter((name) => name.endsWith(".png")) : []); - const names = [...new Set([...list(beforeDir), ...list(afterDir)])].sort(); + const all = [...new Set([...list(beforeDir), ...list(afterDir)])].sort(); + const names = all.filter((name) => isPlainName(basename(name, ".png"))); + const failed = all.filter((name) => !names.includes(name)).map((name) => ({ id: cut(`${prefix}/${basename(name, ".png")}`, 200), + error: "Its name may use only letters, digits, dots, dashes and underscores." })); mkdirSync(outDir, { recursive: true }); - return names.map((name) => { + const screens = names.map((name) => { const before = read(beforeDir, name), after = read(afterDir, name); const id = `${prefix}/${basename(name, ".png")}`; const result = comparePair(before, after); @@ -138,16 +148,18 @@ export function compareDirectories(beforeDir, afterDir, outDir, { prefix, before } const screen = { id, ...result, images }; const error = beforeErrors[sceneName(name)]; - if (result.status === "added" && error) screen.note = `The base couldn't draw this scene: ${error}`; + if (result.status === "added" && error) screen.baseError = cut(error, maximumMessage); return screen; }); + return { screens, failed }; } /** Scene name to error, from a scenes.json report `--render-scenes` wrote. */ function errors(report) { if (!report || !existsSync(report)) return {}; const scenes = JSON.parse(readFileSync(report, "utf8")).scenes ?? {}; - return Object.fromEntries(Object.entries(scenes).filter(([, entry]) => entry.error).map(([name, entry]) => [name, entry.error])); + return Object.fromEntries(Object.entries(scenes).filter(([, entry]) => typeof entry?.error === "string" && entry.error) + .map(([name, entry]) => [name, entry.error])); } export function main(argv) { @@ -160,8 +172,9 @@ export function main(argv) { if (!beforeDir || !afterDir || !outDir || !prefix) { throw new Error("usage: compare-screens.mjs BEFORE AFTER OUT --prefix NAME [--before-report FILE] [--after-report FILE]"); } - const screens = compareDirectories(beforeDir, afterDir, outDir, { prefix, beforeErrors: errors(option("--before-report")) }); - const failed = Object.entries(errors(option("--after-report"))).map(([name, error]) => ({ id: `${prefix}/${name}`, error })); + const { screens, failed: unnamed } = compareDirectories(beforeDir, afterDir, outDir, { prefix, beforeErrors: errors(option("--before-report")) }); + const failed = [...unnamed, ...Object.entries(errors(option("--after-report"))) + .map(([name, error]) => ({ id: cut(`${prefix}/${name}`, 200), error: cut(error, maximumMessage) }))]; const manifestPath = join(outDir, "manifest.json"); const manifest = existsSync(manifestPath) ? JSON.parse(readFileSync(manifestPath, "utf8")) : { screens: [], failed: [] }; manifest.screens.push(...screens); diff --git a/.github/scripts/compare-screens.test.mjs b/.github/scripts/compare-screens.test.mjs index 137caa8..72cf8b3 100644 --- a/.github/scripts/compare-screens.test.mjs +++ b/.github/scripts/compare-screens.test.mjs @@ -42,7 +42,9 @@ test("compareDirectories writes images for changed screens only, and notes what writeFileSync(join(before, "moved-dark.png"), encodePNG(base)); writeFileSync(join(after, "moved-dark.png"), encodePNG(paint(base, 10, 10, 4, 4, [255, 255, 255, 255]))); writeFileSync(join(after, "new-dark.png"), encodePNG(base)); - const screens = compareDirectories(before, after, out, { prefix: "scenes", beforeErrors: { new: "draw: Unknown value" } }); + writeFileSync(join(after, "odd name-dark.png"), encodePNG(base)); + const { screens, failed } = compareDirectories(before, after, out, + { prefix: "scenes", beforeErrors: { new: `draw: Unknown value ${"y".repeat(1200)}` } }); assert.deepEqual(screens.map((screen) => [screen.id, screen.status]), [["scenes/moved-dark", "changed"], ["scenes/new-dark", "added"], ["scenes/same-dark", "unchanged"]]); const moved = screens[0]; @@ -52,6 +54,9 @@ test("compareDirectories writes images for changed screens only, and notes what const crop = decodePNG(readFileSync(join(out, moved.images.after))); // The 4-pixel square at x = 10 gets 24 pixels of padding, cut at the left edge: from 0 to 10 + 4 + 24. assert.equal(crop.width, 38); - assert.match(screens[1].note, /Unknown value/); + // The trusted half takes messages of at most 1,000 characters. + assert.match(screens[1].baseError, /^draw: Unknown value y+…$/); + assert.equal(screens[1].baseError.length, 1000); assert.equal(screens[2].images.after, undefined); + assert.deepEqual(failed, [{ id: "scenes/odd name-dark", error: "Its name may use only letters, digits, dots, dashes and underscores." }]); }); diff --git a/.github/scripts/screens-comment.mjs b/.github/scripts/screens-comment.mjs index bcc44ca..a367563 100644 --- a/.github/scripts/screens-comment.mjs +++ b/.github/scripts/screens-comment.mjs @@ -1,58 +1,171 @@ // The logic of the before/after screens comment and its Visual review status, for screens.yml. The -// workflow runs the default branch's copy and reads the pull request's artifact only as data. +// workflow runs the default branch's copy and reads the pull request's artifact only as data: the +// pull request's CI job writes it, so every name and message in it is untrusted. Such text reaches +// the comment only inside code spans, and the comment's data is always its last line. import { createHash } from "node:crypto"; +export const botLogin = "github-actions[bot]"; const marker = ""; -const dataPattern = //s; +const reviewHeading = "**Visual review:**"; +const dataPattern = /\r?\n\s*$/; const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); const maximumFileSize = 10 * 1024 * 1024; +const maximumManifestSize = 1024 * 1024; const maximumFiles = 300; +const maximumScreens = 300; +const maximumFailures = 50; +const maximumMessage = 1000; +const maximumProblems = 20; +/** GitHub takes 65,536 characters in a comment. */ +const maximumComment = 60000; +const maximumRows = 100; +const maximumListed = 20; +const statuses = ["unchanged", "changed", "resized", "added", "removed"]; +/** The images a screen of each status comes with. */ +const imagesOf = { unchanged: [], changed: ["after", "before", "diff"], resized: ["after", "before"], added: ["after"], removed: ["before"] }; -/** Checks an artifact's files before anything is uploaded: only the manifest and PNG images, small enough. */ +const isObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value); +const hasOnly = (value, keys) => isObject(value) && Object.keys(value).every((key) => keys.includes(key)); +const isCount = (value, minimum = 0) => Number.isSafeInteger(value) && value >= minimum; +const isMessage = (value, limit = maximumMessage) => typeof value === "string" && value.length > 0 && value.length <= limit; +const isHex = (value, length) => typeof value === "string" && value.length === length && /^[0-9a-f]+$/.test(value); +const isImageName = (name) => /^[A-Za-z0-9][A-Za-z0-9._~-]*\.png$/.test(name) && !name.includes("..") && name.length <= 200; +const isScreenId = (id) => /^(docs|scenes|site)\/[A-Za-z0-9][A-Za-z0-9._-]*$/.test(id) && !id.includes("..") && id.length <= 100; + +/** `text` on one line and at most `limit` characters long. */ +function cut(text, limit) { + const characters = Array.from(String(text ?? "").replace(/\s+/g, " ").trim()); + return characters.length <= limit ? characters.join("") : `${characters.slice(0, limit - 1).join("")}…`; +} + +/** Untrusted text as an inline code span: markdown, HTML, links and mentions in it stay plain text. */ +const code = (text, limit = 200) => `\`${cut(text, limit).replaceAll("`", "'") || " "}\``; +const entities = { "&": "&", '"': """, "<": "<", ">": ">" }; +const attribute = (value) => String(value).replace(/[&"<>]/g, (character) => entities[character]); +/** The data's JSON, which then holds no "<" or ">" and so can't close its HTML comment early. */ +const encodeData = (data) => JSON.stringify(data).replace(/[<>&]/g, (character) => `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`); +const plural = (count, word) => `${count} ${word}${count === 1 ? "" : "s"}`; +const list = (ids, limit) => [...ids.slice(0, limit).map((id) => code(id)), ...(ids.length > limit ? [`and ${ids.length - limit} more`] : [])].join(", "); + +/** Checks an artifact's files before anything is read or uploaded: regular files only, PNG images and a small manifest. */ export function validateArtifact(files) { const problems = []; if (files.length > maximumFiles) problems.push(`${files.length} files, more than ${maximumFiles}.`); for (const file of files) { - if (file.name === "manifest.json") continue; - if (!/^[A-Za-z0-9][A-Za-z0-9._~-]*\.png$/.test(file.name) || file.name.includes("..")) { - problems.push(`${file.name}: not an image name this bot writes.`); + const name = code(file.name, 120); + if (!file.regular) { + problems.push(`${name}: not a regular file.`); + } else if (file.name === "manifest.json") { + if (file.size > maximumManifestSize) problems.push(`${name}: larger than 1 MB.`); + } else if (!isImageName(file.name)) { + problems.push(`${name}: not an image name this bot writes.`); } else if (!Buffer.from(file.head ?? []).subarray(0, 8).equals(signature)) { - problems.push(`${file.name}: not a PNG.`); + problems.push(`${name}: not a PNG.`); } else if (file.size > maximumFileSize) { - problems.push(`${file.name}: larger than 10 MB.`); + problems.push(`${name}: larger than 10 MB.`); } } - return { ok: problems.length === 0, problems }; + return { ok: problems.length === 0, problems: problems.slice(0, maximumProblems) }; } -/** One value for the changed screens and their after images, whatever their order. */ -export function fingerprint(changed) { - const lines = changed.map(({ id, afterSha256 }) => `${id} ${afterSha256}`).sort(); - return createHash("sha256").update(lines.join("\n")).digest("hex"); +function readScreen(screen, where, files, ids, problems) { + if (!hasOnly(screen, ["id", "status", "changedPixels", "region", "images", "baseError"])) { + problems.push(`${where} isn't a screen as compare-screens.mjs writes it.`); + return null; + } + if (!isScreenId(screen.id)) problems.push(`${where}: its id isn't docs/, scenes/ or site/ followed by a plain name.`); + else if (ids.has(screen.id)) problems.push(`${where}: ${code(screen.id)} is listed twice.`); + ids.add(screen.id); + if (!statuses.includes(screen.status)) { + problems.push(`${where}: its status isn't one of ${statuses.join(", ")}.`); + return null; + } + if (!isCount(screen.changedPixels)) problems.push(`${where}: changedPixels isn't a count.`); + const { region } = screen; + const regionFits = screen.status === "changed" + ? hasOnly(region, ["x", "y", "width", "height"]) && isCount(region.x) && isCount(region.y) && isCount(region.width, 1) && isCount(region.height, 1) + : region === null; + if (!regionFits) problems.push(`${where}: its region doesn't fit its status.`); + const kinds = imagesOf[screen.status]; + if (!hasOnly(screen.images, kinds) || Object.keys(screen.images).length !== kinds.length) { + problems.push(`${where}: a screen that is ${screen.status} comes with ${kinds.length ? kinds.join(", ") : "no images"}.`); + } else { + for (const kind of kinds) { + if (!isImageName(screen.images[kind]) || !files.has(screen.images[kind])) problems.push(`${where}: its ${kind} image isn't in the artifact.`); + } + } + if (screen.baseError !== undefined && !isMessage(screen.baseError)) problems.push(`${where}: baseError isn't a message of at most ${maximumMessage} characters.`); + return { + id: screen.id, status: screen.status, changedPixels: screen.changedPixels, + region: regionFits && region !== null ? { x: region.x, y: region.y, width: region.width, height: region.height } : null, + images: Object.fromEntries(kinds.map((kind) => [kind, screen.images?.[kind]])), + ...(screen.baseError === undefined ? {} : { baseError: screen.baseError }), + }; } -const plural = (count, word) => `${count} ${word}${count === 1 ? "" : "s"}`; -/** A commit status description holds at most 140 characters. */ -const fit = (text) => (text.length <= 140 ? text : `${text.slice(0, 139)}…`); +/** + * The screens and failures of an artifact's manifest.json, or why it can't be used. `names` are the + * artifact's files. Only known fields come back, and every image a screen names is one of those files. + */ +export function readManifest(text, names) { + const invalid = (problems) => ({ ok: false, problems: problems.slice(0, maximumProblems), screens: [], failed: [] }); + let manifest = null; + try { + manifest = JSON.parse(text); + } catch { + return invalid(["manifest.json isn't JSON."]); + } + if (!hasOnly(manifest, ["screens", "failed"]) || !Array.isArray(manifest.screens) || !Array.isArray(manifest.failed)) { + return invalid(["manifest.json isn't a list of screens and failures."]); + } + if (manifest.screens.length > maximumScreens) return invalid([`manifest.json lists more than ${maximumScreens} screens.`]); + if (manifest.failed.length > maximumFailures) return invalid([`manifest.json lists more than ${maximumFailures} failures.`]); + const problems = [], files = new Set(names), ids = new Set(); + const screens = manifest.screens.map((screen, index) => readScreen(screen, `Screen ${index + 1}`, files, ids, problems)); + const failed = manifest.failed.map((entry, index) => { + if (hasOnly(entry, ["id", "error"]) && isMessage(entry.id, 300) && isMessage(entry.error)) return { id: entry.id, error: entry.error }; + problems.push(`Failure ${index + 1} isn't an id and a message of at most ${maximumMessage} characters.`); + return null; + }); + return problems.length > 0 ? invalid(problems) : { ok: true, problems: [], screens, failed }; +} + +/** + * What a maintainer approves with visual-ok: each changed screen with its status and the image the + * comment shows of it, its after image or, once removed, its before image, and every failure. + * `digest(file)` is the SHA-256 of an image the bot uploads. Order doesn't matter. + */ +export function fingerprint({ screens = [], failed = [] }, digest) { + const entries = [ + ...screens.filter((screen) => screen.status !== "unchanged") + .map((screen) => ["screen", screen.id, screen.status, digest(screen.images.after ?? screen.images.before)]), + ...failed.map((entry) => ["failed", entry.id, entry.error]), + ].map((entry) => JSON.stringify(entry)).sort(); + return createHash("sha256").update(JSON.stringify(entries)).digest("hex"); +} + +/** What the status needs to know of a run's screens, kept in the comment's data for label events. */ +export function summary(screens, failed) { + return { total: screens.length, changed: screens.filter((screen) => screen.status !== "unchanged").length, failed: failed.length, + failure: failed.length > 0 ? cut(`${failed[0].id}: ${failed[0].error}`, 140) : null }; +} /** - * The Visual review status: success when nothing needs a look or a maintainer approved these exact - * screens with visual-ok, pending while changed screens wait for one, failure when drawing failed. - * `removeLabel` asks to take visual-ok off when it approved other screens than these. + * The Visual review status, from a run's summary: success when nothing needs a look or a maintainer + * approved this exact fingerprint with visual-ok, pending while changed screens wait for one, failure + * when drawing failed. `removeLabel` asks to take visual-ok off when it approved something else. */ -export function reviewState({ ranScreens, screens = [], failed = [], labels = [], approved, fingerprint: current }) { +export function reviewState({ ranScreens, total = 0, changed = 0, failed = 0, failure = null, labels = [], approved = null, fingerprint: current = null }) { if (!ranScreens) return { state: "success", description: "No screens to compare.", removeLabel: false }; - const changed = screens.filter((screen) => screen.status !== "unchanged"); const labelled = labels.includes("visual-ok"); - const valid = labelled && approved != null && approved === current; - if (failed.length > 0) { - if (valid) return { state: "success", description: fit(`Approved with visual-ok although ${plural(failed.length, "screen")} failed to draw.`), removeLabel: false }; - return { state: "failure", description: fit(`Drawing the screens failed: ${failed[0].id}: ${failed[0].error}`), removeLabel: labelled }; + const valid = labelled && approved !== null && approved === current; + if (failed > 0) { + if (valid) return { state: "success", description: `Approved with visual-ok although ${plural(failed, "screen")} failed to draw.`, removeLabel: false }; + return { state: "failure", description: cut(`Drawing the screens failed: ${failure ?? "see the comment."}`, 140), removeLabel: labelled }; } - if (changed.length === 0) return { state: "success", description: `${plural(screens.length, "screen")}, none changed.`, removeLabel: false }; - if (valid) return { state: "success", description: `${plural(changed.length, "changed screen")} approved.`, removeLabel: false }; - return { state: "pending", description: `${plural(changed.length, "screen")} changed: a maintainer adds visual-ok after looking.`, - removeLabel: labelled }; + if (changed === 0) return { state: "success", description: `${plural(total, "screen")}, none changed.`, removeLabel: false }; + if (valid) return { state: "success", description: `${plural(changed, "changed screen")} approved.`, removeLabel: false }; + return { state: "pending", description: `${plural(changed, "screen")} changed: a maintainer adds visual-ok after looking.`, removeLabel: labelled }; } function describe(screen) { @@ -68,49 +181,90 @@ function describe(screen) { } } -/** The comment: changed screens with their images, unchanged ones folded, failures, the status and its data. */ -export function commentBody({ base, head, screens, failed = [], imageURL, review, data }) { +function section(screen, imageURL) { + const image = (file, label) => `${label}`; + const lines = [`#### ${code(screen.id)}`]; + if (screen.baseError) lines.push("", `The base couldn't draw this scene: ${code(screen.baseError, 300)}`); + const { before, after, diff } = screen.images; + if (before && after) { + lines.push("", "| Before | After |", "| --- | --- |", `| ${image(before, "Before")} | ${image(after, "After")} |`); + } else if (after) { + lines.push("", image(after, "After")); + } else if (before) { + lines.push("", image(before, "Before")); + } + if (diff) lines.push("", "Difference, changed pixels in red:", "", image(diff, "Difference")); + lines.push(""); + return lines.join("\n"); +} + +/** The review line, which shows no untrusted text, and the data. */ +function reviewSection(review, data) { + const lines = [`${reviewHeading} ${review.state === "failure" ? "Drawing the screens failed, see “Couldn't draw” above." : review.description}`]; + if (review.removeLabel) lines.push("", "`visual-ok` was taken off: it approved other screens than these."); + lines.push("", ``); + return lines.join("\n"); +} + +/** + * The comment: changed screens with their images, unchanged ones folded, failures, the review line and + * the data. When it would be too long for GitHub, the last changed screens lose their images. + */ +export function commentBody({ baseRef = "main", base, head, screens, failed = [], imageURL, review, data }) { const changed = screens.filter((screen) => screen.status !== "unchanged"); const unchanged = screens.filter((screen) => screen.status === "unchanged"); - const image = (file, label) => `${label}`; - const lines = [marker, "### Before and after", "", - `\`main\` at ${base.slice(0, 7)} → this pull request at ${head.slice(0, 7)} · ${plural(screens.length, "screen")} drawn in one CI job · ${changed.length} changed`, ""]; - if (changed.length > 0) { - lines.push("| Screen | Change | Where (pixels at 2×) |", "| --- | --- | --- |"); - for (const screen of changed) lines.push(`| \`${screen.id}\` | ${describe(screen).join(" | ")} |`); - lines.push(""); - for (const screen of changed) { - lines.push(`#### \`${screen.id}\``); - if (screen.note) lines.push("", screen.note); - const { before, after, diff } = screen.images; - if (before && after) { - lines.push("", "| Before | After |", "| --- | --- |", `| ${image(before, "Before")} | ${image(after, "After")} |`); - } else if (after) { - lines.push("", image(after, "After")); - } else if (before) { - lines.push("", image(before, "Before")); + const sections = changed.map((screen) => section(screen, imageURL)); + const assemble = (shown) => { + const lines = [marker, "### Before and after", "", `${code(baseRef, 100)} at ${base.slice(0, 7)} → this pull request at ${head.slice(0, 7)}` + + ` · ${plural(screens.length, "screen")} drawn in one CI job · ${changed.length} changed`, ""]; + if (changed.length > 0) { + lines.push("| Screen | Change | Where (pixels at 2×) |", "| --- | --- | --- |"); + for (const screen of changed.slice(0, maximumRows)) lines.push(`| ${code(screen.id)} | ${describe(screen).join(" | ")} |`); + if (changed.length > maximumRows) lines.push("", `${changed.length - maximumRows} more changed screens aren't in the table.`); + lines.push("", ...sections.slice(0, shown)); + if (shown < changed.length) { + lines.push(`Not shown, to keep this comment under GitHub's size limit: ${list(changed.slice(shown).map((screen) => screen.id), maximumRows)}.`, ""); } - if (diff) lines.push("", `Difference, changed pixels in red:`, "", image(diff, "Difference")); + } + if (unchanged.length > 0) { + lines.push(`
${plural(unchanged.length, "unchanged screen")}`, "", + list(unchanged.map((screen) => screen.id), 2 * maximumRows), "", "
", ""); + } + if (failed.length > 0) { + lines.push("**Couldn't draw:**", "", ...failed.slice(0, maximumListed).map((entry) => `- ${code(entry.id)}: ${code(entry.error, 300)}`)); + if (failed.length > maximumListed) lines.push(`- and ${failed.length - maximumListed} more`); lines.push(""); } - } - if (unchanged.length > 0) { - lines.push(`
${plural(unchanged.length, "unchanged screen")}`, "", - unchanged.map((screen) => `\`${screen.id}\``).join(", "), "", "
", ""); - } - if (failed.length > 0) { - lines.push("**Couldn't draw:**", "", ...failed.map((entry) => `- \`${entry.id}\`: ${entry.error}`), ""); - } - lines.push(`**Visual review:** ${review.description}`, "", ``); - return lines.join("\n"); + lines.push(reviewSection(review, data)); + return lines.join("\n"); + }; + let shown = sections.length; + let body = assemble(shown); + while (body.length > maximumComment && shown > 0) body = assemble(--shown); + return body; } -/** The data a comment from commentBody carries, or null. */ +/** `body` with a new review line and data, for label events, which don't draw anything. */ +export function withReview(body, review, data) { + const start = body.lastIndexOf(`\n${reviewHeading} `); + return `${start >= 0 ? body.slice(0, start + 1) : `${body.trimEnd()}\n\n`}${reviewSection(review, data)}`; +} + +function isData(data) { + return hasOnly(data, ["head", "base", "ranScreens", "fingerprint", "approved", "total", "changed", "failed", "failure"]) + && isHex(data.head, 40) && (data.base === null || isHex(data.base, 40)) && typeof data.ranScreens === "boolean" + && (data.fingerprint === null || isHex(data.fingerprint, 64)) && (data.approved === null || isHex(data.approved, 64)) + && isCount(data.total) && isCount(data.changed) && isCount(data.failed) + && (data.failure === null || (typeof data.failure === "string" && data.failure.length <= 300)); +} + +/** The data on the last line of a comment from commentBody, or null. */ export function readData(body) { const match = dataPattern.exec(body ?? ""); if (!match) return null; try { - return JSON.parse(match[1]); + const data = JSON.parse(match[1]); + return isData(data) ? data : null; } catch { return null; } @@ -118,6 +272,11 @@ export function readData(body) { export const isScreensComment = (body) => (body ?? "").startsWith(marker); +/** The bot's screens comment among a pull request's comments. Anyone can post the marker; only the bot's comment counts. */ +export function findScreensComment(comments) { + return comments.find((comment) => comment.user?.login === botLogin && isScreensComment(comment.body)) ?? null; +} + /** The name of an uploaded image: the pull request and head commit, then the file. */ export const assetName = (pr, sha, file) => `pr${pr}-${sha.slice(0, 7)}-${file}`; diff --git a/.github/scripts/screens-comment.test.mjs b/.github/scripts/screens-comment.test.mjs index 3f466b2..47766bd 100644 --- a/.github/scripts/screens-comment.test.mjs +++ b/.github/scripts/screens-comment.test.mjs @@ -1,67 +1,215 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { validateArtifact, fingerprint, reviewState, commentBody, readData, assetName, releaseTag } from "./screens-comment.mjs"; +import { validateArtifact, readManifest, fingerprint, summary, reviewState, commentBody, withReview, readData, findScreensComment, + assetName, releaseTag } from "./screens-comment.mjs"; const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); -const screen = (id, status, extra = {}) => ({ id, status, changedPixels: status === "changed" ? 120 : 0, - region: status === "changed" ? { x: 10, y: 20, width: 30, height: 40 } : null, images: {}, ...extra }); +const sha = (character) => character.repeat(40); +const digest = (character) => character.repeat(64); +const screen = (id, status, images = {}, extra = {}) => ({ id, status, changedPixels: status === "changed" ? 120 : 0, + region: status === "changed" ? { x: 10, y: 20, width: 30, height: 40 } : null, images, ...extra }); +const data = (extra = {}) => ({ head: sha("a"), base: sha("b"), ranScreens: true, fingerprint: digest("f"), approved: null, + total: 2, changed: 1, failed: 0, failure: null, ...extra }); +const urls = (file) => `https://example.test/${file}`; +// What a pull request's CI job could put in a name or a message. +const hostile = `@octocat \`x\` | #1 https://evil.test **bold**\nsecond line`; -test("validateArtifact accepts only PNG images and the manifest", () => { - const good = [{ name: "manifest.json", size: 100, head: Buffer.from("{") }, { name: "docs~hero-dark-after.png", size: 2048, head: png }]; +test("validateArtifact accepts only regular PNG files and a small manifest", () => { + const good = [{ name: "manifest.json", size: 100, head: Buffer.from("{"), regular: true }, + { name: "docs~hero-dark-after.png", size: 2048, head: png, regular: true }]; assert.deepEqual(validateArtifact(good), { ok: true, problems: [] }); - for (const bad of [{ name: "../x.png", size: 10, head: png }, { name: "a.txt", size: 10, head: png }, - { name: "fake.png", size: 10, head: Buffer.from("GIF89a..") }, { name: "big.png", size: 11 * 1024 * 1024, head: png }]) { - assert.equal(validateArtifact([...good, bad]).ok, false, bad.name); + for (const bad of [{ name: "../x.png", size: 10, head: png, regular: true }, { name: "a.txt", size: 10, head: png, regular: true }, + { name: "fake.png", size: 10, head: Buffer.from("GIF89a.."), regular: true }, + { name: "big.png", size: 11 * 1024 * 1024, head: png, regular: true }, { name: "link.png", size: 10, head: png, regular: false }, + { name: "manifest.json", size: 2 * 1024 * 1024, head: Buffer.from("{"), regular: true }]) { + assert.equal(validateArtifact([...good.filter((file) => file.name !== bad.name), bad]).ok, false, bad.name); } - const many = Array.from({ length: 301 }, (_, index) => ({ name: `s${index}.png`, size: 10, head: png })); + const many = Array.from({ length: 301 }, (_, index) => ({ name: `s${index}.png`, size: 10, head: png, regular: true })); assert.equal(validateArtifact(many).ok, false); }); -test("fingerprint ignores order and follows the after images", () => { - const first = fingerprint([{ id: "docs/a", afterSha256: "11" }, { id: "docs/b", afterSha256: "22" }]); - assert.equal(first, fingerprint([{ id: "docs/b", afterSha256: "22" }, { id: "docs/a", afterSha256: "11" }])); - assert.notEqual(first, fingerprint([{ id: "docs/a", afterSha256: "11" }, { id: "docs/b", afterSha256: "23" }])); +test("readManifest keeps a manifest compare-screens wrote", () => { + const names = ["docs~hero-dark-before.png", "docs~hero-dark-after.png", "docs~hero-dark-diff.png", "scenes~new-dark-after.png"]; + const manifest = { screens: [screen("docs/hero-dark", "changed", { before: names[0], after: names[1], diff: names[2] }), + screen("scenes/new-dark", "added", { after: names[3] }, { baseError: "draw: Unknown value" }), screen("docs/settings-dark", "unchanged")], + failed: [{ id: "scenes//runner/work/scenes/a b.json", error: "Another scene is also named a b." }] }; + assert.deepEqual(readManifest(JSON.stringify(manifest), names), { ok: true, problems: [], screens: manifest.screens, failed: manifest.failed }); +}); + +test("readManifest rejects what compare-screens doesn't write", () => { + const names = ["docs~a-after.png", "docs~a-before.png", "docs~a-diff.png"]; + const good = screen("docs/a", "changed", { before: names[1], after: names[0], diff: names[2] }); + const manifest = (screens, failed = []) => JSON.stringify({ screens, failed }); + const cases = { + "not JSON": "{", + "no lists": JSON.stringify({ screens: {} }), + "unknown top-level field": '{"screens":[],"failed":[],"__proto__":{}}', + "unknown field": manifest([{ ...good, extra: 1 }]), + "id with markup": manifest([{ ...good, id: "docs/a`/); - assert.match(body, /`main` at 1234567 → this pull request at abcdef1/); + assert.match(body, /`main` at 1111111 → this pull request at aaaaaaa/); assert.match(body, /https:\/\/example\.test\/b\.png/); assert.match(body, /https:\/\/example\.test\/d\.png/); - assert.match(body, /couldn't draw this scene/); + assert.match(body, /The base couldn't draw this scene: `draw: Unknown value`/); assert.match(body, /
1 unchanged screen<\/summary>/); assert.doesNotMatch(body, /docs\/settings-dark.*example\.test/); - assert.deepEqual(readData(body), data); + assert.match(body, /\n\*\*Visual review:\*\* 2 screens changed: a maintainer adds visual-ok after looking\.\n/); + assert.deepEqual(readData(body), data()); +}); + +test("commentBody keeps untrusted text in code spans and its data its own", () => { + const screens = [screen("scenes/new-dark", "added", { after: "n.png" }, { baseError: hostile })]; + const failed = [{ id: "scenes/x ", error: hostile }]; + const review = reviewState({ ranScreens: true, ...summary(screens, failed), labels: [] }); + const body = commentBody({ baseRef: "main`", base: sha("1"), head: sha("a"), screens, failed, + imageURL: (file) => `https://example.test/${file}?a="b"`, review, data: data({ failed: 1, failure: hostile }) }); + assert.deepEqual(readData(body), data({ failed: 1, failure: hostile })); + // The HTML comment holding the data ends where its line does, so none of it shows. + const dataLine = body.split("\n").at(-1); + assert.deepEqual([dataLine.indexOf("-->"), dataLine.lastIndexOf("<")], [dataLine.length - 3, 0]); + const shown = body.split("\n").filter((line) => !line.startsWith("`; + assert.deepEqual(readData(`\ntext\n${line(data())}`), data()); + assert.deepEqual(readData(`\r\ntext\r\n${line(data())}\r\n`), data()); + assert.deepEqual(readData(`\n${line(data({ approved: digest("e") }))}\n${line(data())}`), data()); + assert.equal(readData(`\n${line(data())}\nmore text`), null); + assert.equal(readData(`x\n${line({ ...data(), approved: "yes" })}`), null); + assert.equal(readData(`x\n${line({ ...data(), extra: 1 })}`), null); assert.equal(readData("no data here"), null); }); +test("findScreensComment trusts only the bot's comment", () => { + const body = `\n`; + const impostor = { id: 1, user: { login: "someone", type: "User" }, body }; + const other = { id: 2, user: { login: "github-actions[bot]", type: "Bot" }, body: "Something else" }; + const bot = { id: 3, user: { login: "github-actions[bot]", type: "Bot" }, body }; + assert.equal(findScreensComment([impostor, other, bot]), bot); + assert.equal(findScreensComment([impostor, other]), null); +}); + test("assetName and releaseTag name the uploaded images", () => { assert.equal(assetName(412, "abcdef1234", "docs~hero-dark-after.png"), "pr412-abcdef1-docs~hero-dark-after.png"); assert.equal(releaseTag(new Date("2026-11-03T10:00:00Z")), "pr-previews-2026-11"); From 848a2938e2aec0959b592829becf0b0ec2b8d0bf Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 16:01:25 +0100 Subject: [PATCH 04/25] ci: capture the website for the before/after comparison capture-site.mjs draws each page of a site folder at desktop and phone size, and the home page in dark mode too. It answers requests from the folder instead of a local server, which dropped connections when Chrome opened several at once, and fetches nothing else. Reduced motion, a fixed clock and a wait for every image to decode make two captures of one commit match. --- .github/scripts/capture-site.mjs | 93 +++++++++++++++++++++++++++ .github/scripts/capture-site.test.mjs | 24 +++++++ 2 files changed, 117 insertions(+) create mode 100644 .github/scripts/capture-site.mjs create mode 100644 .github/scripts/capture-site.test.mjs diff --git a/.github/scripts/capture-site.mjs b/.github/scripts/capture-site.mjs new file mode 100644 index 0000000..f4bb0a0 --- /dev/null +++ b/.github/scripts/capture-site.mjs @@ -0,0 +1,93 @@ +// Captures the website for the before/after comparison: node capture-site.mjs SITE OUT +// Each page of the SITE folder at desktop and phone size, full page, to OUT/-.png, and the +// home page in dark mode too, to OUT/index--dark.png. Requests are answered from the folder, +// without a server, and nothing else is fetched. Motion is reduced and the clock is fixed at Thursday +// 22 October 2026, 09:41 UTC, so two captures of one commit match. Playwright is resolved from the +// working folder; STABLEBAR_CAPTURE_CHANNEL=chrome uses the installed Chrome instead of its Chromium. +import { createRequire } from "node:module"; +import { mkdirSync, statSync } from "node:fs"; +import { join, resolve, sep } from "node:path"; +import { pathToFileURL } from "node:url"; + +export const origin = "http://stablebar.test"; +const shots = [ + ...["index", "privacy", "terms", "404"].map((page) => ({ page, scheme: "light", suffix: "" })), + { page: "index", scheme: "dark", suffix: "-dark" }, +]; +const sizes = { + desktop: { viewport: { width: 1440, height: 900 }, deviceScaleFactor: 2 }, + phone: { viewport: { width: 390, height: 844 }, deviceScaleFactor: 3, isMobile: true, hasTouch: true }, +}; +const fixedTime = new Date("2026-10-22T09:41:00Z"); + +/** The file in the `root` folder that `url` asks for, or null when it asks for none. */ +export function fileFor(root, url) { + const { origin: asked, pathname } = new URL(url); + if (asked !== origin) return null; + let path; + try { + path = resolve(root, `.${decodeURIComponent(pathname)}`); + } catch { + return null; + } + const file = pathname.endsWith("/") ? join(path, "index.html") : path; + if (!file.startsWith(resolve(root) + sep)) return null; + try { + return statSync(file).isFile() ? file : null; + } catch { + return null; + } +} + +/** Waits for the fonts and for every image to load and decode; a broken image fails the capture. */ +async function settle(tab) { + await tab.waitForLoadState("networkidle"); + const broken = await tab.evaluate(async () => { + await document.fonts.ready; + const images = [...document.images]; + for (const image of images) image.loading = "eager"; + const results = await Promise.all(images.map((image) => image.decode().then(() => null, () => image.getAttribute("src")))); + return results.filter(Boolean); + }); + if (broken.length > 0) throw new Error(`${tab.url()}: these images didn't load: ${broken.join(", ")}`); +} + +export async function capture(site, out, { channel } = {}) { + const root = resolve(site); + const { chromium } = createRequire(join(process.cwd(), "capture-site.cjs"))("playwright"); + mkdirSync(out, { recursive: true }); + const browser = await chromium.launch(channel ? { channel } : {}); + try { + for (const [size, options] of Object.entries(sizes)) { + for (const scheme of ["light", "dark"]) { + const context = await browser.newContext({ ...options, colorScheme: scheme, reducedMotion: "reduce", locale: "en-US", timezoneId: "UTC" }); + await context.route("**/*", (route) => { + const url = route.request().url(); + const file = fileFor(root, url); + if (file) return route.fulfill({ path: file }); + return url.startsWith(`${origin}/`) ? route.fulfill({ status: 404, body: "" }) : route.abort(); + }); + for (const { page, suffix } of shots.filter((shot) => shot.scheme === scheme)) { + const tab = await context.newPage(); + await tab.clock.setFixedTime(fixedTime); + await tab.goto(`${origin}/${page}.html`, { waitUntil: "load" }); + await settle(tab); + await tab.screenshot({ path: join(out, `${page}-${size}${suffix}.png`), fullPage: true, animations: "disabled" }); + await tab.close(); + } + await context.close(); + } + } + } finally { + await browser.close(); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + const [site, out] = process.argv.slice(2); + if (!site || !out) { + console.error("usage: capture-site.mjs SITE OUT"); + process.exit(64); + } + await capture(site, out, { channel: process.env.STABLEBAR_CAPTURE_CHANNEL }); +} diff --git a/.github/scripts/capture-site.test.mjs b/.github/scripts/capture-site.test.mjs new file mode 100644 index 0000000..ee8142f --- /dev/null +++ b/.github/scripts/capture-site.test.mjs @@ -0,0 +1,24 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileFor, origin } from "./capture-site.mjs"; + +test("fileFor answers requests from the site folder only", () => { + const root = mkdtempSync(join(tmpdir(), "site-")); + mkdirSync(join(root, "site", "images"), { recursive: true }); + const site = join(root, "site"); + writeFileSync(join(site, "index.html"), "

home

"); + writeFileSync(join(site, "images", "a b.webp"), "image"); + writeFileSync(join(root, "secret.txt"), "secret"); + assert.equal(fileFor(site, `${origin}/`), join(site, "index.html")); + assert.equal(fileFor(site, `${origin}/index.html?x=1#top`), join(site, "index.html")); + assert.equal(fileFor(site, `${origin}/images/a%20b.webp`), join(site, "images", "a b.webp")); + assert.equal(fileFor(site, `${origin}/missing.html`), null); + assert.equal(fileFor(site, `${origin}/images`), null); + assert.equal(fileFor(site, `${origin}/%2e%2e/secret.txt`), null); + assert.equal(fileFor(site, `${origin}/..%2fsecret.txt`), null); + assert.equal(fileFor(site, `${origin}/%E0%A4%A`), null); + assert.equal(fileFor(site, "https://stablebar.app/index.html"), null); +}); From 2f62d54d6402c1eef442b2004cb33956be27d36d Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 16:04:34 +0100 Subject: [PATCH 05/25] ci: serve the website capture only files inside its folder fileFor followed symbolic links, so a link in site/ could serve a file from outside it. It now compares real paths and serves the real file. --- .github/scripts/capture-site.mjs | 20 +++++++++----------- .github/scripts/capture-site.test.mjs | 13 +++++++++---- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/.github/scripts/capture-site.mjs b/.github/scripts/capture-site.mjs index f4bb0a0..054dda7 100644 --- a/.github/scripts/capture-site.mjs +++ b/.github/scripts/capture-site.mjs @@ -5,7 +5,7 @@ // 22 October 2026, 09:41 UTC, so two captures of one commit match. Playwright is resolved from the // working folder; STABLEBAR_CAPTURE_CHANNEL=chrome uses the installed Chrome instead of its Chromium. import { createRequire } from "node:module"; -import { mkdirSync, statSync } from "node:fs"; +import { mkdirSync, realpathSync, statSync } from "node:fs"; import { join, resolve, sep } from "node:path"; import { pathToFileURL } from "node:url"; @@ -20,20 +20,18 @@ const sizes = { }; const fixedTime = new Date("2026-10-22T09:41:00Z"); -/** The file in the `root` folder that `url` asks for, or null when it asks for none. */ +/** + * The real path of the file in the `root` folder that `url` asks for, or null when it asks for none. + * Symbolic links count only when they lead to a file inside the folder. + */ export function fileFor(root, url) { const { origin: asked, pathname } = new URL(url); if (asked !== origin) return null; - let path; try { - path = resolve(root, `.${decodeURIComponent(pathname)}`); - } catch { - return null; - } - const file = pathname.endsWith("/") ? join(path, "index.html") : path; - if (!file.startsWith(resolve(root) + sep)) return null; - try { - return statSync(file).isFile() ? file : null; + const folder = realpathSync(root); + const path = resolve(folder, `.${decodeURIComponent(pathname)}`); + const file = realpathSync(pathname.endsWith("/") ? join(path, "index.html") : path); + return file.startsWith(folder + sep) && statSync(file).isFile() ? file : null; } catch { return null; } diff --git a/.github/scripts/capture-site.test.mjs b/.github/scripts/capture-site.test.mjs index ee8142f..930c433 100644 --- a/.github/scripts/capture-site.test.mjs +++ b/.github/scripts/capture-site.test.mjs @@ -1,6 +1,6 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { mkdtempSync, mkdirSync, realpathSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileFor, origin } from "./capture-site.mjs"; @@ -12,9 +12,14 @@ test("fileFor answers requests from the site folder only", () => { writeFileSync(join(site, "index.html"), "

home

"); writeFileSync(join(site, "images", "a b.webp"), "image"); writeFileSync(join(root, "secret.txt"), "secret"); - assert.equal(fileFor(site, `${origin}/`), join(site, "index.html")); - assert.equal(fileFor(site, `${origin}/index.html?x=1#top`), join(site, "index.html")); - assert.equal(fileFor(site, `${origin}/images/a%20b.webp`), join(site, "images", "a b.webp")); + symlinkSync(join(root, "secret.txt"), join(site, "leak.txt")); + symlinkSync(join(site, "index.html"), join(site, "home.html")); + const real = realpathSync(site); + assert.equal(fileFor(site, `${origin}/`), join(real, "index.html")); + assert.equal(fileFor(site, `${origin}/index.html?x=1#top`), join(real, "index.html")); + assert.equal(fileFor(site, `${origin}/images/a%20b.webp`), join(real, "images", "a b.webp")); + assert.equal(fileFor(site, `${origin}/home.html`), join(real, "index.html")); + assert.equal(fileFor(site, `${origin}/leak.txt`), null); assert.equal(fileFor(site, `${origin}/missing.html`), null); assert.equal(fileFor(site, `${origin}/images`), null); assert.equal(fileFor(site, `${origin}/%2e%2e/secret.txt`), null); From bb67b733df9e5bc83e5237884c68a66301260660 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 16:09:10 +0100 Subject: [PATCH 06/25] ci: hold the website's clock and random numbers still for captures The redrawn home page animates a canvas scene from Math.random on animation frames, so two captures of one commit differed. Each document now gets a seeded Math.random, and each page a clock paused at the fixed time that runs 5 seconds of timers and frames before the capture. Every page has a dark mode now, so all four are captured in both modes. --- .github/scripts/capture-site.mjs | 41 ++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/scripts/capture-site.mjs b/.github/scripts/capture-site.mjs index 054dda7..d7ba7c0 100644 --- a/.github/scripts/capture-site.mjs +++ b/.github/scripts/capture-site.mjs @@ -1,24 +1,37 @@ // Captures the website for the before/after comparison: node capture-site.mjs SITE OUT -// Each page of the SITE folder at desktop and phone size, full page, to OUT/-.png, and the -// home page in dark mode too, to OUT/index--dark.png. Requests are answered from the folder, -// without a server, and nothing else is fetched. Motion is reduced and the clock is fixed at Thursday -// 22 October 2026, 09:41 UTC, so two captures of one commit match. Playwright is resolved from the -// working folder; STABLEBAR_CAPTURE_CHANNEL=chrome uses the installed Chrome instead of its Chromium. +// Each page of the SITE folder at desktop and phone size, full page, in light mode to +// OUT/-.png and in dark mode to OUT/--dark.png. Requests are answered from +// the folder, without a server, and nothing else is fetched. So that two captures of one commit +// match, motion is reduced, Math.random is seeded, and the page's clock stands at Thursday 22 October +// 2026, 09:41 UTC, moving only when the capture lets 5 seconds of timers and frames run. Playwright +// is resolved from the working folder; STABLEBAR_CAPTURE_CHANNEL=chrome uses the installed Chrome +// instead of Playwright's Chromium. import { createRequire } from "node:module"; import { mkdirSync, realpathSync, statSync } from "node:fs"; import { join, resolve, sep } from "node:path"; import { pathToFileURL } from "node:url"; export const origin = "http://stablebar.test"; -const shots = [ - ...["index", "privacy", "terms", "404"].map((page) => ({ page, scheme: "light", suffix: "" })), - { page: "index", scheme: "dark", suffix: "-dark" }, -]; +const pages = ["index", "privacy", "terms", "404"]; +const schemes = { light: "", dark: "-dark" }; const sizes = { desktop: { viewport: { width: 1440, height: 900 }, deviceScaleFactor: 2 }, phone: { viewport: { width: 390, height: 844 }, deviceScaleFactor: 3, isMobile: true, hasTouch: true }, }; const fixedTime = new Date("2026-10-22T09:41:00Z"); +/** How long the page's timers and animation frames run, on its stopped clock, before the capture. */ +const settleTime = 5000; + +/** Replaces Math.random with a seeded generator (mulberry32), in every document before its scripts. */ +function seedRandom() { + let seed = 0x5eed; + Math.random = () => { + seed = (seed + 0x6d2b79f5) | 0; + let value = Math.imul(seed ^ (seed >>> 15), 1 | seed); + value = (value + Math.imul(value ^ (value >>> 7), 61 | value)) ^ value; + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +} /** * The real path of the file in the `root` folder that `url` asks for, or null when it asks for none. @@ -57,19 +70,23 @@ export async function capture(site, out, { channel } = {}) { const browser = await chromium.launch(channel ? { channel } : {}); try { for (const [size, options] of Object.entries(sizes)) { - for (const scheme of ["light", "dark"]) { + for (const [scheme, suffix] of Object.entries(schemes)) { const context = await browser.newContext({ ...options, colorScheme: scheme, reducedMotion: "reduce", locale: "en-US", timezoneId: "UTC" }); + await context.addInitScript(seedRandom); await context.route("**/*", (route) => { const url = route.request().url(); const file = fileFor(root, url); if (file) return route.fulfill({ path: file }); return url.startsWith(`${origin}/`) ? route.fulfill({ status: 404, body: "" }) : route.abort(); }); - for (const { page, suffix } of shots.filter((shot) => shot.scheme === scheme)) { + for (const page of pages) { const tab = await context.newPage(); - await tab.clock.setFixedTime(fixedTime); + await tab.clock.install({ time: fixedTime }); + await tab.clock.pauseAt(fixedTime); await tab.goto(`${origin}/${page}.html`, { waitUntil: "load" }); await settle(tab); + await tab.clock.runFor(settleTime); + await settle(tab); await tab.screenshot({ path: join(out, `${page}-${size}${suffix}.png`), fullPage: true, animations: "disabled" }); await tab.close(); } From 241df70a779703b42d28426ba3270a1e24d4c1bd Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 16:14:47 +0100 Subject: [PATCH 07/25] ci: draw the screens of pull requests and their base Draw screens renders the docs images and the scenes with the pull request's app and with its base's, each beside its own resources and examples, and compares them into the screens-app artifact. Draw website screens does the same for the website's pages. Both run the pull request's code with a read-only token and stay out of the release jobs. They run when the app, the scenes, the website or the comparison change. A base build that writes no scenes.json, because it can't draw scenes yet, shows every scene as new with a note. A broken image on the website now gets a warning instead of failing the capture, since it shows in the comparison like any other change. --- .github/scripts/capture-site.mjs | 7 +- .github/scripts/compare-screens.mjs | 11 +- .github/scripts/compare-screens.test.mjs | 15 ++- .github/workflows/ci.yml | 148 ++++++++++++++++++++++- 4 files changed, 168 insertions(+), 13 deletions(-) diff --git a/.github/scripts/capture-site.mjs b/.github/scripts/capture-site.mjs index d7ba7c0..c27f4da 100644 --- a/.github/scripts/capture-site.mjs +++ b/.github/scripts/capture-site.mjs @@ -50,7 +50,10 @@ export function fileFor(root, url) { } } -/** Waits for the fonts and for every image to load and decode; a broken image fails the capture. */ +/** + * Waits for the fonts and for every image to load and decode. A broken image is the same in every + * capture, so it only gets a warning, and shows in the comparison. + */ async function settle(tab) { await tab.waitForLoadState("networkidle"); const broken = await tab.evaluate(async () => { @@ -60,7 +63,7 @@ async function settle(tab) { const results = await Promise.all(images.map((image) => image.decode().then(() => null, () => image.getAttribute("src")))); return results.filter(Boolean); }); - if (broken.length > 0) throw new Error(`${tab.url()}: these images didn't load: ${broken.join(", ")}`); + if (broken.length > 0) console.log(`::warning::${new URL(tab.url()).pathname}: these images don't load: ${broken.join(", ")}`); } export async function capture(site, out, { channel } = {}) { diff --git a/.github/scripts/compare-screens.mjs b/.github/scripts/compare-screens.mjs index 313acf8..2e2143a 100644 --- a/.github/scripts/compare-screens.mjs +++ b/.github/scripts/compare-screens.mjs @@ -114,9 +114,10 @@ function sceneName(image) { /** * Compares every PNG in `beforeDir` and `afterDir`, writes the images of changed, added, removed and * resized screens to `outDir`, and returns the screens, and as failures the images whose names the - * bot can't show. `beforeErrors` holds, by scene name, why the base couldn't draw a scene. + * bot can't show. `beforeErrors` holds, by scene name, why the base couldn't draw a scene, and + * `beforeError` why it drew none. */ -export function compareDirectories(beforeDir, afterDir, outDir, { prefix, beforeErrors = {} }) { +export function compareDirectories(beforeDir, afterDir, outDir, { prefix, beforeErrors = {}, beforeError }) { const list = (folder) => (existsSync(folder) ? readdirSync(folder).filter((name) => name.endsWith(".png")) : []); const all = [...new Set([...list(beforeDir), ...list(afterDir)])].sort(); const names = all.filter((name) => isPlainName(basename(name, ".png"))); @@ -147,7 +148,7 @@ export function compareDirectories(beforeDir, afterDir, outDir, { prefix, before save("before", before); } const screen = { id, ...result, images }; - const error = beforeErrors[sceneName(name)]; + const error = beforeErrors[sceneName(name)] ?? beforeError; if (result.status === "added" && error) screen.baseError = cut(error, maximumMessage); return screen; }); @@ -172,7 +173,9 @@ export function main(argv) { if (!beforeDir || !afterDir || !outDir || !prefix) { throw new Error("usage: compare-screens.mjs BEFORE AFTER OUT --prefix NAME [--before-report FILE] [--after-report FILE]"); } - const { screens, failed: unnamed } = compareDirectories(beforeDir, afterDir, outDir, { prefix, beforeErrors: errors(option("--before-report")) }); + const beforeReport = option("--before-report"); + const beforeError = beforeReport && !existsSync(beforeReport) ? "The base build wrote no scenes.json, so it may not draw scenes yet." : undefined; + const { screens, failed: unnamed } = compareDirectories(beforeDir, afterDir, outDir, { prefix, beforeErrors: errors(beforeReport), beforeError }); const failed = [...unnamed, ...Object.entries(errors(option("--after-report"))) .map(([name, error]) => ({ id: cut(`${prefix}/${name}`, 200), error: cut(error, maximumMessage) }))]; const manifestPath = join(outDir, "manifest.json"); diff --git a/.github/scripts/compare-screens.test.mjs b/.github/scripts/compare-screens.test.mjs index 72cf8b3..41cb11c 100644 --- a/.github/scripts/compare-screens.test.mjs +++ b/.github/scripts/compare-screens.test.mjs @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { mkdtempSync, readFileSync, readdirSync, writeFileSync, mkdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { comparePair, compareDirectories } from "./compare-screens.mjs"; +import { comparePair, compareDirectories, main } from "./compare-screens.mjs"; import { decodePNG, encodePNG } from "./png.mjs"; function image(width, height, rgba = [40, 40, 40, 255]) { @@ -60,3 +60,16 @@ test("compareDirectories writes images for changed screens only, and notes what assert.equal(screens[2].images.after, undefined); assert.deepEqual(failed, [{ id: "scenes/odd name-dark", error: "Its name may use only letters, digits, dots, dashes and underscores." }]); }); + +test("main notes every new scene when the base wrote no scenes report", () => { + const root = mkdtempSync(join(tmpdir(), "screens-")); + const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); + for (const folder of [before, after]) mkdirSync(folder, { recursive: true }); + writeFileSync(join(after, "bar-dark.png"), encodePNG(image(8, 8))); + writeFileSync(join(after, "scenes.json"), JSON.stringify({ scenes: { bar: { images: ["bar-dark.png"] }, broken: { error: "draw: Unknown value" } } })); + main([before, after, out, "--prefix", "scenes", "--before-report", join(before, "scenes.json"), "--after-report", join(after, "scenes.json")]); + const manifest = JSON.parse(readFileSync(join(out, "manifest.json"), "utf8")); + assert.deepEqual(manifest.screens.map((screen) => [screen.id, screen.status, screen.baseError]), + [["scenes/bar-dark", "added", "The base build wrote no scenes.json, so it may not draw scenes yet."]]); + assert.deepEqual(manifest.failed, [{ id: "scenes/broken", error: "draw: Unknown value" }]); +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fabe24..e3d2940 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,8 @@ jobs: build: ${{ steps.revision.outputs.build }} app: ${{ steps.changes.outputs.app }} packaging: ${{ steps.changes.outputs.packaging }} + screens: ${{ steps.changes.outputs.screens }} + site: ${{ steps.changes.outputs.site }} steps: - name: Skip superseded pushes id: revision @@ -51,7 +53,9 @@ jobs: persist-credentials: false # A pull request compiles the release and draws the docs images only when it changes what they # build from: the files scripts/unchanged-app.sh compares, or this workflow. Packaging reads - # only the scripts and resources. HEAD merges the pull request into its base, HEAD^. + # only the scripts and resources. The before/after screens are drawn when the app, the scenes + # or the comparison change, and the website's when the website or its capture change. + # HEAD merges the pull request into its base, HEAD^. - name: Find what the pull request changes id: changes if: github.event_name == 'pull_request' @@ -63,6 +67,12 @@ jobs: if grep -qE '^(Resources/|scripts/|\.github/workflows/ci\.yml$)' <<< "$changed"; then echo "packaging=true" >> "$GITHUB_OUTPUT" fi + if grep -qE '^(Sources/|Resources/|Package\.(swift|resolved)$|scripts/|scenes/|\.github/workflows/ci\.yml$|\.github/scripts/(png|compare-screens)\.mjs$)' <<< "$changed"; then + echo "screens=true" >> "$GITHUB_OUTPUT" + fi + if grep -qE '^(site/|\.github/workflows/ci\.yml$|\.github/scripts/(png|compare-screens|capture-site)\.mjs$)' <<< "$changed"; then + echo "site=true" >> "$GITHUB_OUTPUT" + fi test: name: Test @@ -99,11 +109,11 @@ jobs: run: swift test --skip-build # Runs on main, and on pull requests that change the app, so a release-only failure shows up - # before it merges and leaves main without a beta. + # before it merges and leaves main without a beta. Pull requests that draw screens use it too. build: name: Build release needs: current - if: needs.current.outputs.build == 'true' && (github.ref == 'refs/heads/main' || needs.current.outputs.app == 'true') + if: needs.current.outputs.build == 'true' && (github.ref == 'refs/heads/main' || needs.current.outputs.app == 'true' || needs.current.outputs.screens == 'true') runs-on: macos-26 timeout-minutes: 20 steps: @@ -144,7 +154,8 @@ jobs: name: Docs images env: STABLEBAR_PREBUILT_DIR: release-build - needs: build + needs: [current, build] + if: github.ref == 'refs/heads/main' || needs.current.outputs.app == 'true' runs-on: macos-26 timeout-minutes: 15 steps: @@ -234,8 +245,9 @@ jobs: TITLE: ${{ github.event.pull_request.title }} run: printf '%s\n' "$TITLE" | npx commitlint --verbose - # Tests the logic of the Linked issue and Description checks. Those checks run from the default - # branch (pull-request-checks.yml), so a change to them is tested here before it merges. + # Tests the scripts in .github/scripts: the Linked issue and Description checks, and the + # before/after screens. The checks and the screens comment run from the default branch + # (pull-request-checks.yml, screens.yml), so a change to them is tested here before it merges. pull-request-checks: name: Test pull request checks if: github.event_name == 'pull_request' @@ -250,6 +262,130 @@ jobs: node-version: 24 - run: node --test .github/scripts/*.test.mjs + # Draws the docs images and the scenes with the pull request's app and with its base's, and + # compares them into the screens-app artifact. Both draw the pull request's scenes, each beside + # its own resources and examples. This job runs the pull request's code with a read-only token: + # screens.yml, which runs from the default branch, posts the comparison and the Visual review + # status. + screens: + name: Draw screens + env: + STABLEBAR_PREBUILT_DIR: release-build + needs: [current, build] + if: github.event_name == 'pull_request' && needs.current.outputs.screens == 'true' + runs-on: macos-26 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 2 + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: 24 + - uses: actions/download-artifact@v8 + with: + name: release-build + # The scenes and the comparison scripts are kept aside before the checkout moves to the base. + # A scene that fails to draw is reported in scenes.json, and later in the comparison. + - name: Draw the pull request's screens + timeout-minutes: 10 + run: | + mkdir release-build + tar -xf release-build.tar -C release-build + ./scripts/build.sh "$RUNNER_TEMP/head.app" + app="$RUNNER_TEMP/head.app/Contents/MacOS/StableBar" + "$app" --render-docs-images "$RUNNER_TEMP/after/docs" + "$app" --render-scenes "$RUNNER_TEMP/after/scenes" scenes || [[ -f "$RUNNER_TEMP/after/scenes/scenes.json" ]] + cp -R scenes "$RUNNER_TEMP/scenes" + cp -R .github/scripts "$RUNNER_TEMP/scripts" + - name: Check out the base + run: | + git checkout --quiet --detach HEAD^ + rm -rf scenes + cp -R "$RUNNER_TEMP/scenes" scenes + - name: Identify Swift toolchain + if: needs.current.outputs.app == 'true' + id: swift + run: echo "version=$({ swift --version; xcodebuild -version; xcrun --show-sdk-version; } | shasum -a 256 | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT" + - uses: actions/cache/restore@v6 + if: needs.current.outputs.app == 'true' + with: + path: .build + key: ${{ runner.os }}-${{ runner.arch }}-release-${{ steps.swift.outputs.version }}-${{ hashFiles('Package.swift', 'Package.resolved') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-release-${{ steps.swift.outputs.version }}-${{ hashFiles('Package.swift', 'Package.resolved') }}- + # When the app didn't change, the pull request's app draws the base too, and the docs images + # are the same. A base that can't draw scenes leaves no scenes.json; the comparison says so. + - name: Draw the base's screens + timeout-minutes: 15 + env: + APP: ${{ needs.current.outputs.app }} + run: | + mkdir -p "$RUNNER_TEMP/before" + if [[ "$APP" == true ]]; then + STABLEBAR_PREBUILT_DIR= ./scripts/build.sh "$RUNNER_TEMP/base.app" + app="$RUNNER_TEMP/base.app/Contents/MacOS/StableBar" + "$app" --render-docs-images "$RUNNER_TEMP/before/docs" + else + app="$RUNNER_TEMP/head.app/Contents/MacOS/StableBar" + cp -R "$RUNNER_TEMP/after/docs" "$RUNNER_TEMP/before/docs" + fi + "$app" --render-scenes "$RUNNER_TEMP/before/scenes" scenes || true + - name: Compare + run: | + compare="$RUNNER_TEMP/scripts/compare-screens.mjs" + node "$compare" "$RUNNER_TEMP/before/docs" "$RUNNER_TEMP/after/docs" "$RUNNER_TEMP/out" --prefix docs + node "$compare" "$RUNNER_TEMP/before/scenes" "$RUNNER_TEMP/after/scenes" "$RUNNER_TEMP/out" --prefix scenes \ + --before-report "$RUNNER_TEMP/before/scenes/scenes.json" --after-report "$RUNNER_TEMP/after/scenes/scenes.json" + - uses: actions/upload-artifact@v7 + with: + name: screens-app + path: ${{ runner.temp }}/out/ + retention-days: 7 + if-no-files-found: error + + # Captures the website's pages from the pull request and from its base, and compares them into + # the screens-site artifact, for screens.yml as above. + site-screens: + name: Draw website screens + needs: current + if: github.event_name == 'pull_request' && needs.current.outputs.site == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 2 + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: 24 + - name: Install Playwright + working-directory: ${{ runner.temp }} + run: | + mkdir playwright + cd playwright + npm init --yes > /dev/null + npm install --no-audit --no-fund playwright@1 + npx playwright install --with-deps chromium + - name: Capture the website + run: | + mkdir "$RUNNER_TEMP/base" + git archive HEAD^ site | tar -x -C "$RUNNER_TEMP/base" + capture="$GITHUB_WORKSPACE/.github/scripts/capture-site.mjs" + cd "$RUNNER_TEMP/playwright" + node "$capture" "$RUNNER_TEMP/base/site" "$RUNNER_TEMP/before" + node "$capture" "$GITHUB_WORKSPACE/site" "$RUNNER_TEMP/after" + - name: Compare + run: node .github/scripts/compare-screens.mjs "$RUNNER_TEMP/before" "$RUNNER_TEMP/after" "$RUNNER_TEMP/out" --prefix site + - uses: actions/upload-artifact@v7 + with: + name: screens-site + path: ${{ runner.temp }}/out/ + retention-days: 7 + if-no-files-found: error + # Every push to main with releasable commits publishes the next version as a beta prerelease, # unless it would ship the same app as the beta before it. It commits nothing. Both jobs build # with github.run_number, which only goes up, so a stable build always numbers higher than the From fc3c74b847d3a43d87614238835d03b76498edb5 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 16:17:27 +0100 Subject: [PATCH 08/25] ci: run the screen scripts when called through a symbolic link Node names a module by its real path, so compare-screens.mjs and capture-site.mjs did nothing, and exited without an error, when the path they were called by went through a link, as /tmp does on macOS. They now resolve that path first. --- .github/scripts/capture-site.mjs | 3 ++- .github/scripts/compare-screens.mjs | 5 +++-- .github/scripts/compare-screens.test.mjs | 16 +++++++++++++++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/scripts/capture-site.mjs b/.github/scripts/capture-site.mjs index c27f4da..c79377f 100644 --- a/.github/scripts/capture-site.mjs +++ b/.github/scripts/capture-site.mjs @@ -101,7 +101,8 @@ export async function capture(site, out, { channel } = {}) { } } -if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { +// Node names a module by its real path, so the path it was called by is resolved first. +if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) { const [site, out] = process.argv.slice(2); if (!site || !out) { console.error("usage: capture-site.mjs SITE OUT"); diff --git a/.github/scripts/compare-screens.mjs b/.github/scripts/compare-screens.mjs index 2e2143a..a4c0245 100644 --- a/.github/scripts/compare-screens.mjs +++ b/.github/scripts/compare-screens.mjs @@ -3,7 +3,7 @@ // It writes cropped before, after and difference images of each changed screen to OUT, and adds the // screens to OUT/manifest.json: {screens: [{id, status, changedPixels, region, images, baseError?}], failed: [{id, error}]}. // The trusted half of the bot checks that manifest strictly: see readManifest in screens-comment.mjs. -import { existsSync, readFileSync, readdirSync, writeFileSync, mkdirSync } from "node:fs"; +import { existsSync, readFileSync, readdirSync, realpathSync, writeFileSync, mkdirSync } from "node:fs"; import { join, basename } from "node:path"; import { pathToFileURL } from "node:url"; import { decodePNG, encodePNG } from "./png.mjs"; @@ -187,4 +187,5 @@ export function main(argv) { console.log(`${prefix}: ${screens.length} screens, ${changed} changed, ${failed.length} failed`); } -if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) main(process.argv.slice(2)); +// Node names a module by its real path, so the path it was called by is resolved first. +if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) main(process.argv.slice(2)); diff --git a/.github/scripts/compare-screens.test.mjs b/.github/scripts/compare-screens.test.mjs index 41cb11c..c620458 100644 --- a/.github/scripts/compare-screens.test.mjs +++ b/.github/scripts/compare-screens.test.mjs @@ -1,6 +1,9 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, readFileSync, readdirSync, writeFileSync, mkdirSync } from "node:fs"; +import { mkdtempSync, readFileSync, readdirSync, writeFileSync, mkdirSync, symlinkSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { comparePair, compareDirectories, main } from "./compare-screens.mjs"; @@ -73,3 +76,14 @@ test("main notes every new scene when the base wrote no scenes report", () => { [["scenes/bar-dark", "added", "The base build wrote no scenes.json, so it may not draw scenes yet."]]); assert.deepEqual(manifest.failed, [{ id: "scenes/broken", error: "draw: Unknown value" }]); }); + +test("the command lines run when called through a symbolic link", () => { + // /tmp on macOS is a link to /private/tmp, and Node names a module by its real path. + const link = join(mkdtempSync(join(tmpdir(), "scripts-")), "scripts"); + symlinkSync(dirname(fileURLToPath(import.meta.url)), link); + for (const [script, status] of [["compare-screens.mjs", 1], ["capture-site.mjs", 64]]) { + const run = spawnSync(process.execPath, [join(link, script)], { encoding: "utf8" }); + assert.equal(run.status, status, script); + assert.match(run.stderr, /usage: /, script); + } +}); From fc0c0deaeef6a16f36ec5e666cc34670020b1315 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 16:31:42 +0100 Subject: [PATCH 09/25] ci: publish the before/after screens and the visual review status screens.yml runs from the default branch when CI completes on a pull request, and when a pull request is labelled, unlabelled or closed. It reads the screens artifacts as data, uploads the changed screens' images to a monthly pre-release, writes one comment, and sets the Visual review status on the head. visual-ok approves the screens the comment shows; a later run that draws other screens takes it off. Closing without merging deletes the pull request's images. The logic is in screens-bot.mjs, tested against a fake GitHub: - a fork's run is found by its branch and head, and a run for an older head is skipped; - a screens job skipped because the release build failed is a failure, not "no screens"; - the comment links the URLs GitHub answers with, since it renames special characters; - each artifact holds only its own kind of screens, each image named after its screen; - a failure of the bot itself marks the head with an error status. --- .github/scripts/screens-bot.mjs | 258 +++++++++++++++++++++++ .github/scripts/screens-bot.test.mjs | 255 ++++++++++++++++++++++ .github/scripts/screens-comment.mjs | 31 ++- .github/scripts/screens-comment.test.mjs | 32 ++- .github/workflows/screens.yml | 120 +++++++++++ 5 files changed, 687 insertions(+), 9 deletions(-) create mode 100644 .github/scripts/screens-bot.mjs create mode 100644 .github/scripts/screens-bot.test.mjs create mode 100644 .github/workflows/screens.yml diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs new file mode 100644 index 0000000..cb9a318 --- /dev/null +++ b/.github/scripts/screens-bot.mjs @@ -0,0 +1,258 @@ +// The GitHub side of the before/after screens, for screens.yml, which runs the default branch's copy. +// It finds the pull request a CI run drew screens for, uploads their images to a monthly pre-release, +// writes the comment, sets the Visual review status, and handles visual-ok and closed pull requests. +// The artifacts come from the pull request's own CI jobs, so they are read as untrusted data. +import { closeSync, existsSync, lstatSync, openSync, readFileSync, readSync, readdirSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { join } from "node:path"; +import { validateArtifact, readManifest, fingerprint, summary, reviewState, commentBody, withReview, noScreensBody, readData, + findScreensComment, assetName, releaseTag, staleAssets, pullRequestAssets } from "./screens-comment.mjs"; + +const statusContext = "Visual review"; +const approval = "visual-ok"; +/** The CI jobs the screens depend on, by the names they show. */ +const jobNames = { current: "Check revision", build: "Build release", app: "Draw screens", site: "Draw website screens" }; +const kinds = { + app: { artifact: "screens-app", prefixes: ["docs", "scenes"], needs: ["current", "build"] }, + site: { artifact: "screens-site", prefixes: ["site"], needs: ["current"] }, +}; +const broken = ["failure", "cancelled", "timed_out"]; + +/** + * The pull request a completed CI run drew screens for, with what its jobs did and which artifacts it + * uploaded, or null when the run isn't for an open pull request's current head. A fork's run lists no + * pull requests, so it is found by its head branch and commit. + */ +export async function findRun({ github, context }) { + const { owner, repo } = context.repo; + const run = context.payload.workflow_run; + if (run.event !== "pull_request" || !run.head_repository) return null; + let numbers = run.pull_requests.map((pr) => pr.number); + if (numbers.length === 0) { + const pulls = await github.paginate(github.rest.pulls.list, + { owner, repo, state: "open", head: `${run.head_repository.owner.login}:${run.head_branch}`, per_page: 100 }); + numbers = pulls.map((pr) => pr.number); + } + for (const number of numbers) { + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: number }); + if (pr.state !== "open" || pr.head.sha !== run.head_sha) continue; + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { owner, repo, run_id: run.id, filter: "latest", per_page: 100 }); + const uploaded = (await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { owner, repo, run_id: run.id, per_page: 100 })) + .map((artifact) => artifact.name); + return { + number, head: run.head_sha, runId: run.id, + jobs: Object.fromEntries(Object.entries(jobNames).map(([key, name]) => [key, jobs.find((job) => job.name === name)?.conclusion ?? "skipped"])), + artifacts: Object.fromEntries(Object.entries(kinds).map(([kind, { artifact }]) => [kind, uploaded.includes(artifact)])), + }; + } + return null; +} + +/** The screens, failures and files of a downloaded artifact folder, checked as untrusted data. */ +export function readArtifact(folder, prefixes) { + if (!existsSync(folder)) return { ok: false, problems: ["It couldn't be downloaded."] }; + const files = readdirSync(folder).map((name) => { + const path = join(folder, name); + const stat = lstatSync(path); + const head = Buffer.alloc(8); + if (stat.isFile()) { + const descriptor = openSync(path, "r"); + try { + readSync(descriptor, head, 0, 8, 0); + } finally { + closeSync(descriptor); + } + } + return { name, path, size: stat.size, head, regular: stat.isFile() }; + }); + const checked = validateArtifact(files); + if (!checked.ok) return checked; + const manifest = files.find((file) => file.name === "manifest.json"); + if (!manifest) return { ok: false, problems: ["It has no manifest.json."] }; + const read = readManifest(readFileSync(manifest.path, "utf8"), files.map((file) => file.name), prefixes); + return read.ok ? { ...read, files: new Map(files.map((file) => [file.name, file.path])) } : read; +} + +const comments = (github, owner, repo, number) => + github.paginate(github.rest.issues.listComments, { owner, repo, issue_number: number, per_page: 100 }); +const labelsOf = async (github, owner, repo, number) => + (await github.paginate(github.rest.issues.listLabelsOnIssue, { owner, repo, issue_number: number, per_page: 100 })).map((label) => label.name); + +async function setStatus({ github, owner, repo, sha, review, url }) { + await github.rest.repos.createCommitStatus({ owner, repo, sha, state: review.state, context: statusContext, description: review.description, + ...(url ? { target_url: url } : {}) }); +} + +/** This month's pre-release for the images, created when missing, never as the latest release. */ +async function previewRelease({ github, owner, repo, target }) { + const tag = releaseTag(new Date()); + const find = async () => (await github.rest.repos.getReleaseByTag({ owner, repo, tag })).data; + try { + return await find(); + } catch (error) { + if (error.status !== 404) throw error; + } + try { + return (await github.rest.repos.createRelease({ owner, repo, tag_name: tag, target_commitish: target, name: `Pull request screens, ${tag.slice(12)}`, + prerelease: true, make_latest: "false", + body: "Images for the before and after screens the bot posts on pull requests. An unmerged pull request's images are deleted when it closes." })).data; + } catch (error) { + if (error.status === 422) return find(); + throw error; + } +} + +/** Uploads `images`, [file, path] pairs, and returns each file's URL as GitHub answers it. A rerun of a head replaces its images. */ +async function upload({ github, owner, repo, number, head, target, images }) { + const urls = new Map(); + if (images.length === 0) return urls; + const release = await previewRelease({ github, owner, repo, target }); + const assets = await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id: release.id, per_page: 100 }); + for (const asset of pullRequestAssets(assets, number).filter((entry) => entry.name.startsWith(assetName(number, head, "")))) { + await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id }); + } + for (const [file, path] of images) { + const data = readFileSync(path); + const { data: asset } = await github.rest.repos.uploadReleaseAsset({ owner, repo, release_id: release.id, name: assetName(number, head, file), data, + headers: { "content-type": "image/png", "content-length": data.length } }); + urls.set(file, asset.browser_download_url); + } + return urls; +} + +/** Deletes the images `pick` chooses from every pull request screens pre-release, and counts them. */ +async function deleteAssets({ github, owner, repo, pick }) { + const releases = (await github.paginate(github.rest.repos.listReleases, { owner, repo, per_page: 100 })) + .filter((release) => release.tag_name.startsWith("pr-previews-")); + let deleted = 0; + for (const release of releases) { + const assets = await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id: release.id, per_page: 100 }); + for (const asset of pick(assets)) { + await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id }); + deleted++; + } + } + return deleted; +} + +/** What the run's jobs and artifacts give: the screens, the failures, the image files and whether any screens were drawn. */ +function gather(run, folders) { + const screens = [], failed = [], files = new Map(); + let ranScreens = false; + for (const [kind, { artifact, prefixes, needs }] of Object.entries(kinds)) { + const conclusion = run.jobs[kind]; + if (conclusion === "skipped") { + const blocker = needs.find((job) => broken.includes(run.jobs[job])); + if (blocker) { + ranScreens = true; + failed.push({ id: kind, error: `${jobNames[blocker]} ended with ${run.jobs[blocker]}, so ${jobNames[kind]} didn't run.` }); + } + continue; + } + ranScreens = true; + if (conclusion !== "success") { + failed.push({ id: kind, error: `${jobNames[kind]} ended with ${conclusion}.` }); + continue; + } + const read = run.artifacts[kind] ? readArtifact(folders[kind] ?? "", prefixes) : { ok: false, problems: ["The job uploaded none."] }; + if (!read.ok) { + failed.push({ id: kind, error: `The ${artifact} artifact can't be used: ${read.problems.join(" ")}` }); + continue; + } + screens.push(...read.screens); + failed.push(...read.failed); + for (const [name, path] of read.files) files.set(name, path); + } + return { screens, failed, files, ranScreens }; +} + +/** + * Publishes a run from findRun: uploads the images of its changed screens, writes or rewrites the + * comment, sets the status on its head, takes visual-ok off when it approved other screens, and then + * deletes the images of the pull request's other heads. `folders` hold the downloaded artifacts by kind. + */ +export async function publishRun({ github, context, run, folders }) { + const { owner, repo } = context.repo; + const { screens, failed, files, ranScreens } = gather(run, folders); + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: run.number }); + const existing = findScreensComment(await comments(github, owner, repo, run.number)); + const previous = existing ? readData(existing.body) : null; + let review, body; + if (!ranScreens) { + review = reviewState({ ranScreens: false }); + const data = { head: run.head, base: pr.base.sha, ranScreens: false, fingerprint: null, approved: previous?.approved ?? null, + total: 0, changed: 0, failed: 0, failure: null }; + body = noScreensBody({ reason: `This pull request at ${run.head.slice(0, 7)} changes neither the app nor the website, so there are no screens to compare.`, + review, data }); + } else { + const shown = screens.filter((screen) => screen.status !== "unchanged").flatMap((screen) => Object.values(screen.images)); + const urls = await upload({ github, owner, repo, number: run.number, head: run.head, target: context.payload.repository?.default_branch, + images: shown.map((file) => [file, files.get(file)]) }); + const digest = (file) => createHash("sha256").update(readFileSync(files.get(file))).digest("hex"); + const print = fingerprint({ screens, failed }, digest); + const counts = summary(screens, failed); + review = reviewState({ ranScreens: true, ...counts, labels: await labelsOf(github, owner, repo, run.number), + approved: previous?.approved ?? null, fingerprint: print }); + if (review.removeLabel) { + try { + await github.rest.issues.removeLabel({ owner, repo, issue_number: run.number, name: approval }); + } catch (error) { + if (error.status !== 404) throw error; + } + } + const data = { head: run.head, base: pr.base.sha, ranScreens: true, fingerprint: print, + approved: review.removeLabel ? null : previous?.approved ?? null, ...counts }; + body = commentBody({ baseRef: pr.base.ref, base: pr.base.sha, head: run.head, screens, failed, imageURL: (file) => urls.get(file), review, data }); + } + let comment = existing; + if (existing) { + comment = (await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body })).data; + } else if (ranScreens) { + comment = (await github.rest.issues.createComment({ owner, repo, issue_number: run.number, body })).data; + } + await setStatus({ github, owner, repo, sha: run.head, review, url: comment?.html_url }); + await deleteAssets({ github, owner, repo, pick: (assets) => staleAssets(assets, run.number, run.head) }); + return review; +} + +/** Marks the head as errored when the bot itself fails, so a required Visual review doesn't wait forever. */ +export async function failRun({ github, context, run }) { + const { owner, repo } = context.repo; + await github.rest.repos.createCommitStatus({ owner, repo, sha: run.head, state: "error", context: statusContext, + description: "The screens bot failed: see the Screens workflow run.", target_url: `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}` }); +} + +/** + * Follows visual-ok being added or removed: an added label approves the fingerprint of the screens the + * comment shows, a removed one drops the approval. Sets the status when the comment is for the current + * head; a newer head's run sets its own. Returns the review, or null without the bot's comment. + */ +export async function updateReview({ github, context }) { + const { owner, repo } = context.repo; + const pr = context.payload.pull_request; + const existing = findScreensComment(await comments(github, owner, repo, pr.number)); + const data = existing ? readData(existing.body) : null; + if (!data) return null; + const labels = await labelsOf(github, owner, repo, pr.number); + const approved = !labels.includes(approval) ? null : context.payload.action === "labeled" ? data.fingerprint : data.approved; + const next = { ...data, approved }; + const review = reviewState({ ...next, labels }); + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body: withReview(existing.body, review, next) }); + if (data.head === pr.head.sha) await setStatus({ github, owner, repo, sha: data.head, review, url: existing.html_url }); + return review; +} + +/** Deletes an unmerged, closed pull request's images from every month, says so in its comment, and counts them. */ +export async function deleteScreens({ github, context }) { + const { owner, repo } = context.repo; + const number = context.payload.pull_request.number; + const deleted = await deleteAssets({ github, owner, repo, pick: (assets) => pullRequestAssets(assets, number) }); + const existing = findScreensComment(await comments(github, owner, repo, number)); + const data = existing ? readData(existing.body) : null; + if (data) { + const review = { state: "success", description: "Nothing to review: the pull request closed.", removeLabel: false }; + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, + body: noScreensBody({ reason: "This pull request closed without merging, so the images of its screens were deleted.", review, data }) }); + } + return deleted; +} diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs new file mode 100644 index 0000000..ca80185 --- /dev/null +++ b/.github/scripts/screens-bot.test.mjs @@ -0,0 +1,255 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { encodePNG } from "./png.mjs"; +import { main as compare } from "./compare-screens.mjs"; +import { readData, releaseTag } from "./screens-comment.mjs"; +import { deleteScreens, failRun, findRun, publishRun, updateReview } from "./screens-bot.mjs"; + +const head = "a".repeat(40), oldHead = "c".repeat(40), base = "b".repeat(40); +const repo = { owner: "Seb-GRAF", repo: "StableBar" }; +const bot = { login: "github-actions[bot]", type: "Bot" }; +const notFound = () => Object.assign(new Error("Not Found"), { status: 404 }); + +function image(color) { + const rgba = new Uint8Array(40 * 20 * 4); + for (let index = 0; index < 40 * 20; index++) rgba.set([40, 40, 40, 255], index * 4); + for (let y = 5; y < 10; y++) for (let x = 10; x < 15; x++) rgba.set(color, (y * 40 + x) * 4); + return { width: 40, height: 20, rgba }; +} + +/** An artifact folder as the Draw screens job uploads it: docs/hero-dark drawn in `color` after, docs/same-dark unchanged. */ +function artifact(color = [255, 255, 255, 255]) { + const root = mkdtempSync(join(tmpdir(), "artifact-")); + const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); + mkdirSync(before); + mkdirSync(after); + for (const folder of [before, after]) writeFileSync(join(folder, "same-dark.png"), encodePNG(image([40, 40, 40, 255]))); + writeFileSync(join(before, "hero-dark.png"), encodePNG(image([40, 40, 40, 255]))); + writeFileSync(join(after, "hero-dark.png"), encodePNG(image(color))); + const log = console.log; + console.log = () => {}; + try { + compare([before, after, out, "--prefix", "docs"]); + } finally { + console.log = log; + } + return out; +} + +/** The parts of Octokit the bot uses, over a little state. Uploads get GitHub's renaming of special characters. */ +function fakeGitHub({ pulls = [], comments = [], labels = [], releases = [], jobs = [], artifacts = [] } = {}) { + const state = { comments, labels, releases, statuses: [], uploads: [], deleted: [], next: 1000 }; + const rest = { + pulls: { + get: async ({ pull_number }) => { + const pr = pulls.find((entry) => entry.number === pull_number); + if (!pr) throw notFound(); + return { data: pr }; + }, + list: async ({ head: filter, state: wanted }) => + ({ data: pulls.filter((pr) => pr.state === wanted && `${pr.head.repo.owner.login}:${pr.head.ref}` === filter) }), + }, + issues: { + listComments: async () => ({ data: state.comments }), + createComment: async ({ body }) => { + const comment = { id: state.next++, user: bot, body, html_url: `https://example.test/comment/${state.next}` }; + state.comments.push(comment); + return { data: comment }; + }, + updateComment: async ({ comment_id, body }) => { + const comment = state.comments.find((entry) => entry.id === comment_id); + comment.body = body; + return { data: comment }; + }, + listLabelsOnIssue: async () => ({ data: state.labels.map((name) => ({ name })) }), + removeLabel: async ({ name }) => { + if (!state.labels.includes(name)) throw notFound(); + state.labels = state.labels.filter((entry) => entry !== name); + return { data: [] }; + }, + }, + repos: { + createCommitStatus: async (status) => { + state.statuses.push(status); + return { data: status }; + }, + getReleaseByTag: async ({ tag }) => { + const release = state.releases.find((entry) => entry.tag_name === tag); + if (!release) throw notFound(); + return { data: release }; + }, + createRelease: async (params) => { + const release = { id: state.next++, ...params, assets: [] }; + state.releases.push(release); + return { data: release }; + }, + listReleases: async () => ({ data: state.releases }), + listReleaseAssets: async ({ release_id }) => ({ data: state.releases.find((release) => release.id === release_id).assets }), + uploadReleaseAsset: async ({ release_id, name, data, headers }) => { + const renamed = name.replaceAll("~", "."); + const asset = { id: state.next++, name: renamed, browser_download_url: `https://example.test/download/${renamed}` }; + state.releases.find((release) => release.id === release_id).assets.push(asset); + state.uploads.push({ name, size: data.length, type: headers["content-type"] }); + return { data: asset }; + }, + deleteReleaseAsset: async ({ asset_id }) => { + for (const release of state.releases) { + const index = release.assets.findIndex((asset) => asset.id === asset_id); + if (index >= 0) state.deleted.push(...release.assets.splice(index, 1).map((asset) => asset.name)); + } + }, + }, + actions: { + listJobsForWorkflowRun: async () => ({ data: { total_count: jobs.length, jobs } }), + listWorkflowRunArtifacts: async () => ({ data: { total_count: artifacts.length, artifacts } }), + }, + }; + const paginate = async (method, params) => { + const { data } = await method(params); + return Array.isArray(data) ? data : data.jobs ?? data.artifacts; + }; + return { github: { rest, paginate }, state }; +} + +const pr = (extra = {}) => ({ number: 7, state: "open", head: { sha: head, ref: "feat/x", repo: { owner: { login: "someone" } } }, + base: { sha: base, ref: "main" }, ...extra }); +const runContext = (extra = {}) => ({ ...repo, repo, serverUrl: "https://github.com", runId: 5, payload: { + repository: { default_branch: "main" }, + workflow_run: { id: 99, event: "pull_request", head_sha: head, head_branch: "feat/x", + head_repository: { owner: { login: "someone" } }, pull_requests: [], ...extra } } }); +const labelContext = (action, sha = head) => ({ repo, payload: { action, label: { name: "visual-ok" }, pull_request: pr({ head: { ...pr().head, sha } }) } }); +const ran = (extra = {}) => ({ number: 7, head, runId: 99, jobs: { current: "success", build: "success", app: "success", site: "skipped" }, + artifacts: { app: true, site: false }, ...extra }); +const jobsOf = (conclusions) => Object.entries(conclusions).map(([name, conclusion]) => ({ name, conclusion })); + +test("findRun finds a fork's pull request by its branch and head, with what its jobs did", async () => { + const { github } = fakeGitHub({ pulls: [pr()], artifacts: [{ name: "screens-app" }, { name: "release-build" }], + jobs: jobsOf({ "Check revision": "success", "Build release": "success", "Draw screens": "success", "Test": "success" }) }); + assert.deepEqual(await findRun({ github, context: runContext() }), + { number: 7, head, runId: 99, jobs: { current: "success", build: "success", app: "success", site: "skipped" }, artifacts: { app: true, site: false } }); +}); + +test("findRun skips runs that aren't for an open pull request's current head", async () => { + const { github } = fakeGitHub({ pulls: [pr({ head: { ...pr().head, sha: oldHead } })] }); + assert.equal(await findRun({ github, context: runContext() }), null); + assert.equal(await findRun({ github, context: runContext({ event: "push" }) }), null); + const { github: closed } = fakeGitHub({ pulls: [pr({ state: "closed" })] }); + assert.equal(await findRun({ github: closed, context: runContext({ pull_requests: [{ number: 7 }] }) }), null); +}); + +test("publishRun uploads the changed screens, comments, sets the status and drops older heads' images", async () => { + const old = { id: 1, name: `pr7-${oldHead.slice(0, 7)}-docs.hero-dark-after.png` }, other = { id: 2, name: "pr8-1234567-docs.a-after.png" }; + const { github, state } = fakeGitHub({ pulls: [pr()], releases: [{ id: 3, tag_name: "pr-previews-2026-09", assets: [old, other] }] }); + const review = await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); + assert.deepEqual(review, { state: "pending", description: "1 screen changed: a maintainer adds visual-ok after looking.", removeLabel: false }); + const release = state.releases.find((entry) => entry.tag_name === releaseTag(new Date())); + assert.deepEqual([release.prerelease, release.make_latest, release.target_commitish], [true, "false", "main"]); + assert.deepEqual(state.uploads.map((upload) => [upload.name, upload.type]).sort(), + ["after", "before", "diff"].map((kind) => [`pr7-aaaaaaa-docs~hero-dark-${kind}.png`, "image/png"])); + const [comment] = state.comments; + assert.match(comment.body, /src="https:\/\/example\.test\/download\/pr7-aaaaaaa-docs\.hero-dark-after\.png"/); + const data = readData(comment.body); + assert.deepEqual([data.head, data.base, data.ranScreens, data.total, data.changed, data.approved], [head, base, true, 2, 1, null]); + assert.deepEqual(state.statuses, [{ ...repo, sha: head, state: "pending", context: "Visual review", description: review.description, + target_url: comment.html_url }]); + assert.deepEqual(state.deleted, [old.name]); +}); + +test("visual-ok approves the screens shown, survives a rerun of them and comes off for others", async () => { + const { github, state } = fakeGitHub({ pulls: [pr()] }); + await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); + const shown = readData(state.comments[0].body).fingerprint; + state.labels = ["visual-ok"]; + assert.deepEqual(await updateReview({ github, context: labelContext("labeled") }), + { state: "success", description: "1 changed screen approved.", removeLabel: false }); + assert.equal(readData(state.comments[0].body).approved, shown); + assert.equal(state.statuses.at(-1).state, "success"); + const rerun = await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); + assert.deepEqual([rerun.state, state.labels], ["success", ["visual-ok"]]); + const changed = await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact([255, 0, 0, 255]) } }); + assert.deepEqual([changed.state, state.labels, readData(state.comments[0].body).approved], ["pending", [], null]); + assert.match(state.comments[0].body, /`visual-ok` was taken off/); + assert.equal(state.comments.length, 1); + assert.equal(state.statuses.at(-1).state, "pending"); +}); + +test("removing visual-ok sends the review back to pending", async () => { + const { github, state } = fakeGitHub({ pulls: [pr()], labels: ["visual-ok"] }); + await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); + await updateReview({ github, context: labelContext("labeled") }); + state.labels = []; + const review = await updateReview({ github, context: labelContext("unlabeled") }); + assert.equal(review.state, "pending"); + assert.equal(readData(state.comments[0].body).approved, null); +}); + +test("updateReview ignores lookalike comments, and leaves a newer head's status to its run", async () => { + const lookalike = { id: 1, user: { login: "someone" }, body: `\n` }; + const { github, state } = fakeGitHub({ pulls: [pr()], comments: [lookalike], labels: ["visual-ok"] }); + assert.equal(await updateReview({ github, context: labelContext("labeled") }), null); + assert.equal(state.statuses.length, 0); + await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); + // The label came before any screens were drawn, so it approved none, and comes off. + assert.deepEqual(state.labels, []); + state.labels = ["visual-ok"]; + const statuses = state.statuses.length; + await updateReview({ github, context: labelContext("labeled", oldHead) }); + assert.equal(state.statuses.length, statuses); + assert.equal(readData(state.comments[1].body).approved, readData(state.comments[1].body).fingerprint); +}); + +test("publishRun answers a run that drew no screens", async () => { + const skipped = ran({ jobs: { current: "success", build: "skipped", app: "skipped", site: "skipped" }, artifacts: { app: false, site: false } }); + const { github, state } = fakeGitHub({ pulls: [pr()] }); + assert.deepEqual(await publishRun({ github, context: runContext(), run: skipped, folders: {} }), + { state: "success", description: "No screens to compare.", removeLabel: false }); + assert.deepEqual([state.comments.length, state.statuses[0].state], [0, "success"]); + await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); + await publishRun({ github, context: runContext(), run: skipped, folders: {} }); + assert.match(state.comments[0].body, /changes neither the app nor the website/); + assert.equal(readData(state.comments[0].body).ranScreens, false); +}); + +test("publishRun reports failed jobs and refuses unusable artifacts", async () => { + const { github, state } = fakeGitHub({ pulls: [pr()] }); + const blocked = await publishRun({ github, context: runContext(), folders: {}, + run: ran({ jobs: { current: "success", build: "failure", app: "skipped", site: "skipped" }, artifacts: { app: false, site: false } }) }); + assert.equal(blocked.state, "failure"); + assert.match(state.comments[0].body, /- `app`: `Build release ended with failure, so Draw screens didn't run\.`/); + const folder = artifact(); + symlinkSync("/etc/hosts", join(folder, "hosts.png")); + const refused = await publishRun({ github, context: runContext(), run: ran(), folders: { app: folder } }); + assert.equal(refused.state, "failure"); + assert.match(state.comments[0].body, /The screens-app artifact can't be used: 'hosts\.png': not a regular file\./); + const site = await publishRun({ github, context: runContext(), folders: { site: artifact() }, + run: ran({ jobs: { current: "success", build: "skipped", app: "skipped", site: "success" }, artifacts: { app: false, site: true } }) }); + assert.equal(site.state, "failure"); + assert.match(state.comments[0].body, /its id isn't site\/ followed by a plain name/); + assert.equal(state.uploads.length, 0); +}); + +test("failRun marks the head when the bot itself fails", async () => { + const { github, state } = fakeGitHub(); + await failRun({ github, context: runContext(), run: ran() }); + assert.deepEqual(state.statuses, [{ ...repo, sha: head, state: "error", context: "Visual review", + description: "The screens bot failed: see the Screens workflow run.", target_url: "https://github.com/Seb-GRAF/StableBar/actions/runs/5" }]); +}); + +test("deleteScreens deletes an unmerged pull request's images from every month", async () => { + const { github, state } = fakeGitHub({ pulls: [pr()], releases: [ + { id: 1, tag_name: "pr-previews-2026-09", assets: [{ id: 11, name: "pr7-aaaaaaa-a.png" }, { id: 12, name: "pr8-aaaaaaa-a.png" }] }, + { id: 2, tag_name: "pr-previews-2026-10", assets: [{ id: 21, name: "pr7-bbbbbbb-b.png" }] }, + { id: 3, tag_name: "v1.0.0", assets: [{ id: 31, name: "pr7-ccccccc-StableBar.dmg" }] }] }); + await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); + const close = { repo, payload: { action: "closed", pull_request: pr({ state: "closed", merged: false }) } }; + // Publishing already dropped the other head's image; closing takes this head's three and the stray one. + assert.equal(await deleteScreens({ github, context: close }), 4); + assert.deepEqual(state.releases.map((release) => [release.tag_name, release.assets.map((asset) => asset.name)]).slice(0, 3), + [["pr-previews-2026-09", ["pr8-aaaaaaa-a.png"]], ["pr-previews-2026-10", []], ["v1.0.0", ["pr7-ccccccc-StableBar.dmg"]]]); + assert.match(state.comments[0].body, /closed without merging, so the images of its screens were deleted/); + assert.notEqual(readData(state.comments[0].body), null); +}); diff --git a/.github/scripts/screens-comment.mjs b/.github/scripts/screens-comment.mjs index a367563..c584226 100644 --- a/.github/scripts/screens-comment.mjs +++ b/.github/scripts/screens-comment.mjs @@ -30,7 +30,10 @@ const isCount = (value, minimum = 0) => Number.isSafeInteger(value) && value >= const isMessage = (value, limit = maximumMessage) => typeof value === "string" && value.length > 0 && value.length <= limit; const isHex = (value, length) => typeof value === "string" && value.length === length && /^[0-9a-f]+$/.test(value); const isImageName = (name) => /^[A-Za-z0-9][A-Za-z0-9._~-]*\.png$/.test(name) && !name.includes("..") && name.length <= 200; -const isScreenId = (id) => /^(docs|scenes|site)\/[A-Za-z0-9][A-Za-z0-9._-]*$/.test(id) && !id.includes("..") && id.length <= 100; +const isScreenId = (id, prefixes) => typeof id === "string" && prefixes.includes(id.split("/")[0]) + && /^[a-z]+\/[A-Za-z0-9][A-Za-z0-9._-]*$/.test(id) && !id.includes("..") && id.length <= 100; +/** The name compare-screens.mjs gives a screen's image of one kind. */ +const imageOf = (id, kind) => `${id.replaceAll("/", "~")}-${kind}.png`; /** `text` on one line and at most `limit` characters long. */ function cut(text, limit) { @@ -68,12 +71,12 @@ export function validateArtifact(files) { return { ok: problems.length === 0, problems: problems.slice(0, maximumProblems) }; } -function readScreen(screen, where, files, ids, problems) { +function readScreen(screen, where, files, ids, prefixes, problems) { if (!hasOnly(screen, ["id", "status", "changedPixels", "region", "images", "baseError"])) { problems.push(`${where} isn't a screen as compare-screens.mjs writes it.`); return null; } - if (!isScreenId(screen.id)) problems.push(`${where}: its id isn't docs/, scenes/ or site/ followed by a plain name.`); + if (!isScreenId(screen.id, prefixes)) problems.push(`${where}: its id isn't ${prefixes.map((prefix) => `${prefix}/`).join(" or ")} followed by a plain name.`); else if (ids.has(screen.id)) problems.push(`${where}: ${code(screen.id)} is listed twice.`); ids.add(screen.id); if (!statuses.includes(screen.status)) { @@ -91,7 +94,9 @@ function readScreen(screen, where, files, ids, problems) { problems.push(`${where}: a screen that is ${screen.status} comes with ${kinds.length ? kinds.join(", ") : "no images"}.`); } else { for (const kind of kinds) { - if (!isImageName(screen.images[kind]) || !files.has(screen.images[kind])) problems.push(`${where}: its ${kind} image isn't in the artifact.`); + const file = screen.images[kind]; + if (typeof screen.id !== "string" || file !== imageOf(screen.id, kind)) problems.push(`${where}: its ${kind} image isn't named after it.`); + else if (!isImageName(file) || !files.has(file)) problems.push(`${where}: its ${kind} image isn't in the artifact.`); } } if (screen.baseError !== undefined && !isMessage(screen.baseError)) problems.push(`${where}: baseError isn't a message of at most ${maximumMessage} characters.`); @@ -105,9 +110,10 @@ function readScreen(screen, where, files, ids, problems) { /** * The screens and failures of an artifact's manifest.json, or why it can't be used. `names` are the - * artifact's files. Only known fields come back, and every image a screen names is one of those files. + * artifact's files, and `prefixes` the kinds of screens it may hold. Only known fields come back, and + * every image a screen names is one of those files, named after the screen. */ -export function readManifest(text, names) { +export function readManifest(text, names, prefixes = ["docs", "scenes", "site"]) { const invalid = (problems) => ({ ok: false, problems: problems.slice(0, maximumProblems), screens: [], failed: [] }); let manifest = null; try { @@ -121,7 +127,7 @@ export function readManifest(text, names) { if (manifest.screens.length > maximumScreens) return invalid([`manifest.json lists more than ${maximumScreens} screens.`]); if (manifest.failed.length > maximumFailures) return invalid([`manifest.json lists more than ${maximumFailures} failures.`]); const problems = [], files = new Set(names), ids = new Set(); - const screens = manifest.screens.map((screen, index) => readScreen(screen, `Screen ${index + 1}`, files, ids, problems)); + const screens = manifest.screens.map((screen, index) => readScreen(screen, `Screen ${index + 1}`, files, ids, prefixes, problems)); const failed = manifest.failed.map((entry, index) => { if (hasOnly(entry, ["id", "error"]) && isMessage(entry.id, 300) && isMessage(entry.error)) return { id: entry.id, error: entry.error }; problems.push(`Failure ${index + 1} isn't an id and a message of at most ${maximumMessage} characters.`); @@ -244,6 +250,11 @@ export function commentBody({ baseRef = "main", base, head, screens, failed = [] return body; } +/** A comment without screens: why there are none, which is the bot's own text, the review line and the data. */ +export function noScreensBody({ reason, review, data }) { + return [marker, "### Before and after", "", reason, "", reviewSection(review, data)].join("\n"); +} + /** `body` with a new review line and data, for label events, which don't draw anything. */ export function withReview(body, review, data) { const start = body.lastIndexOf(`\n${reviewHeading} `); @@ -282,3 +293,9 @@ export const assetName = (pr, sha, file) => `pr${pr}-${sha.slice(0, 7)}-${file}` /** The pre-release that holds a month's images. */ export const releaseTag = (date) => `pr-previews-${date.toISOString().slice(0, 7)}`; + +/** A pull request's images. GitHub may rename the rest of an asset's name, but not this prefix. */ +export const pullRequestAssets = (assets, pr) => assets.filter((asset) => asset.name.startsWith(`pr${pr}-`)); + +/** A pull request's images of other heads than `sha`. */ +export const staleAssets = (assets, pr, sha) => pullRequestAssets(assets, pr).filter((asset) => !asset.name.startsWith(assetName(pr, sha, ""))); diff --git a/.github/scripts/screens-comment.test.mjs b/.github/scripts/screens-comment.test.mjs index 47766bd..b532661 100644 --- a/.github/scripts/screens-comment.test.mjs +++ b/.github/scripts/screens-comment.test.mjs @@ -1,7 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { validateArtifact, readManifest, fingerprint, summary, reviewState, commentBody, withReview, readData, findScreensComment, - assetName, releaseTag } from "./screens-comment.mjs"; +import { validateArtifact, readManifest, fingerprint, summary, reviewState, commentBody, withReview, noScreensBody, readData, + findScreensComment, assetName, releaseTag, staleAssets, pullRequestAssets } from "./screens-comment.mjs"; const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); const sha = (character) => character.repeat(40); @@ -58,6 +58,7 @@ test("readManifest rejects what compare-screens doesn't write", () => { "failure without message": manifest([], [{ id: "scenes/x" }]), "too many screens": manifest(Array.from({ length: 301 }, (_, index) => screen(`docs/s${index}`, "unchanged"))), "too many failures": manifest([], Array.from({ length: 51 }, (_, index) => ({ id: `scenes/s${index}`, error: "x" }))), + "image of another screen": manifest([{ ...good, id: "docs/b" }]), }; for (const [name, text] of Object.entries(cases)) { const read = readManifest(text, names); @@ -65,6 +66,17 @@ test("readManifest rejects what compare-screens doesn't write", () => { assert.deepEqual([read.screens, read.failed], [[], []], name); assert.ok(read.problems.length > 0 && read.problems.length <= 20, name); } + const missing = readManifest(manifest([good]), names.filter((name) => name !== "docs~a-diff.png")); + assert.deepEqual([missing.ok, missing.problems], [false, ["Screen 1: its diff image isn't in the artifact."]]); +}); + +test("readManifest takes only the ids an artifact may hold", () => { + const names = ["docs~a-after.png"]; + const text = JSON.stringify({ screens: [screen("docs/a", "added", { after: names[0] })], failed: [{ id: "docs", error: "x" }] }); + assert.equal(readManifest(text, names, ["docs", "scenes"]).ok, true); + const site = readManifest(text, names, ["site"]); + assert.equal(site.ok, false); + assert.match(site.problems[0], /site\//); }); test("fingerprint covers each changed screen's image and every failure, in any order", () => { @@ -190,6 +202,15 @@ test("withReview replaces only the review line and the data", () => { assert.doesNotMatch(updated, /taken off/); }); +test("noScreensBody says why there are no screens and keeps the data", () => { + const review = { state: "success", description: "No screens to compare.", removeLabel: false }; + const body = noScreensBody({ reason: "This pull request changes neither the app nor the website.", review, data: data({ ranScreens: false }) }); + assert.match(body, /^\n### Before and after\n\nThis pull request changes neither/); + assert.match(body, /\*\*Visual review:\*\* No screens to compare\./); + assert.deepEqual(readData(body), data({ ranScreens: false })); + assert.equal(withReview(body, review, data()), noScreensBody({ reason: "This pull request changes neither the app nor the website.", review, data: data() })); +}); + test("readData reads only the data on the comment's last line", () => { const line = (value) => ``; assert.deepEqual(readData(`\ntext\n${line(data())}`), data()); @@ -214,3 +235,10 @@ test("assetName and releaseTag name the uploaded images", () => { assert.equal(assetName(412, "abcdef1234", "docs~hero-dark-after.png"), "pr412-abcdef1-docs~hero-dark-after.png"); assert.equal(releaseTag(new Date("2026-11-03T10:00:00Z")), "pr-previews-2026-11"); }); + +test("staleAssets and pullRequestAssets pick a pull request's images by name", () => { + const assets = ["pr7-abcdef1-docs.a-after.png", "pr7-1234567-docs.a-after.png", "pr70-1234567-docs.a-after.png", "pr8-abcdef1-x.png", "other.png"] + .map((name, id) => ({ id, name })); + assert.deepEqual(staleAssets(assets, 7, "abcdef1234").map((asset) => asset.name), ["pr7-1234567-docs.a-after.png"]); + assert.deepEqual(pullRequestAssets(assets, 7).map((asset) => asset.name), ["pr7-abcdef1-docs.a-after.png", "pr7-1234567-docs.a-after.png"]); +}); diff --git a/.github/workflows/screens.yml b/.github/workflows/screens.yml new file mode 100644 index 0000000..ea115be --- /dev/null +++ b/.github/workflows/screens.yml @@ -0,0 +1,120 @@ +name: Screens + +# Posts the before/after screens of a pull request and sets its Visual review status. CI's Draw +# screens and Draw website screens jobs run the pull request's code and upload what they drew. This +# workflow runs the default branch's copy of itself and of .github/scripts, reads those artifacts +# only as data, and holds the write token: it uploads the images to a monthly pre-release, writes +# the comment, follows the visual-ok label, and deletes an unmerged pull request's images when it +# closes. The logic and its tests are in .github/scripts/screens-bot.mjs and screens-comment.mjs. +on: + workflow_run: + workflows: [CI] + types: [completed] + pull_request_target: + types: [labeled, unlabeled, closed] + +permissions: + actions: read + contents: write + issues: write + pull-requests: write + statuses: write + +# One queue per pull request for CI runs and one for its label and close events: GitHub keeps only +# the newest pending run in a queue, so a label event never replaces a run's pending publication. +concurrency: + group: screens-${{ github.event_name }}-${{ github.event.workflow_run.head_repository.full_name || github.event.pull_request.head.repo.full_name }}-${{ github.event.workflow_run.head_branch || github.event.pull_request.head.ref }} + cancel-in-progress: false + +jobs: + publish: + name: Publish screens + if: github.event_name == 'workflow_run' && github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.repository.default_branch }} + sparse-checkout: .github/scripts + persist-credentials: false + - name: Find the pull request + id: run + uses: actions/github-script@v8 + with: + script: | + const { findRun } = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); + const run = await findRun({ github, context }); + if (!run) return core.info("This run isn't for an open pull request's current head."); + core.setOutput("run", JSON.stringify(run)); + core.setOutput("app", run.artifacts.app ? "true" : ""); + core.setOutput("site", run.artifacts.site ? "true" : ""); + # A download that fails leaves its folder missing, which the next step reports. + - uses: actions/download-artifact@v8 + if: steps.run.outputs.app + continue-on-error: true + with: + name: screens-app + path: ${{ runner.temp }}/screens-app + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - uses: actions/download-artifact@v8 + if: steps.run.outputs.site + continue-on-error: true + with: + name: screens-site + path: ${{ runner.temp }}/screens-site + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Publish + if: steps.run.outputs.run + uses: actions/github-script@v8 + env: + RUN: ${{ steps.run.outputs.run }} + with: + script: | + const bot = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); + const run = JSON.parse(process.env.RUN); + const folders = { app: `${process.env.RUNNER_TEMP}/screens-app`, site: `${process.env.RUNNER_TEMP}/screens-site` }; + try { + const review = await bot.publishRun({ github, context, run, folders }); + core.info(`Visual review: ${review.state}, ${review.description}`); + } catch (error) { + await bot.failRun({ github, context, run }); + throw error; + } + + review: + name: Follow visual-ok + if: github.event_name == 'pull_request_target' && github.event.action != 'closed' && github.event.label.name == 'visual-ok' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.repository.default_branch }} + sparse-checkout: .github/scripts + persist-credentials: false + - uses: actions/github-script@v8 + with: + script: | + const { updateReview } = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); + const review = await updateReview({ github, context }); + core.info(review ? `Visual review: ${review.state}, ${review.description}` : "There's no screens comment yet."); + + clean-up: + name: Delete unmerged screens + if: github.event_name == 'pull_request_target' && github.event.action == 'closed' && !github.event.pull_request.merged + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.repository.default_branch }} + sparse-checkout: .github/scripts + persist-credentials: false + - uses: actions/github-script@v8 + with: + script: | + const { deleteScreens } = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); + core.info(`Deleted ${await deleteScreens({ github, context })} images.`); From 76f093348ca5b4b921c4cbb9e8dcef6592ffc393 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 16:32:57 +0100 Subject: [PATCH 10/25] docs: explain the before/after screens and visual-ok CONTRIBUTING gains a Before and after screens section: what Draw screens and Draw website screens draw and when, the comment, where the images live and when they're deleted, the Visual review status, when visual-ok comes off, and that a pull request changing the drawing decides what its own comparison shows. AGENTS.md and the pull request template now point visible changes at the bot's comment and at scenes, instead of screenshots. --- .github/pull_request_template.md | 4 ++-- AGENTS.md | 2 +- CONTRIBUTING.md | 37 +++++++++++++++++++++++++++----- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6787f36..f68ca43 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -14,8 +14,8 @@ CONTRIBUTING.md explains how commit types decide releases. ## Testing +For visible changes, the screens bot comments with before/after images; when none shows your change, +extend a scene in scenes/. Add a short recording when motion matters: upload it here, don't commit it. --> ## Merging diff --git a/AGENTS.md b/AGENTS.md index a191220..871732b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ The developer's own StableBar is usually running while you work. - Run the tests for what you changed with `swift test --filter `. CI runs the whole suite on macOS for every pull request. - `node --test .github/scripts/*.test.mjs` tests the pull request checks in `.github/scripts`. - `StableBar --validate PATH` checks a configuration without starting its scripts. -- Visible changes (the bar, popups, setup, Settings) need before/after screenshots in the pull request. Ask before launching the app or taking screenshots: a second bar appears on the developer's screen. `StableBar --render-docs-images DIR` draws the bar, popups and Settings in offscreen windows instead, so nothing appears on screen. For a before/after of your change, add or extend a scene in `scenes/` and draw it with `StableBar --render-scenes DIR scenes`, from a build of `main` and of your branch: it runs nothing and draws offscreen (see [Scenes](CONTRIBUTING.md#scenes)). +- Visible changes (the bar, popups, setup, Settings, the website) need before/after images in the pull request, and CI draws them: the screens bot comments with the docs images, the scenes in `scenes/` and the website's pages, drawn from `main` and from your branch (see [Before and after screens](CONTRIBUTING.md#before-and-after-screens)). When no screen shows your change, add or extend a scene so the bot draws it, instead of uploading screenshots. To look before pushing, `StableBar --render-scenes DIR scenes` draws the scenes offscreen from a build, running nothing. Ask before launching the app or taking screenshots of it: a second bar appears on the developer's screen. Add a recording when motion matters. ## Hit every surface diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f6ffe64..633f714 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,9 +77,9 @@ There is no web view, custom drawing loop or private window-server API. Styling ## How we work 1. **Start from an issue** for every `fix`, `feat` or `perf` change, using the issue forms, and link it from the pull request. The [triage skill](#agent-skills) checks whether a report is already tracked, fixed or supported. Other changes, such as CI, tests or docs, can go straight to a pull request. -2. **Branch from `main` and open a pull request.** Nobody pushes to `main` directly. Fill in the template: the problem, the changes, how you tested them and how to merge, and end with the model and harness that did the work, or with "No agent". Upload screenshots of visible changes to the pull request instead of committing them. +2. **Branch from `main` and open a pull request.** Nobody pushes to `main` directly. Fill in the template: the problem, the changes, how you tested them and how to merge, and end with the model and harness that did the work, or with "No agent". For visible changes, the screens bot posts before and after images ([Before and after screens](#before-and-after-screens)); upload a recording when motion matters, instead of committing it. 3. **Keep one problem per pull request.** Unrelated fixes and cleanups get their own. -4. **Wait for CI.** `Test` and `Commit messages` must pass. `Linked issue` and `Description` check the pull request itself and rerun when you edit its description: a `fix`, `feat` or `perf` pull request closes an issue ("Fixes #N" in the description), and Problem, Changes and Testing are filled in. Depending on what a pull request changes, `Build release`, `Docs images` and `Package` run too ([Commits and releases](#commits-and-releases) lists when). Fix them before merging: the same failure on `main` stops the next beta. +4. **Wait for CI.** `Test` and `Commit messages` must pass. `Linked issue` and `Description` check the pull request itself and rerun when you edit its description: a `fix`, `feat` or `perf` pull request closes an issue ("Fixes #N" in the description), and Problem, Changes and Testing are filled in. Depending on what a pull request changes, `Build release`, `Docs images`, `Package`, `Draw screens` and `Draw website screens` run too ([Commits and releases](#commits-and-releases) lists when). Fix them before merging: the same failure on `main` stops the next beta. When screens changed, `Visual review` waits until a maintainer has looked at them and added the `visual-ok` label. 5. **Merge** with a merge commit or a rebase merge, so semantic-release sees each commit. Squash only when the title alone should reach the changelog. Reviews are requested automatically from the people listed in [CODEOWNERS](.github/CODEOWNERS), but they are not required. Ask for one when a change is risky or outside your usual area. @@ -88,9 +88,9 @@ Reviews are requested automatically from the people listed in [CODEOWNERS](.gith ## Commits and releases -Pull requests run `swift test` on macOS and check that every commit message, and the pull request title, follows [Conventional Commits](https://www.conventionalcommits.org). Squash merges use the title as the commit message. The tests load every configuration, module and package in `examples`, and the **Test** job parses each script in `scripts` and `examples` with the shell its first line names. The **Test pull request checks** job runs `node --test` on the logic of `Linked issue` and `Description`, in `.github/scripts`. +Pull requests run `swift test` on macOS and check that every commit message, and the pull request title, follows [Conventional Commits](https://www.conventionalcommits.org). Squash merges use the title as the commit message. The tests load every configuration, module and package in `examples`, and the **Test** job parses each script in `scripts` and `examples` with the shell its first line names. The **Test pull request checks** job runs `node --test` on the scripts in `.github/scripts`: the logic of `Linked issue`, `Description` and the [before and after screens](#before-and-after-screens). -A pull request that changes `Sources`, `Resources`, `Package.swift`, `Package.resolved`, `scripts` or the CI workflow also compiles the release and draws the [docs images](#docs-images), as `main` does. If it changes `Resources`, `scripts` or the workflow, the **Package** job builds the DMG as a beta would, ad-hoc signed because pull requests get no Developer ID, and checks its signature, version and starter configuration. +A pull request that changes `Sources`, `Resources`, `Package.swift`, `Package.resolved`, `scripts` or the CI workflow also compiles the release and draws the [docs images](#docs-images), as `main` does. If it changes `Resources`, `scripts` or the workflow, the **Package** job builds the DMG as a beta would, ad-hoc signed because pull requests get no Developer ID, and checks its signature, version and starter configuration. **Draw screens** runs when the app, `scenes` or the comparison scripts change, and **Draw website screens** when `site` changes. [semantic-release](https://github.com/semantic-release/semantic-release) reads the commits since the last stable release and decides the next version: @@ -139,7 +139,34 @@ StableBar --render-scenes /tmp/scenes scenes It writes `-dark.png` and `-light.png` for each scene, and `scenes.json` with each scene's images or the error that stopped it. It runs nothing (no command, provider, window manager or network request), and every clock shows Thursday 22 October 2026, 09:41 UTC, so two runs of one commit on one Mac draw the same pixels. Text about the Mac itself, such as a display's name or the window managers installed, still comes from the Mac that draws. -A change you can see extends the scene that shows its area, or adds one when none does. Scenes can be edited, merged or deleted. Keep the format additive: a build reports a scene it can't read as an error and draws the others. +A change you can see extends the scene that shows its area, or adds one when none does. Scenes can be edited, merged or deleted. Keep the format additive: a build reports a scene it can't read as an error and draws the others. Name scene files with letters, digits, dots, dashes and underscores, so the screens bot can show them. + +## Before and after screens + +On a pull request, CI draws the screens twice, from the pull request and from its base, and a bot posts what changed. + +- **Draw screens** draws the [docs images](#docs-images) and the [scenes](#scenes) when the app, the scenes or the comparison scripts change. Both sides draw the pull request's scene files, each beside its own resources and examples, so a new scene shows how `main` draws it too. When the app didn't change, the pull request's app draws both sides. +- **Draw website screens** captures the four pages of `site` at desktop (1440×900) and phone (390×844) size, in light and dark mode, when the website changes. The capture answers requests from the folder and fetches nothing else, seeds `Math.random` and stops the page's clock at the scenes' date, so two captures of one commit match. + +Both sides are drawn in one job on one machine. A pixel counts as changed when a color channel moves by more than 1 and a pixel next to it changed too. + +These jobs run the pull request's code with a read-only token. The **Screens** workflow runs from `main` when CI finishes, reads what they uploaded only as data, and: + +- comments once on the pull request and rewrites that comment on every push: a table of the changed screens, their before, after and difference images cropped to the change, the unchanged screens folded, and what couldn't be drawn; +- keeps the images as assets of a monthly prerelease, `pr-previews-YYYY-MM`, and deletes a head's images once a newer head replaces them. Closing a pull request without merging deletes all of its images; a merged one keeps its last; +- sets the **Visual review** status on the pull request's head. + +| Visual review | When | +| --- | --- | +| Success: "No screens to compare." | The pull request changes neither the app nor the website. | +| Success: "N screens, none changed." | Nothing looks different. | +| Pending | Screens changed. A maintainer looks at them, then adds the `visual-ok` label. | +| Success: "M changed screens approved." | `visual-ok` approved exactly these screens. | +| Failure | Drawing failed; the comment says what. A `visual-ok` added while it showed the failure still approves it. | + +`visual-ok` approves the screens the comment showed when it was added. Any maintainer can add it, the author too. A later push that changes those screens, or what fails to draw, takes the label off and the review goes back to pending: look again, and add it again. Removing the label also sends the review back to pending. + +Pull requests from forks get the same comment and status: the drawing runs in the fork's pull request, and **Screens** finds the pull request by its branch and head. The comparison runs in the pull request's own CI, so a pull request that changes the CI workflow or the drawing scripts decides what its comparison shows. Review such a change itself before you trust its screens. ## Agent skills From f9ffd401af7dc2264ef3339b6f1a96c3d38952e8 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 16:35:43 +0100 Subject: [PATCH 11/25] ci: fail the visual review when CI's screens jobs are missing or skipped The bot took a job missing from the run as skipped, and a skipped screens job as "no screens to compare", while the pull request decides its own CI's jobs. It now takes only runs of .github/workflows/ci.yml, counts a missing job as a failure, and reads the pull request's changed files from GitHub: a screens job they call for that was skipped fails the review. A test keeps the bot's file patterns the same as the Check revision job's. CONTRIBUTING asks maintainers to reload before adding visual-ok, since it approves the screens the comment shows at that moment. --- .github/scripts/screens-bot.mjs | 35 ++++++++++++++---- .github/scripts/screens-bot.test.mjs | 53 +++++++++++++++++++++------- CONTRIBUTING.md | 4 +-- 3 files changed, 71 insertions(+), 21 deletions(-) diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs index cb9a318..0f8a05e 100644 --- a/.github/scripts/screens-bot.mjs +++ b/.github/scripts/screens-bot.mjs @@ -16,17 +16,27 @@ const kinds = { app: { artifact: "screens-app", prefixes: ["docs", "scenes"], needs: ["current", "build"] }, site: { artifact: "screens-site", prefixes: ["site"], needs: ["current"] }, }; -const broken = ["failure", "cancelled", "timed_out"]; +const broken = ["failure", "cancelled", "timed_out", "missing"]; +/** + * The files whose changes call for each kind of screens: the patterns CI's Check revision job greps + * for its screens and site outputs. A test keeps them the same. The bot reads the pull request's + * files from GitHub, so a run whose job graph skips or drops a screens job can't pass as having none. + */ +export const changePatterns = { + app: /^(Sources\/|Resources\/|Package\.(swift|resolved)$|scripts\/|scenes\/|\.github\/workflows\/ci\.yml$|\.github\/scripts\/(png|compare-screens)\.mjs$)/, + site: /^(site\/|\.github\/workflows\/ci\.yml$|\.github\/scripts\/(png|compare-screens|capture-site)\.mjs$)/, +}; /** - * The pull request a completed CI run drew screens for, with what its jobs did and which artifacts it - * uploaded, or null when the run isn't for an open pull request's current head. A fork's run lists no - * pull requests, so it is found by its head branch and commit. + * The pull request a completed CI run drew screens for, with what its jobs did, which artifacts it + * uploaded and which screens its changed files call for, or null when the run isn't CI's for an open + * pull request's current head. A fork's run lists no pull requests, so it is found by its head branch + * and commit. A job the run doesn't have is "missing". */ export async function findRun({ github, context }) { const { owner, repo } = context.repo; const run = context.payload.workflow_run; - if (run.event !== "pull_request" || !run.head_repository) return null; + if (run.event !== "pull_request" || run.path !== ".github/workflows/ci.yml" || !run.head_repository) return null; let numbers = run.pull_requests.map((pr) => pr.number); if (numbers.length === 0) { const pulls = await github.paginate(github.rest.pulls.list, @@ -39,10 +49,13 @@ export async function findRun({ github, context }) { const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { owner, repo, run_id: run.id, filter: "latest", per_page: 100 }); const uploaded = (await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { owner, repo, run_id: run.id, per_page: 100 })) .map((artifact) => artifact.name); + const files = (await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number: number, per_page: 100 })) + .flatMap((file) => [file.filename, file.previous_filename].filter(Boolean)); return { number, head: run.head_sha, runId: run.id, - jobs: Object.fromEntries(Object.entries(jobNames).map(([key, name]) => [key, jobs.find((job) => job.name === name)?.conclusion ?? "skipped"])), + jobs: Object.fromEntries(Object.entries(jobNames).map(([key, name]) => [key, jobs.find((job) => job.name === name)?.conclusion ?? "missing"])), artifacts: Object.fromEntries(Object.entries(kinds).map(([kind, { artifact }]) => [kind, uploaded.includes(artifact)])), + expected: Object.fromEntries(Object.keys(kinds).map((kind) => [kind, files.some((file) => changePatterns[kind].test(file))])), }; } return null; @@ -141,11 +154,19 @@ function gather(run, folders) { let ranScreens = false; for (const [kind, { artifact, prefixes, needs }] of Object.entries(kinds)) { const conclusion = run.jobs[kind]; + if (conclusion === "missing") { + ranScreens = true; + failed.push({ id: kind, error: `${jobNames[kind]} is missing from this CI run.` }); + continue; + } if (conclusion === "skipped") { const blocker = needs.find((job) => broken.includes(run.jobs[job])); if (blocker) { ranScreens = true; - failed.push({ id: kind, error: `${jobNames[blocker]} ended with ${run.jobs[blocker]}, so ${jobNames[kind]} didn't run.` }); + failed.push({ id: kind, error: `${jobNames[blocker]} ${run.jobs[blocker] === "missing" ? "is missing" : `ended with ${run.jobs[blocker]}`}, so ${jobNames[kind]} didn't run.` }); + } else if (run.expected[kind]) { + ranScreens = true; + failed.push({ id: kind, error: `${jobNames[kind]} was skipped, but this pull request changes what it draws.` }); } continue; } diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs index ca80185..43d9315 100644 --- a/.github/scripts/screens-bot.test.mjs +++ b/.github/scripts/screens-bot.test.mjs @@ -1,12 +1,12 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, mkdirSync, symlinkSync, writeFileSync } from "node:fs"; +import { mkdtempSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { encodePNG } from "./png.mjs"; import { main as compare } from "./compare-screens.mjs"; import { readData, releaseTag } from "./screens-comment.mjs"; -import { deleteScreens, failRun, findRun, publishRun, updateReview } from "./screens-bot.mjs"; +import { changePatterns, deleteScreens, failRun, findRun, publishRun, updateReview } from "./screens-bot.mjs"; const head = "a".repeat(40), oldHead = "c".repeat(40), base = "b".repeat(40); const repo = { owner: "Seb-GRAF", repo: "StableBar" }; @@ -40,7 +40,7 @@ function artifact(color = [255, 255, 255, 255]) { } /** The parts of Octokit the bot uses, over a little state. Uploads get GitHub's renaming of special characters. */ -function fakeGitHub({ pulls = [], comments = [], labels = [], releases = [], jobs = [], artifacts = [] } = {}) { +function fakeGitHub({ pulls = [], comments = [], labels = [], releases = [], jobs = [], artifacts = [], files = [] } = {}) { const state = { comments, labels, releases, statuses: [], uploads: [], deleted: [], next: 1000 }; const rest = { pulls: { @@ -51,6 +51,7 @@ function fakeGitHub({ pulls = [], comments = [], labels = [], releases = [], job }, list: async ({ head: filter, state: wanted }) => ({ data: pulls.filter((pr) => pr.state === wanted && `${pr.head.repo.owner.login}:${pr.head.ref}` === filter) }), + listFiles: async () => ({ data: files.map((filename) => ({ filename })) }), }, issues: { listComments: async () => ({ data: state.comments }), @@ -118,18 +119,35 @@ const pr = (extra = {}) => ({ number: 7, state: "open", head: { sha: head, ref: base: { sha: base, ref: "main" }, ...extra }); const runContext = (extra = {}) => ({ ...repo, repo, serverUrl: "https://github.com", runId: 5, payload: { repository: { default_branch: "main" }, - workflow_run: { id: 99, event: "pull_request", head_sha: head, head_branch: "feat/x", + workflow_run: { id: 99, event: "pull_request", path: ".github/workflows/ci.yml", head_sha: head, head_branch: "feat/x", head_repository: { owner: { login: "someone" } }, pull_requests: [], ...extra } } }); const labelContext = (action, sha = head) => ({ repo, payload: { action, label: { name: "visual-ok" }, pull_request: pr({ head: { ...pr().head, sha } }) } }); const ran = (extra = {}) => ({ number: 7, head, runId: 99, jobs: { current: "success", build: "success", app: "success", site: "skipped" }, - artifacts: { app: true, site: false }, ...extra }); + artifacts: { app: true, site: false }, expected: { app: true, site: false }, ...extra }); const jobsOf = (conclusions) => Object.entries(conclusions).map(([name, conclusion]) => ({ name, conclusion })); -test("findRun finds a fork's pull request by its branch and head, with what its jobs did", async () => { - const { github } = fakeGitHub({ pulls: [pr()], artifacts: [{ name: "screens-app" }, { name: "release-build" }], - jobs: jobsOf({ "Check revision": "success", "Build release": "success", "Draw screens": "success", "Test": "success" }) }); - assert.deepEqual(await findRun({ github, context: runContext() }), - { number: 7, head, runId: 99, jobs: { current: "success", build: "success", app: "success", site: "skipped" }, artifacts: { app: true, site: false } }); +test("findRun finds a fork's pull request by its branch and head, with what its jobs did and should have done", async () => { + const { github } = fakeGitHub({ pulls: [pr()], artifacts: [{ name: "screens-app" }, { name: "release-build" }], files: ["Sources/StableBar/BarView.swift", "README.md"], + jobs: jobsOf({ "Check revision": "success", "Build release": "success", "Draw screens": "success", "Draw website screens": "skipped", "Test": "success" }) }); + assert.deepEqual(await findRun({ github, context: runContext() }), { number: 7, head, runId: 99, + jobs: { current: "success", build: "success", app: "success", site: "skipped" }, artifacts: { app: true, site: false }, expected: { app: true, site: false } }); + // A job the run doesn't have is missing, not skipped. + const { github: renamed } = fakeGitHub({ pulls: [pr()], jobs: jobsOf({ "Check revision": "success" }) }); + assert.deepEqual((await findRun({ github: renamed, context: runContext() })).jobs, { current: "success", build: "missing", app: "missing", site: "missing" }); +}); + +test("findRun takes only runs of the CI workflow file", async () => { + const { github } = fakeGitHub({ pulls: [pr()] }); + assert.equal(await findRun({ github, context: runContext({ path: ".github/workflows/other.yml" }) }), null); +}); + +test("changePatterns match the files CI's Check revision job draws screens for", () => { + const workflow = readFileSync(new URL("../workflows/ci.yml", import.meta.url), "utf8"); + const found = Object.fromEntries([...workflow.matchAll(/grep -qE '([^']+)' <<< "\$changed"; then\n\s+echo "(\w+)=true"/g)] + .map(([, pattern, output]) => [output, pattern])); + // A JavaScript pattern's source writes "/" as "\/". + const grep = (pattern) => pattern.source.replaceAll("\\/", "/"); + assert.deepEqual({ app: grep(changePatterns.app), site: grep(changePatterns.site) }, { app: found.screens, site: found.site }); }); test("findRun skips runs that aren't for an open pull request's current head", async () => { @@ -203,7 +221,8 @@ test("updateReview ignores lookalike comments, and leaves a newer head's status }); test("publishRun answers a run that drew no screens", async () => { - const skipped = ran({ jobs: { current: "success", build: "skipped", app: "skipped", site: "skipped" }, artifacts: { app: false, site: false } }); + const skipped = ran({ jobs: { current: "success", build: "skipped", app: "skipped", site: "skipped" }, artifacts: { app: false, site: false }, + expected: { app: false, site: false } }); const { github, state } = fakeGitHub({ pulls: [pr()] }); assert.deepEqual(await publishRun({ github, context: runContext(), run: skipped, folders: {} }), { state: "success", description: "No screens to compare.", removeLabel: false }); @@ -226,9 +245,19 @@ test("publishRun reports failed jobs and refuses unusable artifacts", async () = assert.equal(refused.state, "failure"); assert.match(state.comments[0].body, /The screens-app artifact can't be used: 'hosts\.png': not a regular file\./); const site = await publishRun({ github, context: runContext(), folders: { site: artifact() }, - run: ran({ jobs: { current: "success", build: "skipped", app: "skipped", site: "success" }, artifacts: { app: false, site: true } }) }); + run: ran({ jobs: { current: "success", build: "skipped", app: "skipped", site: "success" }, artifacts: { app: false, site: true }, + expected: { app: false, site: true } }) }); assert.equal(site.state, "failure"); assert.match(state.comments[0].body, /its id isn't site\/ followed by a plain name/); + // Screens the changed files call for, skipped or missing from the run, fail rather than pass as none. + const missing = await publishRun({ github, context: runContext(), folders: {}, + run: ran({ jobs: { current: "success", build: "success", app: "missing", site: "skipped" }, artifacts: { app: false, site: false } }) }); + assert.equal(missing.state, "failure"); + assert.match(state.comments[0].body, /- `app`: `Draw screens is missing from this CI run\.`/); + const skipped = await publishRun({ github, context: runContext(), folders: {}, + run: ran({ jobs: { current: "success", build: "success", app: "skipped", site: "skipped" }, artifacts: { app: false, site: false } }) }); + assert.equal(skipped.state, "failure"); + assert.match(state.comments[0].body, /- `app`: `Draw screens was skipped, but this pull request changes what it draws\.`/); assert.equal(state.uploads.length, 0); }); diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 633f714..198cbbe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -164,9 +164,9 @@ These jobs run the pull request's code with a read-only token. The **Screens** w | Success: "M changed screens approved." | `visual-ok` approved exactly these screens. | | Failure | Drawing failed; the comment says what. A `visual-ok` added while it showed the failure still approves it. | -`visual-ok` approves the screens the comment showed when it was added. Any maintainer can add it, the author too. A later push that changes those screens, or what fails to draw, takes the label off and the review goes back to pending: look again, and add it again. Removing the label also sends the review back to pending. +`visual-ok` approves the screens the comment shows when it's added, so reload the pull request first if a push just landed. Any maintainer can add it, the author too. A later push that changes those screens, or what fails to draw, takes the label off and the review goes back to pending: look again, and add it again. Removing the label also sends the review back to pending. -Pull requests from forks get the same comment and status: the drawing runs in the fork's pull request, and **Screens** finds the pull request by its branch and head. The comparison runs in the pull request's own CI, so a pull request that changes the CI workflow or the drawing scripts decides what its comparison shows. Review such a change itself before you trust its screens. +Pull requests from forks get the same comment and status: the drawing runs in the fork's pull request, and **Screens** finds the pull request by its branch and head. The comparison runs in the pull request's own CI, so a pull request that changes the CI workflow or the drawing scripts decides what its comparison shows. **Screens** reads the pull request's changed files from GitHub, so screens they call for that weren't drawn fail the review, but review such a change itself before you trust its screens. ## Agent skills From 79ef7b35804e768a90d4c0967debb45ffa27d006 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 17:08:13 +0100 Subject: [PATCH 12/25] ci: crop screens of a new height from their first change, with more context A website page whose height changed was posted as two whole pages, about 2880 by 11840 pixels, without a difference. Screens of one width are now compared from the top: the crops run from the first change, or from where the shorter one ends, across the full width, for at most 1,600 rows, with a difference. A new width still shows both whole. Crops get 48 pixels of context instead of 24, which was 12 points at 2x and 8 at 3x, and are at least 480 by 160 pixels, moved inside the image near its edges. The comment's table now says "image pixels", since phone captures are drawn at 3x. --- .github/scripts/compare-screens.mjs | 107 +++++++++++++++++------ .github/scripts/compare-screens.test.mjs | 60 ++++++++++++- .github/scripts/screens-comment.mjs | 21 +++-- .github/scripts/screens-comment.test.mjs | 10 +++ 4 files changed, 158 insertions(+), 40 deletions(-) diff --git a/.github/scripts/compare-screens.mjs b/.github/scripts/compare-screens.mjs index a4c0245..8c31ab3 100644 --- a/.github/scripts/compare-screens.mjs +++ b/.github/scripts/compare-screens.mjs @@ -10,8 +10,12 @@ import { decodePNG, encodePNG } from "./png.mjs"; /** A pixel changes when one of its channels differs by more than this. */ const noise = 1; -/** Points of context around a changed region, at the renders' 2× scale. */ -const padding = 24; +/** Context around a changed region, in image pixels: 24 points at 2×, 16 at 3×. */ +const padding = 48; +/** The smallest crop, so a small change still shows what's around it. */ +const smallest = { width: 480, height: 160 }; +/** A crop's most rows: a long change shows its start. */ +const tallest = 1600; /** The longest message the trusted half takes. */ const maximumMessage = 1000; @@ -19,13 +23,17 @@ const cut = (text, limit) => (text.length <= limit ? text : `${text.slice(0, lim /** Image names the bot can show, since they become file and asset names. */ const isPlainName = (name) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name) && !name.includes("..") && name.length <= 80; -function changedMask(before, after) { - const mask = new Uint8Array(before.width * before.height); - for (let index = 0; index < mask.length; index++) { - for (let channel = 0; channel < 4; channel++) { - if (Math.abs(before.rgba[index * 4 + channel] - after.rgba[index * 4 + channel]) > noise) { - mask[index] = 1; - break; +/** Which pixels of the top-left `width` by `height` of both images differ. */ +function changedMask(before, after, width, height) { + const mask = new Uint8Array(width * height); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const a = (y * before.width + x) * 4, b = (y * after.width + x) * 4; + for (let channel = 0; channel < 4; channel++) { + if (Math.abs(before.rgba[a + channel] - after.rgba[b + channel]) > noise) { + mask[y * width + x] = 1; + break; + } } } } @@ -34,14 +42,25 @@ function changedMask(before, after) { /** * How `after` differs from `before`, either of which may be null. A changed pixel with no changed - * neighbour is noise: it neither makes a screen changed nor widens its region. + * neighbour is noise: it neither makes a screen changed nor widens its region. Screens of one width + * but new heights are compared from the top: their region runs from the first change, or from where + * the shorter one ends, to the longer one's end, across the full width. A new width can't be lined up. */ export function comparePair(before, after) { if (!before) return { status: "added", changedPixels: 0, region: null }; if (!after) return { status: "removed", changedPixels: 0, region: null }; - if (before.width !== after.width || before.height !== after.height) return { status: "resized", changedPixels: 0, region: null }; - const { width, height } = before; - const mask = changedMask(before, after); + if (before.width !== after.width) return { status: "resized", changedPixels: 0, region: null }; + const width = before.width, height = Math.min(before.height, after.height); + const { changedPixels, region } = significant(changedMask(before, after, width, height), width, height); + if (before.height === after.height) { + return changedPixels ? { status: "changed", changedPixels, region } : { status: "unchanged", changedPixels: 0, region: null }; + } + const longer = Math.max(before.height, after.height), top = region ? region.y : height; + return { status: "resized", changedPixels: changedPixels + (longer - height) * width, region: { x: 0, y: top, width, height: longer - top } }; +} + +/** The changed pixels of `mask` that have a changed neighbour, and the region around them. */ +function significant(mask, width, height) { let changedPixels = 0, minX = width, minY = height, maxX = -1, maxY = -1; for (let y = 0; y < height; y++) { for (let x = 0; x < width; x++) { @@ -64,17 +83,38 @@ export function comparePair(before, after) { maxY = Math.max(maxY, y); } } - if (!changedPixels) return { status: "unchanged", changedPixels: 0, region: null }; - return { status: "changed", changedPixels, region: { x: minX, y: minY, width: maxX - minX + 1, height: maxY - minY + 1 } }; + if (!changedPixels) return { changedPixels: 0, region: null }; + return { changedPixels, region: { x: minX, y: minY, width: maxX - minX + 1, height: maxY - minY + 1 } }; } +/** `region` with `padding` around it, grown to the smallest crop, moved inside `image` and cut to it and to `tallest`. */ function padded(region, image) { - const x = Math.max(0, region.x - padding), y = Math.max(0, region.y - padding); - return { - x, y, - width: Math.min(image.width, region.x + region.width + padding) - x, - height: Math.min(image.height, region.y + region.height + padding) - y, + const span = (start, length, size, minimum) => { + let from = start - padding, to = start + length + padding; + if (to - from < minimum) { + const grow = minimum - (to - from); + from -= Math.floor(grow / 2); + to += Math.ceil(grow / 2); + } + if (from < 0) { + to -= from; + from = 0; + } + if (to > size) { + from = Math.max(0, from - (to - size)); + to = size; + } + return [from, to]; }; + const [x0, x1] = span(region.x, region.width, image.width, smallest.width); + const [y0, y1] = span(region.y, region.height, image.height, smallest.height); + return { x: x0, y: y0, width: x1 - x0, height: Math.min(y1, y0 + tallest) - y0 }; +} + +/** The crop of `image` for a screen of a new height: full width, from `padding` above the first change, at most `tallest` rows. */ +function resizedCrop(region, image) { + const y = Math.max(0, region.y - padding); + return { x: 0, y, width: image.width, height: Math.min(image.height, y + tallest) - y }; } /** The part of `image` inside `region`. */ @@ -87,18 +127,23 @@ export function crop(image, region) { return { width: region.width, height: region.height, rgba }; } -/** `after` at 30% brightness with every changed pixel red, inside `region`. */ +/** `after` inside `region`, at 30% brightness, with every changed pixel, and every row `before` doesn't have, red. */ export function differenceImage(before, after, region) { - const mask = changedMask(before, after); - const rgba = new Uint8Array(after.rgba.length); - for (let index = 0; index < mask.length; index++) { - if (mask[index]) { - rgba.set([255, 59, 48, 255], index * 4); - } else { - rgba.set([after.rgba[index * 4] * 0.3, after.rgba[index * 4 + 1] * 0.3, after.rgba[index * 4 + 2] * 0.3, 255], index * 4); + const overlap = Math.min(before.height, after.height); + const mask = changedMask(before, after, after.width, overlap); + const rgba = new Uint8Array(region.width * region.height * 4); + for (let row = 0; row < region.height; row++) { + const y = region.y + row; + for (let column = 0; column < region.width; column++) { + const x = region.x + column, source = (y * after.width + x) * 4, target = (row * region.width + column) * 4; + if (y >= overlap || mask[y * after.width + x]) { + rgba.set([255, 59, 48, 255], target); + } else { + rgba.set([after.rgba[source] * 0.3, after.rgba[source + 1] * 0.3, after.rgba[source + 2] * 0.3, 255], target); + } } } - return crop({ width: after.width, height: after.height, rgba }, region); + return { width: region.width, height: region.height, rgba }; } function read(folder, name) { @@ -139,6 +184,10 @@ export function compareDirectories(beforeDir, afterDir, outDir, { prefix, before save("before", crop(before, region)); save("after", crop(after, region)); save("diff", differenceImage(before, after, region)); + } else if (result.status === "resized" && result.region) { + save("before", crop(before, resizedCrop(result.region, before))); + save("after", crop(after, resizedCrop(result.region, after))); + save("diff", differenceImage(before, after, resizedCrop(result.region, after))); } else if (result.status === "resized") { save("before", before); save("after", after); diff --git a/.github/scripts/compare-screens.test.mjs b/.github/scripts/compare-screens.test.mjs index c620458..a8905d0 100644 --- a/.github/scripts/compare-screens.test.mjs +++ b/.github/scripts/compare-screens.test.mjs @@ -30,11 +30,21 @@ test("comparePair finds the changed region and ignores noise", () => { { status: "changed", changedPixels: 9, region: { x: 5, y: 2, width: 3, height: 3 } }); assert.equal(comparePair(base, paint(base, 12, 6, 1, 1, [200, 40, 40, 255])).status, "unchanged"); assert.equal(comparePair(base, paint(base, 0, 0, 20, 10, [41, 41, 41, 255])).status, "unchanged"); - assert.equal(comparePair(base, image(21, 10)).status, "resized"); + assert.deepEqual(comparePair(base, image(21, 10)), { status: "resized", changedPixels: 0, region: null }); assert.equal(comparePair(null, base).status, "added"); assert.equal(comparePair(base, null).status, "removed"); }); +test("comparePair compares screens of new heights from the top, to the first change", () => { + const base = image(20, 10); + // Rows added at the bottom change from where the shorter image ends. + assert.deepEqual(comparePair(base, image(20, 14)), { status: "resized", changedPixels: 80, region: { x: 0, y: 10, width: 20, height: 4 } }); + assert.deepEqual(comparePair(base, image(20, 8)), { status: "resized", changedPixels: 40, region: { x: 0, y: 8, width: 20, height: 2 } }); + // A change in the rows both have starts the region there, across the full width, down to the longer image's end. + const taller = paint(image(20, 14), 5, 2, 3, 3, [200, 40, 40, 255]); + assert.deepEqual(comparePair(base, taller), { status: "resized", changedPixels: 89, region: { x: 0, y: 2, width: 20, height: 12 } }); +}); + test("compareDirectories writes images for changed screens only, and notes what the base couldn't draw", () => { const root = mkdtempSync(join(tmpdir(), "screens-")); const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); @@ -54,9 +64,9 @@ test("compareDirectories writes images for changed screens only, and notes what assert.deepEqual(Object.keys(moved.images).sort(), ["after", "before", "diff"]); assert.deepEqual(readdirSync(out).sort(), ["scenes~moved-dark-after.png", "scenes~moved-dark-before.png", "scenes~moved-dark-diff.png", "scenes~new-dark-after.png"]); + // An image smaller than the smallest crop is shown whole. const crop = decodePNG(readFileSync(join(out, moved.images.after))); - // The 4-pixel square at x = 10 gets 24 pixels of padding, cut at the left edge: from 0 to 10 + 4 + 24. - assert.equal(crop.width, 38); + assert.deepEqual([crop.width, crop.height], [60, 30]); // The trusted half takes messages of at most 1,000 characters. assert.match(screens[1].baseError, /^draw: Unknown value y+…$/); assert.equal(screens[1].baseError.length, 1000); @@ -87,3 +97,47 @@ test("the command lines run when called through a symbolic link", () => { assert.match(run.stderr, /usage: /, script); } }); + +test("crops give a change context: 48 pixels around it, at least 480 by 160, inside the image", () => { + const root = mkdtempSync(join(tmpdir(), "screens-")); + const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); + for (const folder of [before, after]) mkdirSync(folder, { recursive: true }); + const base = image(600, 300); + writeFileSync(join(before, "middle-dark.png"), encodePNG(base)); + writeFileSync(join(after, "middle-dark.png"), encodePNG(paint(base, 300, 150, 4, 4, [255, 255, 255, 255]))); + writeFileSync(join(before, "edge-dark.png"), encodePNG(base)); + writeFileSync(join(after, "edge-dark.png"), encodePNG(paint(base, 2, 150, 4, 4, [255, 255, 255, 255]))); + const { screens } = compareDirectories(before, after, out, { prefix: "scenes" }); + const size = (file) => { const { width, height } = decodePNG(readFileSync(join(out, file))); return [width, height]; }; + for (const screen of screens) { + for (const kind of ["before", "after", "diff"]) assert.deepEqual(size(screen.images[kind]), [480, 160], `${screen.id} ${kind}`); + } + // The edge's crop moves inside the image rather than shrinking: its change is at its left. + const edge = decodePNG(readFileSync(join(out, screens[0].images.after))); + assert.deepEqual([...edge.rgba.subarray((80 * 480 + 2) * 4, (80 * 480 + 2) * 4 + 4)], [255, 255, 255, 255]); +}); + +test("a screen of a new height is cropped from its first change, full width", () => { + const root = mkdtempSync(join(tmpdir(), "screens-")); + const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); + for (const folder of [before, after]) mkdirSync(folder, { recursive: true }); + writeFileSync(join(before, "page-dark.png"), encodePNG(image(100, 50))); + writeFileSync(join(after, "page-dark.png"), encodePNG(image(100, 80))); + writeFileSync(join(before, "tall-dark.png"), encodePNG(image(10, 2000))); + writeFileSync(join(after, "tall-dark.png"), encodePNG(paint(image(10, 2100), 0, 100, 10, 2, [255, 255, 255, 255]))); + writeFileSync(join(before, "wide-dark.png"), encodePNG(image(100, 50))); + writeFileSync(join(after, "wide-dark.png"), encodePNG(image(120, 50))); + const { screens } = compareDirectories(before, after, out, { prefix: "site" }); + const size = (file) => { const { width, height } = decodePNG(readFileSync(join(out, file))); return [width, height]; }; + const [page, tall, wide] = screens; + // Rows added from y = 50: the crops start 48 rows above, and run to each image's end. + assert.deepEqual([page.status, size(page.images.before), size(page.images.after), size(page.images.diff)], + ["resized", [100, 48], [100, 78], [100, 78]]); + const diff = decodePNG(readFileSync(join(out, page.images.diff))); + assert.deepEqual([...diff.rgba.subarray(0, 4)], [12, 12, 12, 255]); + assert.deepEqual([...diff.rgba.subarray((60 * 100) * 4, (60 * 100) * 4 + 4)], [255, 59, 48, 255]); + // A long change shows its first 1,600 rows. + assert.deepEqual([size(tall.images.before), size(tall.images.after)], [[10, 1600], [10, 1600]]); + // A new width can't be lined up: both images are shown whole. + assert.deepEqual([wide.region, Object.keys(wide.images).sort(), size(wide.images.after)], [null, ["after", "before"], [120, 50]]); +}); diff --git a/.github/scripts/screens-comment.mjs b/.github/scripts/screens-comment.mjs index c584226..dc7e0cb 100644 --- a/.github/scripts/screens-comment.mjs +++ b/.github/scripts/screens-comment.mjs @@ -21,8 +21,11 @@ const maximumComment = 60000; const maximumRows = 100; const maximumListed = 20; const statuses = ["unchanged", "changed", "resized", "added", "removed"]; -/** The images a screen of each status comes with. */ -const imagesOf = { unchanged: [], changed: ["after", "before", "diff"], resized: ["after", "before"], added: ["after"], removed: ["before"] }; +/** The images a screen comes with: a resized screen lined up from the top has a region and a difference too. */ +const imagesOf = (status, region) => ({ unchanged: [], changed: ["after", "before", "diff"], added: ["after"], removed: ["before"], + resized: region ? ["after", "before", "diff"] : ["after", "before"] })[status]; +const isRegion = (region) => hasOnly(region, ["x", "y", "width", "height"]) && isCount(region.x) && isCount(region.y) + && isCount(region.width, 1) && isCount(region.height, 1); const isObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value); const hasOnly = (value, keys) => isObject(value) && Object.keys(value).every((key) => keys.includes(key)); @@ -85,11 +88,9 @@ function readScreen(screen, where, files, ids, prefixes, problems) { } if (!isCount(screen.changedPixels)) problems.push(`${where}: changedPixels isn't a count.`); const { region } = screen; - const regionFits = screen.status === "changed" - ? hasOnly(region, ["x", "y", "width", "height"]) && isCount(region.x) && isCount(region.y) && isCount(region.width, 1) && isCount(region.height, 1) - : region === null; + const regionFits = screen.status === "changed" ? isRegion(region) : screen.status === "resized" ? region === null || isRegion(region) : region === null; if (!regionFits) problems.push(`${where}: its region doesn't fit its status.`); - const kinds = imagesOf[screen.status]; + const kinds = imagesOf(screen.status, regionFits && region !== null); if (!hasOnly(screen.images, kinds) || Object.keys(screen.images).length !== kinds.length) { problems.push(`${where}: a screen that is ${screen.status} comes with ${kinds.length ? kinds.join(", ") : "no images"}.`); } else { @@ -182,7 +183,11 @@ function describe(screen) { } case "added": return ["new", "the whole screen"]; case "removed": return ["removed", "the whole screen"]; - case "resized": return ["new size", "the whole screen"]; + case "resized": { + if (!screen.region) return ["new size", "the whole screen"]; + const { x, y, width, height } = screen.region; + return [`${screen.changedPixels.toLocaleString("en-US")} pixels, new size`, `${x},${y} · ${width}×${height}`]; + } default: return ["", ""]; } } @@ -224,7 +229,7 @@ export function commentBody({ baseRef = "main", base, head, screens, failed = [] const lines = [marker, "### Before and after", "", `${code(baseRef, 100)} at ${base.slice(0, 7)} → this pull request at ${head.slice(0, 7)}` + ` · ${plural(screens.length, "screen")} drawn in one CI job · ${changed.length} changed`, ""]; if (changed.length > 0) { - lines.push("| Screen | Change | Where (pixels at 2×) |", "| --- | --- | --- |"); + lines.push("| Screen | Change | Where (image pixels) |", "| --- | --- | --- |"); for (const screen of changed.slice(0, maximumRows)) lines.push(`| ${code(screen.id)} | ${describe(screen).join(" | ")} |`); if (changed.length > maximumRows) lines.push("", `${changed.length - maximumRows} more changed screens aren't in the table.`); lines.push("", ...sections.slice(0, shown)); diff --git a/.github/scripts/screens-comment.test.mjs b/.github/scripts/screens-comment.test.mjs index b532661..5e8f563 100644 --- a/.github/scripts/screens-comment.test.mjs +++ b/.github/scripts/screens-comment.test.mjs @@ -70,6 +70,15 @@ test("readManifest rejects what compare-screens doesn't write", () => { assert.deepEqual([missing.ok, missing.problems], [false, ["Screen 1: its diff image isn't in the artifact."]]); }); +test("readManifest takes resized screens with or without a region", () => { + const names = ["site~a-after.png", "site~a-before.png", "site~a-diff.png"]; + const lined = { ...screen("site/a", "resized", { after: names[0], before: names[1], diff: names[2] }), region: { x: 0, y: 2, width: 20, height: 12 } }; + const whole = screen("site/a", "resized", { after: names[0], before: names[1] }); + for (const entry of [lined, whole]) assert.equal(readManifest(JSON.stringify({ screens: [entry], failed: [] }), names).ok, true); + const missing = { ...lined, images: { after: names[0], before: names[1] } }; + assert.equal(readManifest(JSON.stringify({ screens: [missing], failed: [] }), names).ok, false); +}); + test("readManifest takes only the ids an artifact may hold", () => { const names = ["docs~a-after.png"]; const text = JSON.stringify({ screens: [screen("docs/a", "added", { after: names[0] })], failed: [{ id: "docs", error: "x" }] }); @@ -148,6 +157,7 @@ test("commentBody shows changed screens, folds the rest and keeps its data", () const body = commentBody({ baseRef: "main", base: sha("1"), head: sha("a"), screens, failed: [], imageURL: urls, review, data: data() }); assert.match(body, /^/); assert.match(body, /`main` at 1111111 → this pull request at aaaaaaa/); + assert.match(body, /\| Screen \| Change \| Where \(image pixels\) \|/); assert.match(body, /https:\/\/example\.test\/b\.png/); assert.match(body, /https:\/\/example\.test\/d\.png/); assert.match(body, /The base couldn't draw this scene: `draw: Unknown value`/); From 4ba28afcd70ed95804b012f538f5e9e4db331fee Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 17:08:36 +0100 Subject: [PATCH 13/25] ci: draw screens when the examples the scenes read change A scene can draw a configuration from examples/, as the Essentials clock popup does, but a change there skipped Draw screens and the review said there were no screens to compare. examples/ now calls for screens in the Check revision job and in the bot. --- .github/scripts/screens-bot.mjs | 2 +- .github/scripts/screens-bot.test.mjs | 8 ++++++++ .github/workflows/ci.yml | 7 ++++--- CONTRIBUTING.md | 4 ++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs index 0f8a05e..d23cae5 100644 --- a/.github/scripts/screens-bot.mjs +++ b/.github/scripts/screens-bot.mjs @@ -23,7 +23,7 @@ const broken = ["failure", "cancelled", "timed_out", "missing"]; * files from GitHub, so a run whose job graph skips or drops a screens job can't pass as having none. */ export const changePatterns = { - app: /^(Sources\/|Resources\/|Package\.(swift|resolved)$|scripts\/|scenes\/|\.github\/workflows\/ci\.yml$|\.github\/scripts\/(png|compare-screens)\.mjs$)/, + app: /^(Sources\/|Resources\/|Package\.(swift|resolved)$|scripts\/|scenes\/|examples\/|\.github\/workflows\/ci\.yml$|\.github\/scripts\/(png|compare-screens)\.mjs$)/, site: /^(site\/|\.github\/workflows\/ci\.yml$|\.github\/scripts\/(png|compare-screens|capture-site)\.mjs$)/, }; diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs index 43d9315..357db14 100644 --- a/.github/scripts/screens-bot.test.mjs +++ b/.github/scripts/screens-bot.test.mjs @@ -141,6 +141,14 @@ test("findRun takes only runs of the CI workflow file", async () => { assert.equal(await findRun({ github, context: runContext({ path: ".github/workflows/other.yml" }) }), null); }); +test("changePatterns call for screens when the files the scenes read change", () => { + for (const file of ["Sources/StableBar/BarView.swift", "Resources/config.json", "scenes/starter-bar.json", "examples/essentials.json"]) { + assert.equal(changePatterns.app.test(file), true, file); + } + assert.equal(changePatterns.app.test("docs/README.md"), false); + assert.equal(changePatterns.site.test("site/index.html"), true); +}); + test("changePatterns match the files CI's Check revision job draws screens for", () => { const workflow = readFileSync(new URL("../workflows/ci.yml", import.meta.url), "utf8"); const found = Object.fromEntries([...workflow.matchAll(/grep -qE '([^']+)' <<< "\$changed"; then\n\s+echo "(\w+)=true"/g)] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3d2940..155d73b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,8 +53,9 @@ jobs: persist-credentials: false # A pull request compiles the release and draws the docs images only when it changes what they # build from: the files scripts/unchanged-app.sh compares, or this workflow. Packaging reads - # only the scripts and resources. The before/after screens are drawn when the app, the scenes - # or the comparison change, and the website's when the website or its capture change. + # only the scripts and resources. The before/after screens are drawn when the app, the scenes, + # the examples they read or the comparison change, and the website's when the website or its + # capture change. # HEAD merges the pull request into its base, HEAD^. - name: Find what the pull request changes id: changes @@ -67,7 +68,7 @@ jobs: if grep -qE '^(Resources/|scripts/|\.github/workflows/ci\.yml$)' <<< "$changed"; then echo "packaging=true" >> "$GITHUB_OUTPUT" fi - if grep -qE '^(Sources/|Resources/|Package\.(swift|resolved)$|scripts/|scenes/|\.github/workflows/ci\.yml$|\.github/scripts/(png|compare-screens)\.mjs$)' <<< "$changed"; then + if grep -qE '^(Sources/|Resources/|Package\.(swift|resolved)$|scripts/|scenes/|examples/|\.github/workflows/ci\.yml$|\.github/scripts/(png|compare-screens)\.mjs$)' <<< "$changed"; then echo "screens=true" >> "$GITHUB_OUTPUT" fi if grep -qE '^(site/|\.github/workflows/ci\.yml$|\.github/scripts/(png|compare-screens|capture-site)\.mjs$)' <<< "$changed"; then diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 198cbbe..73d73c3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -90,7 +90,7 @@ Reviews are requested automatically from the people listed in [CODEOWNERS](.gith Pull requests run `swift test` on macOS and check that every commit message, and the pull request title, follows [Conventional Commits](https://www.conventionalcommits.org). Squash merges use the title as the commit message. The tests load every configuration, module and package in `examples`, and the **Test** job parses each script in `scripts` and `examples` with the shell its first line names. The **Test pull request checks** job runs `node --test` on the scripts in `.github/scripts`: the logic of `Linked issue`, `Description` and the [before and after screens](#before-and-after-screens). -A pull request that changes `Sources`, `Resources`, `Package.swift`, `Package.resolved`, `scripts` or the CI workflow also compiles the release and draws the [docs images](#docs-images), as `main` does. If it changes `Resources`, `scripts` or the workflow, the **Package** job builds the DMG as a beta would, ad-hoc signed because pull requests get no Developer ID, and checks its signature, version and starter configuration. **Draw screens** runs when the app, `scenes` or the comparison scripts change, and **Draw website screens** when `site` changes. +A pull request that changes `Sources`, `Resources`, `Package.swift`, `Package.resolved`, `scripts` or the CI workflow also compiles the release and draws the [docs images](#docs-images), as `main` does. If it changes `Resources`, `scripts` or the workflow, the **Package** job builds the DMG as a beta would, ad-hoc signed because pull requests get no Developer ID, and checks its signature, version and starter configuration. **Draw screens** runs when the app, `scenes`, the `examples` scenes read or the comparison scripts change, and **Draw website screens** when `site` changes. [semantic-release](https://github.com/semantic-release/semantic-release) reads the commits since the last stable release and decides the next version: @@ -145,7 +145,7 @@ A change you can see extends the scene that shows its area, or adds one when non On a pull request, CI draws the screens twice, from the pull request and from its base, and a bot posts what changed. -- **Draw screens** draws the [docs images](#docs-images) and the [scenes](#scenes) when the app, the scenes or the comparison scripts change. Both sides draw the pull request's scene files, each beside its own resources and examples, so a new scene shows how `main` draws it too. When the app didn't change, the pull request's app draws both sides. +- **Draw screens** draws the [docs images](#docs-images) and the [scenes](#scenes) when the app, the scenes, the examples they read or the comparison scripts change. Both sides draw the pull request's scene files, each beside its own resources and examples, so a new scene shows how `main` draws it too. When the app didn't change, the pull request's app draws both sides. - **Draw website screens** captures the four pages of `site` at desktop (1440×900) and phone (390×844) size, in light and dark mode, when the website changes. The capture answers requests from the folder and fetches nothing else, seeds `Math.random` and stops the page's clock at the scenes' date, so two captures of one commit match. Both sides are drawn in one job on one machine. A pixel counts as changed when a color channel moves by more than 1 and a pixel next to it changed too. From 2db63e519a80f5a3a78554cd717829da180b4def Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 17:10:33 +0100 Subject: [PATCH 14/25] ci: upload only images the bot decoded and encoded itself The artifact check looked at a PNG's first eight bytes, so a file holding only the signature, or anything after it, could be uploaded as a screen. The bot now decodes every image its screens name, within limits checked before anything is inflated, and uploads its own encoding of the pixels. The fingerprint hashes the pixels too, so an approval holds whatever encoder drew them. decodePNG refuses images over 20,000 by 100,000 or 80 million pixels, chunks cut short, and image data that inflates past the image's size. --- .github/scripts/png.mjs | 20 +++++++++++--- .github/scripts/png.test.mjs | 21 +++++++++++++++ .github/scripts/screens-bot.mjs | 35 +++++++++++++++++------- .github/scripts/screens-bot.test.mjs | 40 ++++++++++++++++++++++++++-- 4 files changed, 102 insertions(+), 14 deletions(-) diff --git a/.github/scripts/png.mjs b/.github/scripts/png.mjs index 8d332e6..9f39849 100644 --- a/.github/scripts/png.mjs +++ b/.github/scripts/png.mjs @@ -3,6 +3,8 @@ import zlib from "node:zlib"; const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); +/** The largest image decodePNG takes, before inflating anything: a long website page is about 2880 by 12000. */ +const limits = { maxWidth: 20000, maxHeight: 100000, maxPixels: 80_000_000 }; function paeth(a, b, c) { const p = a + b - c, pa = Math.abs(p - a), pb = Math.abs(p - b), pc = Math.abs(p - c); @@ -18,14 +20,18 @@ function chunk(type, data) { return Buffer.concat([length, body, crc]); } -/** The pixels of a PNG as `{width, height, rgba}`, every pixel four bytes. */ -export function decodePNG(input) { +/** + * The pixels of a PNG as `{width, height, rgba}`, every pixel four bytes. It throws for an image larger + * than its limits, a chunk cut short, or image data that doesn't inflate to exactly the image's size. + */ +export function decodePNG(input, { maxWidth = limits.maxWidth, maxHeight = limits.maxHeight, maxPixels = limits.maxPixels } = {}) { const buffer = Buffer.isBuffer(input) ? input : Buffer.from(input); if (buffer.length < 8 || !buffer.subarray(0, 8).equals(signature)) throw new Error("Unsupported PNG: no PNG signature."); let offset = 8, width = 0, height = 0, bitDepth = 0, colorType = -1, interlace = -1; const data = []; while (offset + 8 <= buffer.length) { const length = buffer.readUInt32BE(offset); + if (offset + 12 + length > buffer.length) throw new Error("Unsupported PNG: a chunk is cut short."); const type = buffer.toString("latin1", offset + 4, offset + 8); const body = buffer.subarray(offset + 8, offset + 8 + length); if (type === "IHDR") { @@ -44,9 +50,17 @@ export function decodePNG(input) { if (!width || !height || bitDepth !== 8 || (colorType !== 2 && colorType !== 6) || interlace !== 0) { throw new Error("Unsupported PNG: only 8-bit RGB or RGBA without interlacing."); } + if (width > maxWidth || height > maxHeight || width * height > maxPixels) { + throw new Error(`Unsupported PNG: ${width}×${height} is larger than it takes.`); + } const channels = colorType === 6 ? 4 : 3; const stride = width * channels; - const raw = zlib.inflateSync(Buffer.concat(data)); + let raw; + try { + raw = zlib.inflateSync(Buffer.concat(data), { maxOutputLength: height * (stride + 1) }); + } catch (error) { + throw new Error(`Unsupported PNG: its image data doesn't inflate to its size (${error.code ?? error.message}).`); + } if (raw.length < height * (stride + 1)) throw new Error("Unsupported PNG: the image data is cut short."); const pixels = new Uint8Array(height * stride); for (let y = 0; y < height; y++) { diff --git a/.github/scripts/png.test.mjs b/.github/scripts/png.test.mjs index c8cacf3..2e18905 100644 --- a/.github/scripts/png.test.mjs +++ b/.github/scripts/png.test.mjs @@ -62,3 +62,24 @@ test("decodePNG reads RGB scanlines with every filter as opaque RGBA", () => { test("decodePNG rejects what isn't an 8-bit RGB or RGBA PNG", () => { assert.throws(() => decodePNG(Buffer.from("not a png")), /Unsupported PNG/); }); + +test("decodePNG refuses images larger than its limits, cut short, or inflating past their size", () => { + const header = (width, height) => { + const data = Buffer.alloc(13); + data.writeUInt32BE(width, 0); + data.writeUInt32BE(height, 4); + data[8] = 8; + data[9] = 6; + return chunk("IHDR", data); + }; + const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); + const png = (width, height, raw) => Buffer.concat([signature, header(width, height), chunk("IDAT", zlib.deflateSync(raw)), chunk("IEND", Buffer.alloc(0))]); + // Its header alone says too much: nothing is inflated. + assert.throws(() => decodePNG(png(100000, 100000, Buffer.alloc(10))), /larger than/); + assert.throws(() => decodePNG(png(4, 4, Buffer.alloc(4 * 17)), { maxPixels: 15 }), /larger than/); + // A chunk that claims more bytes than the file has. + const whole = encodePNG({ width: 2, height: 2, rgba: new Uint8Array(16) }); + assert.throws(() => decodePNG(whole.subarray(0, whole.length - 20)), /cut short/); + // Image data that inflates to far more than a 2 by 2 image holds. + assert.throws(() => decodePNG(png(2, 2, Buffer.alloc(50 * 1024 * 1024))), /Unsupported PNG/); +}); diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs index d23cae5..ac55e7f 100644 --- a/.github/scripts/screens-bot.mjs +++ b/.github/scripts/screens-bot.mjs @@ -7,6 +7,7 @@ import { createHash } from "node:crypto"; import { join } from "node:path"; import { validateArtifact, readManifest, fingerprint, summary, reviewState, commentBody, withReview, noScreensBody, readData, findScreensComment, assetName, releaseTag, staleAssets, pullRequestAssets } from "./screens-comment.mjs"; +import { decodePNG, encodePNG } from "./png.mjs"; const statusContext = "Visual review"; const approval = "visual-ok"; @@ -61,7 +62,14 @@ export async function findRun({ github, context }) { return null; } -/** The screens, failures and files of a downloaded artifact folder, checked as untrusted data. */ +/** A digest of an image's pixels, the same for any encoding of them. */ +const pixelDigest = ({ width, height, rgba }) => createHash("sha256").update(`${width}x${height}\n`).update(rgba).digest("hex"); + +/** + * The screens, failures and images of a downloaded artifact folder, checked as untrusted data. Every + * image its screens name is decoded within limits and encoded again, so only the bot's own PNGs are + * uploaded: `files` maps each name to `{data, digest}`, the digest being of the pixels. + */ export function readArtifact(folder, prefixes) { if (!existsSync(folder)) return { ok: false, problems: ["It couldn't be downloaded."] }; const files = readdirSync(folder).map((name) => { @@ -83,7 +91,18 @@ export function readArtifact(folder, prefixes) { const manifest = files.find((file) => file.name === "manifest.json"); if (!manifest) return { ok: false, problems: ["It has no manifest.json."] }; const read = readManifest(readFileSync(manifest.path, "utf8"), files.map((file) => file.name), prefixes); - return read.ok ? { ...read, files: new Map(files.map((file) => [file.name, file.path])) } : read; + if (!read.ok) return read; + const images = new Map(); + for (const name of new Set(read.screens.flatMap((screen) => Object.values(screen.images)))) { + let image; + try { + image = decodePNG(readFileSync(join(folder, name))); + } catch (error) { + return { ok: false, problems: [`\`${name}\` isn't a PNG the bot can read: ${error.message}`] }; + } + images.set(name, { data: encodePNG(image), digest: pixelDigest(image) }); + } + return { ...read, files: images }; } const comments = (github, owner, repo, number) => @@ -115,7 +134,7 @@ async function previewRelease({ github, owner, repo, target }) { } } -/** Uploads `images`, [file, path] pairs, and returns each file's URL as GitHub answers it. A rerun of a head replaces its images. */ +/** Uploads `images`, [file, PNG data] pairs, and returns each file's URL as GitHub answers it. A rerun of a head replaces its images. */ async function upload({ github, owner, repo, number, head, target, images }) { const urls = new Map(); if (images.length === 0) return urls; @@ -124,8 +143,7 @@ async function upload({ github, owner, repo, number, head, target, images }) { for (const asset of pullRequestAssets(assets, number).filter((entry) => entry.name.startsWith(assetName(number, head, "")))) { await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id }); } - for (const [file, path] of images) { - const data = readFileSync(path); + for (const [file, data] of images) { const { data: asset } = await github.rest.repos.uploadReleaseAsset({ owner, repo, release_id: release.id, name: assetName(number, head, file), data, headers: { "content-type": "image/png", "content-length": data.length } }); urls.set(file, asset.browser_download_url); @@ -182,7 +200,7 @@ function gather(run, folders) { } screens.push(...read.screens); failed.push(...read.failed); - for (const [name, path] of read.files) files.set(name, path); + for (const [name, image] of read.files) files.set(name, image); } return { screens, failed, files, ranScreens }; } @@ -208,9 +226,8 @@ export async function publishRun({ github, context, run, folders }) { } else { const shown = screens.filter((screen) => screen.status !== "unchanged").flatMap((screen) => Object.values(screen.images)); const urls = await upload({ github, owner, repo, number: run.number, head: run.head, target: context.payload.repository?.default_branch, - images: shown.map((file) => [file, files.get(file)]) }); - const digest = (file) => createHash("sha256").update(readFileSync(files.get(file))).digest("hex"); - const print = fingerprint({ screens, failed }, digest); + images: shown.map((file) => [file, files.get(file).data]) }); + const print = fingerprint({ screens, failed }, (file) => files.get(file).digest); const counts = summary(screens, failed); review = reviewState({ ranScreens: true, ...counts, labels: await labelsOf(github, owner, repo, run.number), approved: previous?.approved ?? null, fingerprint: print }); diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs index 357db14..aa40c79 100644 --- a/.github/scripts/screens-bot.test.mjs +++ b/.github/scripts/screens-bot.test.mjs @@ -3,7 +3,8 @@ import assert from "node:assert/strict"; import { mkdtempSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { encodePNG } from "./png.mjs"; +import zlib from "node:zlib"; +import { decodePNG, encodePNG } from "./png.mjs"; import { main as compare } from "./compare-screens.mjs"; import { readData, releaseTag } from "./screens-comment.mjs"; import { changePatterns, deleteScreens, failRun, findRun, publishRun, updateReview } from "./screens-bot.mjs"; @@ -93,7 +94,7 @@ function fakeGitHub({ pulls = [], comments = [], labels = [], releases = [], job const renamed = name.replaceAll("~", "."); const asset = { id: state.next++, name: renamed, browser_download_url: `https://example.test/download/${renamed}` }; state.releases.find((release) => release.id === release_id).assets.push(asset); - state.uploads.push({ name, size: data.length, type: headers["content-type"] }); + state.uploads.push({ name, size: data.length, type: headers["content-type"], data: Buffer.from(data) }); return { data: asset }; }, deleteReleaseAsset: async ({ asset_id }) => { @@ -269,6 +270,41 @@ test("publishRun reports failed jobs and refuses unusable artifacts", async () = assert.equal(state.uploads.length, 0); }); +/** `file` with a text chunk after its header: other bytes, the same pixels. */ +function withText(file) { + const png = readFileSync(file); + const body = Buffer.concat([Buffer.from("tEXt", "latin1"), Buffer.from("Comment\0from a pull request", "latin1")]); + const length = Buffer.alloc(4); + length.writeUInt32BE(body.length - 4); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(zlib.crc32(body)); + writeFileSync(file, Buffer.concat([png.subarray(0, 33), length, body, crc, png.subarray(33)])); +} + +test("publishRun uploads its own encoding of the pixels, and fingerprints the pixels", async () => { + const { github, state } = fakeGitHub({ pulls: [pr()] }); + const plain = artifact(); + await publishRun({ github, context: runContext(), run: ran(), folders: { app: plain } }); + const first = readData(state.comments[0].body).fingerprint; + const uploaded = state.uploads.find((upload) => upload.name.endsWith("after.png")); + assert.deepEqual(uploaded.data, encodePNG(decodePNG(readFileSync(join(plain, "docs~hero-dark-after.png"))))); + const texted = artifact(); + for (const kind of ["before", "after", "diff"]) withText(join(texted, `docs~hero-dark-${kind}.png`)); + await publishRun({ github, context: runContext(), run: ran(), folders: { app: texted } }); + assert.equal(readData(state.comments[0].body).fingerprint, first); + assert.ok(!state.uploads.at(-1).data.includes("from a pull request")); +}); + +test("publishRun refuses an image it can't decode", async () => { + const { github, state } = fakeGitHub({ pulls: [pr()] }); + const folder = artifact(); + writeFileSync(join(folder, "docs~hero-dark-diff.png"), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])); + const review = await publishRun({ github, context: runContext(), run: ran(), folders: { app: folder } }); + assert.equal(review.state, "failure"); + assert.match(state.comments[0].body, /'docs~hero-dark-diff\.png' isn't a PNG the bot can read/); + assert.equal(state.uploads.length, 0); +}); + test("failRun marks the head when the bot itself fails", async () => { const { github, state } = fakeGitHub(); await failRun({ github, context: runContext(), run: ran() }); From 0b966b09b6bb2d08e02fbccb1048aa5a1b254b7e Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 17:12:16 +0100 Subject: [PATCH 15/25] ci: bind visual-ok to the whole after image, not only its crop A crop shows at most 1,600 rows, so a second change further down a long page could leave the crop, and so the fingerprint and visual-ok, as they were. Changed and resized screens now carry a digest of their whole after image, and the fingerprint includes it beside the bot's own digest of the crop. The pull request's CI computes that digest, as it draws every screen. A comment whose crop stops short says it shows the first 1,600 rows. --- .github/scripts/compare-screens.mjs | 22 +++++++++++++------ .github/scripts/compare-screens.test.mjs | 27 ++++++++++++++++++++++-- .github/scripts/screens-comment.mjs | 18 +++++++++++----- .github/scripts/screens-comment.test.mjs | 15 +++++++++++-- 4 files changed, 67 insertions(+), 15 deletions(-) diff --git a/.github/scripts/compare-screens.mjs b/.github/scripts/compare-screens.mjs index 8c31ab3..1feb271 100644 --- a/.github/scripts/compare-screens.mjs +++ b/.github/scripts/compare-screens.mjs @@ -1,8 +1,10 @@ // Compares the screens drawn from a pull request (after) with those drawn from its base (before): // node compare-screens.mjs BEFORE AFTER OUT --prefix docs [--before-report scenes.json] [--after-report scenes.json] // It writes cropped before, after and difference images of each changed screen to OUT, and adds the -// screens to OUT/manifest.json: {screens: [{id, status, changedPixels, region, images, baseError?}], failed: [{id, error}]}. +// screens to OUT/manifest.json: {screens: [{id, status, changedPixels, region, images, afterDigest?, +// truncated?, baseError?}], failed: [{id, error}]}. // The trusted half of the bot checks that manifest strictly: see readManifest in screens-comment.mjs. +import { createHash } from "node:crypto"; import { existsSync, readFileSync, readdirSync, realpathSync, writeFileSync, mkdirSync } from "node:fs"; import { join, basename } from "node:path"; import { pathToFileURL } from "node:url"; @@ -20,6 +22,8 @@ const tallest = 1600; const maximumMessage = 1000; const cut = (text, limit) => (text.length <= limit ? text : `${text.slice(0, limit - 1)}…`); +/** A digest of an image's pixels: a changed or resized screen carries its whole after image's, since its crop may show only part. */ +const digestOf = ({ width, height, rgba }) => createHash("sha256").update(`${width}x${height}\n`).update(rgba).digest("hex"); /** Image names the bot can show, since they become file and asset names. */ const isPlainName = (name) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name) && !name.includes("..") && name.length <= 80; @@ -108,13 +112,13 @@ function padded(region, image) { }; const [x0, x1] = span(region.x, region.width, image.width, smallest.width); const [y0, y1] = span(region.y, region.height, image.height, smallest.height); - return { x: x0, y: y0, width: x1 - x0, height: Math.min(y1, y0 + tallest) - y0 }; + return { x: x0, y: y0, width: x1 - x0, height: Math.min(y1, y0 + tallest) - y0, cut: y1 > y0 + tallest }; } /** The crop of `image` for a screen of a new height: full width, from `padding` above the first change, at most `tallest` rows. */ function resizedCrop(region, image) { const y = Math.max(0, region.y - padding); - return { x: 0, y, width: image.width, height: Math.min(image.height, y + tallest) - y }; + return { x: 0, y, width: image.width, height: Math.min(image.height, y + tallest) - y, cut: image.height > y + tallest }; } /** The part of `image` inside `region`. */ @@ -175,6 +179,7 @@ export function compareDirectories(beforeDir, afterDir, outDir, { prefix, before const result = comparePair(before, after); const file = (kind) => `${id.replaceAll("/", "~")}-${kind}.png`; const images = {}; + let truncated = false; const save = (kind, image) => { writeFileSync(join(outDir, file(kind)), encodePNG(image)); images[kind] = file(kind); @@ -184,10 +189,13 @@ export function compareDirectories(beforeDir, afterDir, outDir, { prefix, before save("before", crop(before, region)); save("after", crop(after, region)); save("diff", differenceImage(before, after, region)); + truncated = region.cut; } else if (result.status === "resized" && result.region) { - save("before", crop(before, resizedCrop(result.region, before))); - save("after", crop(after, resizedCrop(result.region, after))); - save("diff", differenceImage(before, after, resizedCrop(result.region, after))); + const beforeCrop = resizedCrop(result.region, before), afterCrop = resizedCrop(result.region, after); + save("before", crop(before, beforeCrop)); + save("after", crop(after, afterCrop)); + save("diff", differenceImage(before, after, afterCrop)); + truncated = beforeCrop.cut || afterCrop.cut; } else if (result.status === "resized") { save("before", before); save("after", after); @@ -197,6 +205,8 @@ export function compareDirectories(beforeDir, afterDir, outDir, { prefix, before save("before", before); } const screen = { id, ...result, images }; + if (result.status === "changed" || result.status === "resized") screen.afterDigest = digestOf(after); + if (truncated) screen.truncated = true; const error = beforeErrors[sceneName(name)] ?? beforeError; if (result.status === "added" && error) screen.baseError = cut(error, maximumMessage); return screen; diff --git a/.github/scripts/compare-screens.test.mjs b/.github/scripts/compare-screens.test.mjs index a8905d0..6a750af 100644 --- a/.github/scripts/compare-screens.test.mjs +++ b/.github/scripts/compare-screens.test.mjs @@ -136,8 +136,31 @@ test("a screen of a new height is cropped from its first change, full width", () const diff = decodePNG(readFileSync(join(out, page.images.diff))); assert.deepEqual([...diff.rgba.subarray(0, 4)], [12, 12, 12, 255]); assert.deepEqual([...diff.rgba.subarray((60 * 100) * 4, (60 * 100) * 4 + 4)], [255, 59, 48, 255]); - // A long change shows its first 1,600 rows. - assert.deepEqual([size(tall.images.before), size(tall.images.after)], [[10, 1600], [10, 1600]]); + // A long change shows its first 1,600 rows, and says so. + assert.deepEqual([size(tall.images.before), size(tall.images.after), tall.truncated], [[10, 1600], [10, 1600], true]); + assert.equal(page.truncated, undefined); // A new width can't be lined up: both images are shown whole. assert.deepEqual([wide.region, Object.keys(wide.images).sort(), size(wide.images.after)], [null, ["after", "before"], [120, 50]]); }); + +test("changed and resized screens carry a digest of their whole after image", () => { + const root = mkdtempSync(join(tmpdir(), "screens-")); + const before = join(root, "before"); + mkdirSync(before); + const page = image(10, 2000); + writeFileSync(join(before, "page-dark.png"), encodePNG(page)); + // Both changes span rows 100 to 1901, and the crop shows rows 52 to 1651. + const tall = paint(paint(page, 0, 100, 10, 2, [255, 255, 255, 255]), 0, 1900, 10, 2, [255, 255, 255, 255]); + const runs = [tall, paint(tall, 0, 1800, 10, 2, [255, 255, 255, 255])].map((after) => { + const folder = join(root, `after-${Math.random()}`), out = join(root, `out-${Math.random()}`); + mkdirSync(folder); + writeFileSync(join(folder, "page-dark.png"), encodePNG(after)); + const [screen] = compareDirectories(before, folder, out, { prefix: "site" }).screens; + return { screen, crop: readFileSync(join(out, screen.images.after)) }; + }); + // A change inside the region but past the crop leaves the crop as it was, but not the digest. + assert.deepEqual(runs[0].crop, runs[1].crop); + assert.equal(runs[0].screen.truncated, true); + assert.match(runs[0].screen.afterDigest, /^[0-9a-f]{64}$/); + assert.notEqual(runs[0].screen.afterDigest, runs[1].screen.afterDigest); +}); diff --git a/.github/scripts/screens-comment.mjs b/.github/scripts/screens-comment.mjs index dc7e0cb..1a7721e 100644 --- a/.github/scripts/screens-comment.mjs +++ b/.github/scripts/screens-comment.mjs @@ -75,7 +75,7 @@ export function validateArtifact(files) { } function readScreen(screen, where, files, ids, prefixes, problems) { - if (!hasOnly(screen, ["id", "status", "changedPixels", "region", "images", "baseError"])) { + if (!hasOnly(screen, ["id", "status", "changedPixels", "region", "images", "afterDigest", "truncated", "baseError"])) { problems.push(`${where} isn't a screen as compare-screens.mjs writes it.`); return null; } @@ -101,10 +101,15 @@ function readScreen(screen, where, files, ids, prefixes, problems) { } } if (screen.baseError !== undefined && !isMessage(screen.baseError)) problems.push(`${where}: baseError isn't a message of at most ${maximumMessage} characters.`); + const cropped = screen.status === "changed" || screen.status === "resized"; + if (cropped ? !isHex(screen.afterDigest, 64) : screen.afterDigest !== undefined) problems.push(`${where}: afterDigest doesn't fit its status.`); + if (screen.truncated !== undefined && typeof screen.truncated !== "boolean") problems.push(`${where}: truncated isn't true or false.`); return { id: screen.id, status: screen.status, changedPixels: screen.changedPixels, region: regionFits && region !== null ? { x: region.x, y: region.y, width: region.width, height: region.height } : null, images: Object.fromEntries(kinds.map((kind) => [kind, screen.images?.[kind]])), + ...(cropped ? { afterDigest: screen.afterDigest } : {}), + ...(screen.truncated === true ? { truncated: true } : {}), ...(screen.baseError === undefined ? {} : { baseError: screen.baseError }), }; } @@ -138,14 +143,16 @@ export function readManifest(text, names, prefixes = ["docs", "scenes", "site"]) } /** - * What a maintainer approves with visual-ok: each changed screen with its status and the image the - * comment shows of it, its after image or, once removed, its before image, and every failure. - * `digest(file)` is the SHA-256 of an image the bot uploads. Order doesn't matter. + * What a maintainer approves with visual-ok: each changed screen with its status, the image the + * comment shows of it, its after image or, once removed, its before image, the digest of its whole + * after image, since a crop may show only part of a change, and every failure. `digest(file)` is the + * bot's own digest of an image it uploads; the whole image's digest comes from the pull request's CI, + * which decides its screens anyway. Order doesn't matter. */ export function fingerprint({ screens = [], failed = [] }, digest) { const entries = [ ...screens.filter((screen) => screen.status !== "unchanged") - .map((screen) => ["screen", screen.id, screen.status, digest(screen.images.after ?? screen.images.before)]), + .map((screen) => ["screen", screen.id, screen.status, digest(screen.images.after ?? screen.images.before), screen.afterDigest ?? null]), ...failed.map((entry) => ["failed", entry.id, entry.error]), ].map((entry) => JSON.stringify(entry)).sort(); return createHash("sha256").update(JSON.stringify(entries)).digest("hex"); @@ -196,6 +203,7 @@ function section(screen, imageURL) { const image = (file, label) => `${label}`; const lines = [`#### ${code(screen.id)}`]; if (screen.baseError) lines.push("", `The base couldn't draw this scene: ${code(screen.baseError, 300)}`); + if (screen.truncated) lines.push("", "The images show the first 1,600 rows of this change."); const { before, after, diff } = screen.images; if (before && after) { lines.push("", "| Before | After |", "| --- | --- |", `| ${image(before, "Before")} | ${image(after, "After")} |`); diff --git a/.github/scripts/screens-comment.test.mjs b/.github/scripts/screens-comment.test.mjs index 5e8f563..9bec1ce 100644 --- a/.github/scripts/screens-comment.test.mjs +++ b/.github/scripts/screens-comment.test.mjs @@ -7,7 +7,8 @@ const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); const sha = (character) => character.repeat(40); const digest = (character) => character.repeat(64); const screen = (id, status, images = {}, extra = {}) => ({ id, status, changedPixels: status === "changed" ? 120 : 0, - region: status === "changed" ? { x: 10, y: 20, width: 30, height: 40 } : null, images, ...extra }); + region: status === "changed" ? { x: 10, y: 20, width: 30, height: 40 } : null, images, + ...(status === "changed" || status === "resized" ? { afterDigest: "e".repeat(64) } : {}), ...extra }); const data = (extra = {}) => ({ head: sha("a"), base: sha("b"), ranScreens: true, fingerprint: digest("f"), approved: null, total: 2, changed: 1, failed: 0, failure: null, ...extra }); const urls = (file) => `https://example.test/${file}`; @@ -55,6 +56,9 @@ test("readManifest rejects what compare-screens doesn't write", () => { "no region": manifest([{ ...good, region: null }]), "negative count": manifest([{ ...good, changedPixels: -1 }]), "long base error": manifest([screen("docs/a", "added", { after: names[0] }, { baseError: "x".repeat(1001) })]), + "changed without its digest": manifest([{ ...good, afterDigest: undefined }]), + "digest not hex": manifest([{ ...good, afterDigest: "x".repeat(64) }]), + "truncated not a flag": manifest([{ ...good, truncated: "yes" }]), "failure without message": manifest([], [{ id: "scenes/x" }]), "too many screens": manifest(Array.from({ length: 301 }, (_, index) => screen(`docs/s${index}`, "unchanged"))), "too many failures": manifest([], Array.from({ length: 51 }, (_, index) => ({ id: `scenes/s${index}`, error: "x" }))), @@ -104,10 +108,13 @@ test("fingerprint covers each changed screen's image and every failure, in any o } const resized = screens.map((entry) => (entry.id === "docs/b" ? { ...entry, status: "resized", images: { before: "x.png", after: "b-after.png" } } : entry)); assert.notEqual(fingerprint({ screens: resized, failed: [] }, hash), first); + // The whole after image counts, not only the crop shown. + const further = screens.map((entry) => (entry.id === "docs/a" ? { ...entry, afterDigest: "f".repeat(64) } : entry)); + assert.notEqual(fingerprint({ screens: further, failed: [] }, hash), first); assert.notEqual(fingerprint({ screens, failed: [{ id: "scenes/x", error: "draw: Unknown value" }] }, hash), first); // A message can't pass for another entry. const real = fingerprint({ screens: [screens[0]], failed: [{ id: "scenes/x", error: "y" }] }, hash); - const forged = fingerprint({ screens: [], failed: [{ id: "scenes/x", error: "y\nscreen docs/a changed 11" }] }, hash); + const forged = fingerprint({ screens: [], failed: [{ id: "scenes/x", error: `y\nscreen docs/a changed 11 ${"e".repeat(64)}` }] }, hash); assert.notEqual(real, forged); }); @@ -158,6 +165,10 @@ test("commentBody shows changed screens, folds the rest and keeps its data", () assert.match(body, /^/); assert.match(body, /`main` at 1111111 → this pull request at aaaaaaa/); assert.match(body, /\| Screen \| Change \| Where \(image pixels\) \|/); + assert.doesNotMatch(body, /first 1,600 rows/); + const long = commentBody({ base: sha("1"), head: sha("a"), failed: [], imageURL: urls, review, data: data(), + screens: [screen("site/page-dark", "changed", { before: "b.png", after: "a.png", diff: "d.png" }, { truncated: true })] }); + assert.match(long, /The images show the first 1,600 rows of this change\./); assert.match(body, /https:\/\/example\.test\/b\.png/); assert.match(body, /https:\/\/example\.test\/d\.png/); assert.match(body, /The base couldn't draw this scene: `draw: Unknown value`/); From 47ebccb37c2bb34b4e41718f1f0affa709dac16e Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 17:12:53 +0100 Subject: [PATCH 16/25] ci: fail the review when drawn screens are missing A screens job that succeeded but uploaded no screens passed as "0 screens, none changed", and an image a scenes report listed but nobody drew went unnoticed. compare-screens now reports each image the pull request's scenes.json lists that wasn't drawn, and the bot refuses an artifact that holds no screens, or none of those it always has: docs images from Draw screens, website screens from Draw website screens. --- .github/scripts/compare-screens.mjs | 24 ++++++++++++++++++------ .github/scripts/compare-screens.test.mjs | 14 ++++++++++++++ .github/scripts/screens-bot.mjs | 14 +++++++++----- .github/scripts/screens-bot.test.mjs | 14 ++++++++++++++ 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/.github/scripts/compare-screens.mjs b/.github/scripts/compare-screens.mjs index 1feb271..db3826f 100644 --- a/.github/scripts/compare-screens.mjs +++ b/.github/scripts/compare-screens.mjs @@ -214,14 +214,25 @@ export function compareDirectories(beforeDir, afterDir, outDir, { prefix, before return { screens, failed }; } -/** Scene name to error, from a scenes.json report `--render-scenes` wrote. */ -function errors(report) { +/** The scenes of a scenes.json report `--render-scenes` wrote, by name: `{images}` or `{error}`. */ +function scenesOf(report) { if (!report || !existsSync(report)) return {}; - const scenes = JSON.parse(readFileSync(report, "utf8")).scenes ?? {}; - return Object.fromEntries(Object.entries(scenes).filter(([, entry]) => typeof entry?.error === "string" && entry.error) + return JSON.parse(readFileSync(report, "utf8")).scenes ?? {}; +} + +/** Scene name to error, from a scenes.json report. */ +function errors(report) { + return Object.fromEntries(Object.entries(scenesOf(report)).filter(([, entry]) => typeof entry?.error === "string" && entry.error) .map(([name, entry]) => [name, entry.error])); } +/** The images a scenes.json report lists that aren't in `folder`, as failures. */ +function undrawn(report, folder, prefix) { + return Object.entries(scenesOf(report)).flatMap(([name, entry]) => (Array.isArray(entry?.images) ? entry.images : []) + .filter((image) => typeof image !== "string" || !existsSync(join(folder, basename(image)))) + .map((image) => ({ id: cut(`${prefix}/${name}`, 200), error: cut(`scenes.json lists ${image}, but it wasn't drawn.`, maximumMessage) }))); +} + export function main(argv) { const [beforeDir, afterDir, outDir, ...options] = argv; const option = (name) => { @@ -235,8 +246,9 @@ export function main(argv) { const beforeReport = option("--before-report"); const beforeError = beforeReport && !existsSync(beforeReport) ? "The base build wrote no scenes.json, so it may not draw scenes yet." : undefined; const { screens, failed: unnamed } = compareDirectories(beforeDir, afterDir, outDir, { prefix, beforeErrors: errors(beforeReport), beforeError }); - const failed = [...unnamed, ...Object.entries(errors(option("--after-report"))) - .map(([name, error]) => ({ id: cut(`${prefix}/${name}`, 200), error: cut(error, maximumMessage) }))]; + const afterReport = option("--after-report"); + const failed = [...unnamed, ...Object.entries(errors(afterReport)) + .map(([name, error]) => ({ id: cut(`${prefix}/${name}`, 200), error: cut(error, maximumMessage) })), ...undrawn(afterReport, afterDir, prefix)]; const manifestPath = join(outDir, "manifest.json"); const manifest = existsSync(manifestPath) ? JSON.parse(readFileSync(manifestPath, "utf8")) : { screens: [], failed: [] }; manifest.screens.push(...screens); diff --git a/.github/scripts/compare-screens.test.mjs b/.github/scripts/compare-screens.test.mjs index 6a750af..a6a5525 100644 --- a/.github/scripts/compare-screens.test.mjs +++ b/.github/scripts/compare-screens.test.mjs @@ -164,3 +164,17 @@ test("changed and resized screens carry a digest of their whole after image", () assert.match(runs[0].screen.afterDigest, /^[0-9a-f]{64}$/); assert.notEqual(runs[0].screen.afterDigest, runs[1].screen.afterDigest); }); + +test("main reports images a scenes report lists that weren't drawn", () => { + const root = mkdtempSync(join(tmpdir(), "screens-")); + const before = join(root, "before"), after = join(root, "after"), out = join(root, "out"); + for (const folder of [before, after]) mkdirSync(folder, { recursive: true }); + writeFileSync(join(before, "bar-dark.png"), encodePNG(image(8, 8))); + writeFileSync(join(after, "bar-dark.png"), encodePNG(image(8, 8))); + const report = { scenes: { bar: { images: ["bar-dark.png", "bar-light.png"] } } }; + writeFileSync(join(before, "scenes.json"), JSON.stringify(report)); + writeFileSync(join(after, "scenes.json"), JSON.stringify(report)); + main([before, after, out, "--prefix", "scenes", "--before-report", join(before, "scenes.json"), "--after-report", join(after, "scenes.json")]); + const manifest = JSON.parse(readFileSync(join(out, "manifest.json"), "utf8")); + assert.deepEqual(manifest.failed, [{ id: "scenes/bar", error: "scenes.json lists bar-light.png, but it wasn't drawn." }]); +}); diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs index ac55e7f..5719148 100644 --- a/.github/scripts/screens-bot.mjs +++ b/.github/scripts/screens-bot.mjs @@ -13,9 +13,10 @@ const statusContext = "Visual review"; const approval = "visual-ok"; /** The CI jobs the screens depend on, by the names they show. */ const jobNames = { current: "Check revision", build: "Build release", app: "Draw screens", site: "Draw website screens" }; +/** Each kind of screens: its artifact, the ids it may hold, the screens it always has, and the jobs it needs. */ const kinds = { - app: { artifact: "screens-app", prefixes: ["docs", "scenes"], needs: ["current", "build"] }, - site: { artifact: "screens-site", prefixes: ["site"], needs: ["current"] }, + app: { artifact: "screens-app", prefixes: ["docs", "scenes"], always: { prefix: "docs", name: "docs images" }, needs: ["current", "build"] }, + site: { artifact: "screens-site", prefixes: ["site"], always: { prefix: "site", name: "website screens" }, needs: ["current"] }, }; const broken = ["failure", "cancelled", "timed_out", "missing"]; /** @@ -70,7 +71,7 @@ const pixelDigest = ({ width, height, rgba }) => createHash("sha256").update(`${ * image its screens name is decoded within limits and encoded again, so only the bot's own PNGs are * uploaded: `files` maps each name to `{data, digest}`, the digest being of the pixels. */ -export function readArtifact(folder, prefixes) { +export function readArtifact(folder, prefixes, always) { if (!existsSync(folder)) return { ok: false, problems: ["It couldn't be downloaded."] }; const files = readdirSync(folder).map((name) => { const path = join(folder, name); @@ -92,6 +93,9 @@ export function readArtifact(folder, prefixes) { if (!manifest) return { ok: false, problems: ["It has no manifest.json."] }; const read = readManifest(readFileSync(manifest.path, "utf8"), files.map((file) => file.name), prefixes); if (!read.ok) return read; + // A job that drew nothing, or not what it always draws, failed even if it says it didn't. + if (read.screens.length === 0 && read.failed.length === 0) return { ok: false, problems: ["It holds no screens."] }; + if (always && !read.screens.some((screen) => screen.id.startsWith(`${always.prefix}/`))) return { ok: false, problems: [`It holds no ${always.name}.`] }; const images = new Map(); for (const name of new Set(read.screens.flatMap((screen) => Object.values(screen.images)))) { let image; @@ -170,7 +174,7 @@ async function deleteAssets({ github, owner, repo, pick }) { function gather(run, folders) { const screens = [], failed = [], files = new Map(); let ranScreens = false; - for (const [kind, { artifact, prefixes, needs }] of Object.entries(kinds)) { + for (const [kind, { artifact, prefixes, always, needs }] of Object.entries(kinds)) { const conclusion = run.jobs[kind]; if (conclusion === "missing") { ranScreens = true; @@ -193,7 +197,7 @@ function gather(run, folders) { failed.push({ id: kind, error: `${jobNames[kind]} ended with ${conclusion}.` }); continue; } - const read = run.artifacts[kind] ? readArtifact(folders[kind] ?? "", prefixes) : { ok: false, problems: ["The job uploaded none."] }; + const read = run.artifacts[kind] ? readArtifact(folders[kind] ?? "", prefixes, always) : { ok: false, problems: ["The job uploaded none."] }; if (!read.ok) { failed.push({ id: kind, error: `The ${artifact} artifact can't be used: ${read.problems.join(" ")}` }); continue; diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs index aa40c79..565382a 100644 --- a/.github/scripts/screens-bot.test.mjs +++ b/.github/scripts/screens-bot.test.mjs @@ -305,6 +305,20 @@ test("publishRun refuses an image it can't decode", async () => { assert.equal(state.uploads.length, 0); }); +test("publishRun fails an artifact without the screens it always has", async () => { + const { github, state } = fakeGitHub({ pulls: [pr()] }); + const empty = mkdtempSync(join(tmpdir(), "artifact-")); + writeFileSync(join(empty, "manifest.json"), JSON.stringify({ screens: [], failed: [] })); + assert.equal((await publishRun({ github, context: runContext(), run: ran(), folders: { app: empty } })).state, "failure"); + assert.match(state.comments[0].body, /The screens-app artifact can't be used: It holds no screens\./); + // Draw screens always draws the docs images. + const scenes = mkdtempSync(join(tmpdir(), "artifact-")); + writeFileSync(join(scenes, "manifest.json"), JSON.stringify({ screens: [{ id: "scenes/bar-dark", status: "unchanged", changedPixels: 0, + region: null, images: {} }], failed: [] })); + assert.equal((await publishRun({ github, context: runContext(), run: ran(), folders: { app: scenes } })).state, "failure"); + assert.match(state.comments[0].body, /It holds no docs images\./); +}); + test("failRun marks the head when the bot itself fails", async () => { const { github, state } = fakeGitHub(); await failRun({ github, context: runContext(), run: ran() }); From 23b972faa4427dce22e70960621a6c9c13ef9244 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 17:13:35 +0100 Subject: [PATCH 17/25] ci: spread pull request screens over releases with room A GitHub release holds 1,000 assets, and every publication chose the month's one release, so a full month failed every pull request's screens until someone deleted images. Uploads now go to the first of the month's releases with room, pr-previews-YYYY-MM, then -2 and on, created as needed. A rerun of a head first deletes its earlier images, wherever they are. --- .github/scripts/screens-bot.mjs | 61 ++++++++++++++++++---------- .github/scripts/screens-bot.test.mjs | 14 ++++++- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs index 5719148..c57264f 100644 --- a/.github/scripts/screens-bot.mjs +++ b/.github/scripts/screens-bot.mjs @@ -119,34 +119,51 @@ async function setStatus({ github, owner, repo, sha, review, url }) { ...(url ? { target_url: url } : {}) }); } -/** This month's pre-release for the images, created when missing, never as the latest release. */ -async function previewRelease({ github, owner, repo, target }) { - const tag = releaseTag(new Date()); - const find = async () => (await github.rest.repos.getReleaseByTag({ owner, repo, tag })).data; - try { - return await find(); - } catch (error) { - if (error.status !== 404) throw error; - } - try { - return (await github.rest.repos.createRelease({ owner, repo, tag_name: tag, target_commitish: target, name: `Pull request screens, ${tag.slice(12)}`, - prerelease: true, make_latest: "false", - body: "Images for the before and after screens the bot posts on pull requests. An unmerged pull request's images are deleted when it closes." })).data; - } catch (error) { - if (error.status === 422) return find(); - throw error; +/** A release holds at most this many assets. */ +const capacity = 1000; + +/** + * The first of this month's pre-releases for the images with room for `needed` more, `pr-previews-YYYY-MM`, + * then `-2`, `-3` and on, created when missing, never as the latest release. + */ +async function previewRelease({ github, owner, repo, target, needed }) { + const month = releaseTag(new Date()); + const find = async (tag) => { + try { + return (await github.rest.repos.getReleaseByTag({ owner, repo, tag })).data; + } catch (error) { + if (error.status === 404) return null; + throw error; + } + }; + for (let shard = 1; shard <= 50; shard++) { + const tag = shard === 1 ? month : `${month}-${shard}`; + let release = await find(tag); + if (!release) { + try { + release = (await github.rest.repos.createRelease({ owner, repo, tag_name: tag, target_commitish: target, name: `Pull request screens, ${tag.slice(12)}`, + prerelease: true, make_latest: "false", + body: "Images for the before and after screens the bot posts on pull requests. An unmerged pull request's images are deleted when it closes." })).data; + } catch (error) { + if (error.status !== 422) throw error; + release = await find(tag); + } + } + const assets = await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id: release.id, per_page: 100 }); + if (assets.length + needed <= capacity) return release; } + throw new Error(`No pull request screens release of ${month} has room for ${needed} images.`); } -/** Uploads `images`, [file, PNG data] pairs, and returns each file's URL as GitHub answers it. A rerun of a head replaces its images. */ +/** + * Uploads `images`, [file, PNG data] pairs, and returns each file's URL as GitHub answers it. The head's + * images from an earlier run of it are deleted first, wherever they are. + */ async function upload({ github, owner, repo, number, head, target, images }) { const urls = new Map(); if (images.length === 0) return urls; - const release = await previewRelease({ github, owner, repo, target }); - const assets = await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id: release.id, per_page: 100 }); - for (const asset of pullRequestAssets(assets, number).filter((entry) => entry.name.startsWith(assetName(number, head, "")))) { - await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id }); - } + await deleteAssets({ github, owner, repo, pick: (assets) => pullRequestAssets(assets, number).filter((asset) => asset.name.startsWith(assetName(number, head, ""))) }); + const release = await previewRelease({ github, owner, repo, target, needed: images.length }); for (const [file, data] of images) { const { data: asset } = await github.rest.repos.uploadReleaseAsset({ owner, repo, release_id: release.id, name: assetName(number, head, file), data, headers: { "content-type": "image/png", "content-length": data.length } }); diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs index 565382a..27c97d0 100644 --- a/.github/scripts/screens-bot.test.mjs +++ b/.github/scripts/screens-bot.test.mjs @@ -319,6 +319,16 @@ test("publishRun fails an artifact without the screens it always has", async () assert.match(state.comments[0].body, /It holds no docs images\./); }); +test("publishRun moves on to the month's next release when one is full", async () => { + const month = releaseTag(new Date()); + const full = { id: 3, tag_name: month, assets: Array.from({ length: 999 }, (_, index) => ({ id: 10000 + index, name: `pr1-1234567-${index}.png` })) }; + const { github, state } = fakeGitHub({ pulls: [pr()], releases: [full] }); + await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); + const next = state.releases.find((release) => release.tag_name === `${month}-2`); + assert.deepEqual([full.assets.length, next?.assets.length, next?.prerelease], [999, 3, true]); + assert.match(state.comments[0].body, /download\/pr7-aaaaaaa-docs\.hero-dark-after\.png/); +}); + test("failRun marks the head when the bot itself fails", async () => { const { github, state } = fakeGitHub(); await failRun({ github, context: runContext(), run: ran() }); @@ -333,8 +343,8 @@ test("deleteScreens deletes an unmerged pull request's images from every month", { id: 3, tag_name: "v1.0.0", assets: [{ id: 31, name: "pr7-ccccccc-StableBar.dmg" }] }] }); await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); const close = { repo, payload: { action: "closed", pull_request: pr({ state: "closed", merged: false }) } }; - // Publishing already dropped the other head's image; closing takes this head's three and the stray one. - assert.equal(await deleteScreens({ github, context: close }), 4); + // Publishing replaced this head's earlier image and dropped the other head's; closing takes the three it uploaded. + assert.equal(await deleteScreens({ github, context: close }), 3); assert.deepEqual(state.releases.map((release) => [release.tag_name, release.assets.map((asset) => asset.name)]).slice(0, 3), [["pr-previews-2026-09", ["pr8-aaaaaaa-a.png"]], ["pr-previews-2026-10", []], ["v1.0.0", ["pr7-ccccccc-StableBar.dmg"]]]); assert.match(state.comments[0].body, /closed without merging, so the images of its screens were deleted/); From b754ec9e8714aa1eb5cc6ae0959e9e5a88f5eec0 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 17:21:56 +0100 Subject: [PATCH 18/25] ci: reconcile each pull request's screens one event at a time Publications and label events ran in separate queues and wrote the same comment and status, so a label reconcile could restore a stale comment and success over a newer failure, and a run could publish after its pull request closed and its images were deleted. A base change kept the status of the old comparison. Every event now runs one reconcile of its pull request, in one queue per pull request. plan() reads where it stands: closed, or the newest finished CI run for its head, by run and attempt, against what the comment shows. The workflow downloads that run's artifacts and apply() rechecks the pull request and publishes, reviews or cleans up. GitHub keeps only the newest waiting event, so nothing depends on which events ran: visual-ok approves when it was added after the comment began showing the screens, by GitHub's clock, and the same screens on a later run keep that time. A base other than the one the screens were drawn against sends the review back to pending until a new run. --- .github/scripts/screens-bot.mjs | 218 +++++++----- .github/scripts/screens-bot.test.mjs | 421 +++++++++++++---------- .github/scripts/screens-comment.mjs | 26 +- .github/scripts/screens-comment.test.mjs | 49 +-- .github/workflows/screens.yml | 107 +++--- 5 files changed, 453 insertions(+), 368 deletions(-) diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs index c57264f..af32f92 100644 --- a/.github/scripts/screens-bot.mjs +++ b/.github/scripts/screens-bot.mjs @@ -1,7 +1,11 @@ // The GitHub side of the before/after screens, for screens.yml, which runs the default branch's copy. -// It finds the pull request a CI run drew screens for, uploads their images to a monthly pre-release, -// writes the comment, sets the Visual review status, and handles visual-ok and closed pull requests. -// The artifacts come from the pull request's own CI jobs, so they are read as untrusted data. +// Every event, a CI run finishing or a pull request labelled, retargeted, closed or reopened, runs one +// reconcile of its pull request, one at a time: plan() reads where the pull request stands and which CI +// run's screens to publish, the workflow downloads that run's artifacts, and apply() makes the comment, +// the images and the Visual review status match. GitHub may drop an event waiting behind another; the +// next one still reconciles everything, since nothing depends on which events ran. visual-ok approves +// the screens the comment shows when it was added after they appeared. The artifacts come from the +// pull request's own CI jobs, so they are read as untrusted data. import { closeSync, existsSync, lstatSync, openSync, readFileSync, readSync, readdirSync } from "node:fs"; import { createHash } from "node:crypto"; import { join } from "node:path"; @@ -11,6 +15,7 @@ import { decodePNG, encodePNG } from "./png.mjs"; const statusContext = "Visual review"; const approval = "visual-ok"; +const workflowPath = ".github/workflows/ci.yml"; /** The CI jobs the screens depend on, by the names they show. */ const jobNames = { current: "Check revision", build: "Build release", app: "Draw screens", site: "Draw website screens" }; /** Each kind of screens: its artifact, the ids it may hold, the screens it always has, and the jobs it needs. */ @@ -29,38 +34,55 @@ export const changePatterns = { site: /^(site\/|\.github\/workflows\/ci\.yml$|\.github\/scripts\/(png|compare-screens|capture-site)\.mjs$)/, }; +/** The open pull request a pull_request run of CI is for: listed on the run, or for a fork, found by its head branch. */ +async function pullRequestOfRun({ github, owner, repo, run }) { + if (!run || run.event !== "pull_request" || run.path !== workflowPath || !run.head_repository) return null; + if (run.pull_requests.length > 0) return run.pull_requests[0].number; + const pulls = await github.paginate(github.rest.pulls.list, + { owner, repo, state: "open", head: `${run.head_repository.owner.login}:${run.head_branch}`, per_page: 100 }); + return (pulls.find((pr) => pr.head.sha === run.head_sha) ?? pulls[0])?.number ?? null; +} + +/** The newest finished CI run for a head, preferring the runs GitHub lists the pull request on. */ +async function latestRun({ github, owner, repo, number, sha }) { + const runs = (await github.paginate(github.rest.actions.listWorkflowRuns, + { owner, repo, workflow_id: "ci.yml", head_sha: sha, event: "pull_request", status: "completed", per_page: 100 })) + .filter((run) => run.path === workflowPath); + const listed = runs.filter((run) => run.pull_requests.some((pr) => pr.number === number)); + return [...(listed.length > 0 ? listed : runs)].sort((a, b) => b.run_number - a.run_number)[0] ?? null; +} + /** - * The pull request a completed CI run drew screens for, with what its jobs did, which artifacts it - * uploaded and which screens its changed files call for, or null when the run isn't CI's for an open - * pull request's current head. A fork's run lists no pull requests, so it is found by its head branch - * and commit. A job the run doesn't have is "missing". + * What a reconcile of the event's pull request does, decided before the workflow downloads anything: + * "none" for a run that isn't CI's for an open pull request, "close" for a closed pull request, + * "publish" with the newest finished CI run for its head when the comment doesn't show that run yet, + * and "review" otherwise. A publication carries what the run's jobs did, which artifacts it uploaded + * and which screens the pull request's files call for. A job the run doesn't have is "missing". */ -export async function findRun({ github, context }) { +export async function plan({ github, context }) { const { owner, repo } = context.repo; - const run = context.payload.workflow_run; - if (run.event !== "pull_request" || run.path !== ".github/workflows/ci.yml" || !run.head_repository) return null; - let numbers = run.pull_requests.map((pr) => pr.number); - if (numbers.length === 0) { - const pulls = await github.paginate(github.rest.pulls.list, - { owner, repo, state: "open", head: `${run.head_repository.owner.login}:${run.head_branch}`, per_page: 100 }); - numbers = pulls.map((pr) => pr.number); - } - for (const number of numbers) { - const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: number }); - if (pr.state !== "open" || pr.head.sha !== run.head_sha) continue; - const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { owner, repo, run_id: run.id, filter: "latest", per_page: 100 }); - const uploaded = (await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { owner, repo, run_id: run.id, per_page: 100 })) - .map((artifact) => artifact.name); - const files = (await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number: number, per_page: 100 })) - .flatMap((file) => [file.filename, file.previous_filename].filter(Boolean)); - return { - number, head: run.head_sha, runId: run.id, + const number = context.payload.pull_request?.number ?? await pullRequestOfRun({ github, owner, repo, run: context.payload.workflow_run }); + if (!number) return { action: "none" }; + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: number }); + if (pr.state !== "open") return { action: "close", number }; + const run = await latestRun({ github, owner, repo, number, sha: pr.head.sha }); + const existing = findScreensComment(await comments(github, owner, repo, number)); + const data = existing ? readData(existing.body) : null; + if (!run || (data?.runId === run.id && data?.runAttempt === run.run_attempt)) return { action: "review", number }; + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { owner, repo, run_id: run.id, filter: "latest", per_page: 100 }); + const uploaded = (await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { owner, repo, run_id: run.id, per_page: 100 })) + .map((artifact) => artifact.name); + const files = (await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number: number, per_page: 100 })) + .flatMap((file) => [file.filename, file.previous_filename].filter(Boolean)); + return { + action: "publish", number, + run: { + id: run.id, attempt: run.run_attempt, head: pr.head.sha, jobs: Object.fromEntries(Object.entries(jobNames).map(([key, name]) => [key, jobs.find((job) => job.name === name)?.conclusion ?? "missing"])), artifacts: Object.fromEntries(Object.entries(kinds).map(([kind, { artifact }]) => [kind, uploaded.includes(artifact)])), expected: Object.fromEntries(Object.keys(kinds).map((kind) => [kind, files.some((file) => changePatterns[kind].test(file))])), - }; - } - return null; + }, + }; } /** A digest of an image's pixels, the same for any encoding of them. */ @@ -111,8 +133,20 @@ export function readArtifact(folder, prefixes, always) { const comments = (github, owner, repo, number) => github.paginate(github.rest.issues.listComments, { owner, repo, issue_number: number, per_page: 100 }); -const labelsOf = async (github, owner, repo, number) => - (await github.paginate(github.rest.issues.listLabelsOnIssue, { owner, repo, issue_number: number, per_page: 100 })).map((label) => label.name); + +/** When visual-ok was last added to the pull request, as GitHub's ISO time, or null. */ +async function lastLabeled({ github, owner, repo, number }) { + const events = await github.paginate(github.rest.issues.listEvents, { owner, repo, issue_number: number, per_page: 100 }); + return events.filter((event) => event.event === "labeled" && event.label?.name === approval).map((event) => event.created_at).sort().at(-1) ?? null; +} + +async function removeApproval({ github, owner, repo, number }) { + try { + await github.rest.issues.removeLabel({ owner, repo, issue_number: number, name: approval }); + } catch (error) { + if (error.status !== 404) throw error; + } +} async function setStatus({ github, owner, repo, sha, review, url }) { await github.rest.repos.createCommitStatus({ owner, repo, sha, state: review.state, context: statusContext, description: review.description, @@ -227,91 +261,103 @@ function gather(run, folders) { } /** - * Publishes a run from findRun: uploads the images of its changed screens, writes or rewrites the - * comment, sets the status on its head, takes visual-ok off when it approved other screens, and then - * deletes the images of the pull request's other heads. `folders` hold the downloaded artifacts by kind. + * Publishes a run from plan(): uploads the images of its changed screens, writes or rewrites the + * comment, sets the status on the head, takes visual-ok off when it came before these screens, and then + * deletes the images of the pull request's other heads. Screens the comment already showed keep the + * time they first appeared, and so their approval. */ -export async function publishRun({ github, context, run, folders }) { - const { owner, repo } = context.repo; +async function publish({ github, owner, repo, pr, existing, data: previous, now, labels, labeledAt, run, folders, target }) { + const number = pr.number, head = pr.head.sha; const { screens, failed, files, ranScreens } = gather(run, folders); - const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: run.number }); - const existing = findScreensComment(await comments(github, owner, repo, run.number)); - const previous = existing ? readData(existing.body) : null; + const drawn = { head, base: pr.base.sha, baseRef: pr.base.ref, runId: run.id, runAttempt: run.attempt }; let review, body; if (!ranScreens) { review = reviewState({ ranScreens: false }); - const data = { head: run.head, base: pr.base.sha, ranScreens: false, fingerprint: null, approved: previous?.approved ?? null, - total: 0, changed: 0, failed: 0, failure: null }; - body = noScreensBody({ reason: `This pull request at ${run.head.slice(0, 7)} changes neither the app nor the website, so there are no screens to compare.`, + const data = { ...drawn, ranScreens: false, fingerprint: null, shownAt: null, total: 0, changed: 0, failed: 0, failure: null }; + body = noScreensBody({ reason: `This pull request at ${head.slice(0, 7)} changes neither the app nor the website, so there are no screens to compare.`, review, data }); } else { const shown = screens.filter((screen) => screen.status !== "unchanged").flatMap((screen) => Object.values(screen.images)); - const urls = await upload({ github, owner, repo, number: run.number, head: run.head, target: context.payload.repository?.default_branch, - images: shown.map((file) => [file, files.get(file).data]) }); + const urls = await upload({ github, owner, repo, number, head, target, images: shown.map((file) => [file, files.get(file).data]) }); const print = fingerprint({ screens, failed }, (file) => files.get(file).digest); + const shownAt = previous?.fingerprint === print && previous.shownAt ? previous.shownAt : now; const counts = summary(screens, failed); - review = reviewState({ ranScreens: true, ...counts, labels: await labelsOf(github, owner, repo, run.number), - approved: previous?.approved ?? null, fingerprint: print }); - if (review.removeLabel) { - try { - await github.rest.issues.removeLabel({ owner, repo, issue_number: run.number, name: approval }); - } catch (error) { - if (error.status !== 404) throw error; - } - } - const data = { head: run.head, base: pr.base.sha, ranScreens: true, fingerprint: print, - approved: review.removeLabel ? null : previous?.approved ?? null, ...counts }; - body = commentBody({ baseRef: pr.base.ref, base: pr.base.sha, head: run.head, screens, failed, imageURL: (file) => urls.get(file), review, data }); + review = reviewState({ ranScreens: true, ...counts, labels, labeledAt, shownAt }); + if (review.removeLabel) await removeApproval({ github, owner, repo, number }); + const data = { ...drawn, ranScreens: true, fingerprint: print, shownAt, ...counts }; + body = commentBody({ baseRef: pr.base.ref, base: pr.base.sha, head, screens, failed, imageURL: (file) => urls.get(file), review, data }); } let comment = existing; if (existing) { comment = (await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body })).data; } else if (ranScreens) { - comment = (await github.rest.issues.createComment({ owner, repo, issue_number: run.number, body })).data; + comment = (await github.rest.issues.createComment({ owner, repo, issue_number: number, body })).data; } - await setStatus({ github, owner, repo, sha: run.head, review, url: comment?.html_url }); - await deleteAssets({ github, owner, repo, pick: (assets) => staleAssets(assets, run.number, run.head) }); + await setStatus({ github, owner, repo, sha: head, review, url: comment?.html_url }); + await deleteAssets({ github, owner, repo, pick: (assets) => staleAssets(assets, number, head) }); return review; } -/** Marks the head as errored when the bot itself fails, so a required Visual review doesn't wait forever. */ -export async function failRun({ github, context, run }) { - const { owner, repo } = context.repo; - await github.rest.repos.createCommitStatus({ owner, repo, sha: run.head, state: "error", context: statusContext, - description: "The screens bot failed: see the Screens workflow run.", target_url: `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}` }); -} - /** - * Follows visual-ok being added or removed: an added label approves the fingerprint of the screens the - * comment shows, a removed one drops the approval. Sets the status when the comment is for the current - * head; a newer head's run sets its own. Returns the review, or null without the bot's comment. + * Brings the review up to date without drawing anything: the labels, when visual-ok was added, and + * whether the pull request still has the base its screens were compared with. Sets the status when + * the comment is for the current head; a newer head's run sets its own. */ -export async function updateReview({ github, context }) { - const { owner, repo } = context.repo; - const pr = context.payload.pull_request; - const existing = findScreensComment(await comments(github, owner, repo, pr.number)); - const data = existing ? readData(existing.body) : null; +async function review({ github, owner, repo, pr, existing, data, labels, labeledAt }) { if (!data) return null; - const labels = await labelsOf(github, owner, repo, pr.number); - const approved = !labels.includes(approval) ? null : context.payload.action === "labeled" ? data.fingerprint : data.approved; - const next = { ...data, approved }; - const review = reviewState({ ...next, labels }); - await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body: withReview(existing.body, review, next) }); - if (data.head === pr.head.sha) await setStatus({ github, owner, repo, sha: data.head, review, url: existing.html_url }); - return review; + const baseChanged = data.baseRef !== null && data.baseRef !== pr.base.ref; + const result = reviewState({ ...data, labels, labeledAt, baseChanged }); + if (result.removeLabel) await removeApproval({ github, owner, repo, number: pr.number }); + const body = withReview(existing.body, result, data); + if (body !== existing.body) await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + if (data.head === pr.head.sha) await setStatus({ github, owner, repo, sha: data.head, review: result, url: existing.html_url }); + return result; } /** Deletes an unmerged, closed pull request's images from every month, says so in its comment, and counts them. */ -export async function deleteScreens({ github, context }) { - const { owner, repo } = context.repo; - const number = context.payload.pull_request.number; +async function closeUnmerged({ github, owner, repo, number }) { const deleted = await deleteAssets({ github, owner, repo, pick: (assets) => pullRequestAssets(assets, number) }); const existing = findScreensComment(await comments(github, owner, repo, number)); const data = existing ? readData(existing.body) : null; if (data) { const review = { state: "success", description: "Nothing to review: the pull request closed.", removeLabel: false }; - await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, - body: noScreensBody({ reason: "This pull request closed without merging, so the images of its screens were deleted.", review, data }) }); + const body = noScreensBody({ reason: "This pull request closed without merging, so the images of its screens were deleted.", review, data }); + if (body !== existing.body) await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); } return deleted; } + +/** + * Makes the pull request's comment, images and Visual review status match where it stands now, + * following plan(): `folders` hold the artifacts the workflow downloaded for a publication. A closed + * pull request loses its images unless it merged; a head that moved since the plan waits for its own + * run. Returns what it did: `{action, review}`, or `{action: "close", deleted}`. + */ +export async function apply({ github, context, plan: chosen, folders = {} }) { + if (chosen.action === "none") return { action: "none" }; + const { owner, repo } = context.repo; + const number = chosen.number; + const response = await github.rest.pulls.get({ owner, repo, pull_number: number }); + const pr = response.data; + if (pr.state !== "open") return { action: "close", deleted: pr.merged ? 0 : await closeUnmerged({ github, owner, repo, number }) }; + const existing = findScreensComment(await comments(github, owner, repo, number)); + const state = { + github, owner, repo, pr, existing, data: existing ? readData(existing.body) : null, + now: new Date(response.headers?.date ?? Date.now()).toISOString(), + labels: pr.labels.map((label) => label.name), labeledAt: await lastLabeled({ github, owner, repo, number }), + }; + if (chosen.action === "publish" && chosen.run.head === pr.head.sha) { + return { action: "publish", review: await publish({ ...state, run: chosen.run, folders, target: context.payload.repository?.default_branch }) }; + } + return { action: "review", review: await review(state) }; +} + +/** Marks an open pull request's head as errored when the bot itself fails, so a required Visual review doesn't wait forever. */ +export async function failRun({ github, context, number }) { + if (!number) return; + const { owner, repo } = context.repo; + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: number }); + if (pr.state !== "open") return; + await github.rest.repos.createCommitStatus({ owner, repo, sha: pr.head.sha, state: "error", context: statusContext, + description: "The screens bot failed: see the Screens workflow run.", target_url: `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}` }); +} diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs index 27c97d0..219bc99 100644 --- a/.github/scripts/screens-bot.test.mjs +++ b/.github/scripts/screens-bot.test.mjs @@ -7,12 +7,14 @@ import zlib from "node:zlib"; import { decodePNG, encodePNG } from "./png.mjs"; import { main as compare } from "./compare-screens.mjs"; import { readData, releaseTag } from "./screens-comment.mjs"; -import { changePatterns, deleteScreens, failRun, findRun, publishRun, updateReview } from "./screens-bot.mjs"; +import { apply, changePatterns, failRun, plan } from "./screens-bot.mjs"; const head = "a".repeat(40), oldHead = "c".repeat(40), base = "b".repeat(40); const repo = { owner: "Seb-GRAF", repo: "StableBar" }; const bot = { login: "github-actions[bot]", type: "Bot" }; const notFound = () => Object.assign(new Error("Not Found"), { status: 404 }); +const conclusions = { "Check revision": "success", "Build release": "success", "Draw screens": "success", "Draw website screens": "skipped" }; +const jobsOf = (entries) => Object.entries(entries).map(([name, conclusion]) => ({ name, conclusion })); function image(color) { const rgba = new Uint8Array(40 * 20 * 4); @@ -40,19 +42,26 @@ function artifact(color = [255, 255, 255, 255]) { return out; } -/** The parts of Octokit the bot uses, over a little state. Uploads get GitHub's renaming of special characters. */ -function fakeGitHub({ pulls = [], comments = [], labels = [], releases = [], jobs = [], artifacts = [], files = [] } = {}) { - const state = { comments, labels, releases, statuses: [], uploads: [], deleted: [], next: 1000 }; +/** + * The parts of Octokit the bot uses, over a little state with its own clock, which answers in the Date + * header. Uploads get GitHub's renaming of special characters. + */ +function fakeGitHub({ pull = {}, comments = [], labels = [], releases = [], files = ["Sources/StableBar/BarView.swift"] } = {}) { + const state = { + now: new Date("2026-10-31T10:00:00Z"), comments, labels, releases, files, events: [], statuses: [], uploads: [], deleted: [], next: 1000, + pr: { number: 7, state: "open", merged: false, head: { sha: head, ref: "feat/x", repo: { owner: { login: "someone" } } }, base: { sha: base, ref: "main" }, ...pull }, + runs: [{ id: 99, run_number: 50, run_attempt: 1, head_sha: head, path: ".github/workflows/ci.yml", pull_requests: [] }], + jobs: { 99: jobsOf(conclusions) }, artifacts: { 99: [{ name: "screens-app" }] }, + }; + const at = () => state.now.toISOString(); const rest = { pulls: { get: async ({ pull_number }) => { - const pr = pulls.find((entry) => entry.number === pull_number); - if (!pr) throw notFound(); - return { data: pr }; + if (pull_number !== state.pr.number) throw notFound(); + return { data: { ...state.pr, labels: state.labels.map((name) => ({ name })) }, headers: { date: state.now.toUTCString() } }; }, - list: async ({ head: filter, state: wanted }) => - ({ data: pulls.filter((pr) => pr.state === wanted && `${pr.head.repo.owner.login}:${pr.head.ref}` === filter) }), - listFiles: async () => ({ data: files.map((filename) => ({ filename })) }), + list: async ({ head: filter, state: wanted }) => ({ data: [state.pr].filter((pr) => pr.state === wanted && `${pr.head.repo.owner.login}:${pr.head.ref}` === filter) }), + listFiles: async () => ({ data: state.files.map((filename) => ({ filename })) }), }, issues: { listComments: async () => ({ data: state.comments }), @@ -66,10 +75,11 @@ function fakeGitHub({ pulls = [], comments = [], labels = [], releases = [], job comment.body = body; return { data: comment }; }, - listLabelsOnIssue: async () => ({ data: state.labels.map((name) => ({ name })) }), + listEvents: async () => ({ data: state.events }), removeLabel: async ({ name }) => { if (!state.labels.includes(name)) throw notFound(); state.labels = state.labels.filter((entry) => entry !== name); + state.events.push({ event: "unlabeled", label: { name }, created_at: at() }); return { data: [] }; }, }, @@ -105,169 +115,213 @@ function fakeGitHub({ pulls = [], comments = [], labels = [], releases = [], job }, }, actions: { - listJobsForWorkflowRun: async () => ({ data: { total_count: jobs.length, jobs } }), - listWorkflowRunArtifacts: async () => ({ data: { total_count: artifacts.length, artifacts } }), + listWorkflowRuns: async ({ head_sha }) => ({ data: { workflow_runs: state.runs.filter((run) => run.head_sha === head_sha) } }), + listJobsForWorkflowRun: async ({ run_id }) => ({ data: { jobs: state.jobs[run_id] ?? [] } }), + listWorkflowRunArtifacts: async ({ run_id }) => ({ data: { artifacts: state.artifacts[run_id] ?? [] } }), }, }; const paginate = async (method, params) => { const { data } = await method(params); - return Array.isArray(data) ? data : data.jobs ?? data.artifacts; + return Array.isArray(data) ? data : data.jobs ?? data.artifacts ?? data.workflow_runs; + }; + /** Time passes; then a maintainer adds a label. */ + const label = (name = "visual-ok") => { + state.now = new Date(state.now.getTime() + 60_000); + state.labels.push(name); + state.events.push({ event: "labeled", label: { name }, created_at: at() }); + }; + /** A push or a rerun: a new finished CI run for `sha`, with these jobs and artifacts. */ + const run = ({ sha = head, jobs = conclusions, artifacts = ["screens-app"] } = {}) => { + state.now = new Date(state.now.getTime() + 60_000); + const id = state.next++; + state.runs.push({ id, run_number: id, run_attempt: 1, head_sha: sha, path: ".github/workflows/ci.yml", pull_requests: [] }); + state.jobs[id] = jobsOf(jobs); + state.artifacts[id] = artifacts.map((name) => ({ name })); + return id; }; - return { github: { rest, paginate }, state }; + return { github: { rest, paginate }, state, label, run }; } -const pr = (extra = {}) => ({ number: 7, state: "open", head: { sha: head, ref: "feat/x", repo: { owner: { login: "someone" } } }, - base: { sha: base, ref: "main" }, ...extra }); -const runContext = (extra = {}) => ({ ...repo, repo, serverUrl: "https://github.com", runId: 5, payload: { +const runEvent = (extra = {}) => ({ repo, serverUrl: "https://github.com", runId: 5, payload: { repository: { default_branch: "main" }, workflow_run: { id: 99, event: "pull_request", path: ".github/workflows/ci.yml", head_sha: head, head_branch: "feat/x", head_repository: { owner: { login: "someone" } }, pull_requests: [], ...extra } } }); -const labelContext = (action, sha = head) => ({ repo, payload: { action, label: { name: "visual-ok" }, pull_request: pr({ head: { ...pr().head, sha } }) } }); -const ran = (extra = {}) => ({ number: 7, head, runId: 99, jobs: { current: "success", build: "success", app: "success", site: "skipped" }, - artifacts: { app: true, site: false }, expected: { app: true, site: false }, ...extra }); -const jobsOf = (conclusions) => Object.entries(conclusions).map(([name, conclusion]) => ({ name, conclusion })); - -test("findRun finds a fork's pull request by its branch and head, with what its jobs did and should have done", async () => { - const { github } = fakeGitHub({ pulls: [pr()], artifacts: [{ name: "screens-app" }, { name: "release-build" }], files: ["Sources/StableBar/BarView.swift", "README.md"], - jobs: jobsOf({ "Check revision": "success", "Build release": "success", "Draw screens": "success", "Draw website screens": "skipped", "Test": "success" }) }); - assert.deepEqual(await findRun({ github, context: runContext() }), { number: 7, head, runId: 99, - jobs: { current: "success", build: "success", app: "success", site: "skipped" }, artifacts: { app: true, site: false }, expected: { app: true, site: false } }); - // A job the run doesn't have is missing, not skipped. - const { github: renamed } = fakeGitHub({ pulls: [pr()], jobs: jobsOf({ "Check revision": "success" }) }); - assert.deepEqual((await findRun({ github: renamed, context: runContext() })).jobs, { current: "success", build: "missing", app: "missing", site: "missing" }); -}); +const prEvent = (action, extra = {}) => ({ repo, serverUrl: "https://github.com", runId: 6, payload: { + repository: { default_branch: "main" }, action, label: { name: "visual-ok" }, pull_request: { number: 7 }, ...extra } }); +/** What screens.yml does for one event: plan, download the run's artifacts, apply. */ +const reconcile = async (github, context, folders = {}) => apply({ github, context, plan: await plan({ github, context }), folders }); +const comment = (state) => state.comments.find((entry) => entry.user.login === bot.login); -test("findRun takes only runs of the CI workflow file", async () => { - const { github } = fakeGitHub({ pulls: [pr()] }); - assert.equal(await findRun({ github, context: runContext({ path: ".github/workflows/other.yml" }) }), null); -}); - -test("changePatterns call for screens when the files the scenes read change", () => { - for (const file of ["Sources/StableBar/BarView.swift", "Resources/config.json", "scenes/starter-bar.json", "examples/essentials.json"]) { - assert.equal(changePatterns.app.test(file), true, file); - } - assert.equal(changePatterns.app.test("docs/README.md"), false); - assert.equal(changePatterns.site.test("site/index.html"), true); -}); - -test("changePatterns match the files CI's Check revision job draws screens for", () => { - const workflow = readFileSync(new URL("../workflows/ci.yml", import.meta.url), "utf8"); - const found = Object.fromEntries([...workflow.matchAll(/grep -qE '([^']+)' <<< "\$changed"; then\n\s+echo "(\w+)=true"/g)] - .map(([, pattern, output]) => [output, pattern])); - // A JavaScript pattern's source writes "/" as "\/". - const grep = (pattern) => pattern.source.replaceAll("\\/", "/"); - assert.deepEqual({ app: grep(changePatterns.app), site: grep(changePatterns.site) }, { app: found.screens, site: found.site }); +test("plan finds a fork's pull request, and the newest finished CI run for its head", async () => { + const { github, state, run } = fakeGitHub({ files: ["Sources/StableBar/BarView.swift", "README.md"] }); + assert.deepEqual(await plan({ github, context: runEvent() }), { action: "publish", number: 7, run: { id: 99, attempt: 1, head, + jobs: { current: "success", build: "success", app: "success", site: "skipped" }, artifacts: { app: true, site: false }, expected: { app: true, site: false } } }); + // A newer run for the head, a rerun or a run another event started, is the one to publish. + const newer = run({ jobs: { "Check revision": "success" }, artifacts: [] }); + const chosen = await plan({ github, context: prEvent("labeled") }); + assert.deepEqual([chosen.run.id, chosen.run.jobs], [newer, { current: "success", build: "missing", app: "missing", site: "missing" }]); + assert.equal(state.uploads.length, 0); }); -test("findRun skips runs that aren't for an open pull request's current head", async () => { - const { github } = fakeGitHub({ pulls: [pr({ head: { ...pr().head, sha: oldHead } })] }); - assert.equal(await findRun({ github, context: runContext() }), null); - assert.equal(await findRun({ github, context: runContext({ event: "push" }) }), null); - const { github: closed } = fakeGitHub({ pulls: [pr({ state: "closed" })] }); - assert.equal(await findRun({ github: closed, context: runContext({ pull_requests: [{ number: 7 }] }) }), null); +test("plan takes only CI's pull request runs, and leaves a run already shown to review", async () => { + const { github } = fakeGitHub(); + assert.deepEqual(await plan({ github, context: runEvent({ path: ".github/workflows/other.yml" }) }), { action: "none" }); + assert.deepEqual(await plan({ github, context: runEvent({ event: "push" }) }), { action: "none" }); + await reconcile(github, runEvent(), { app: artifact() }); + assert.deepEqual(await plan({ github, context: runEvent() }), { action: "review", number: 7 }); }); -test("publishRun uploads the changed screens, comments, sets the status and drops older heads' images", async () => { +test("a publication uploads the changed screens, comments, sets the status and drops older heads' images", async () => { const old = { id: 1, name: `pr7-${oldHead.slice(0, 7)}-docs.hero-dark-after.png` }, other = { id: 2, name: "pr8-1234567-docs.a-after.png" }; - const { github, state } = fakeGitHub({ pulls: [pr()], releases: [{ id: 3, tag_name: "pr-previews-2026-09", assets: [old, other] }] }); - const review = await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); + const { github, state } = fakeGitHub({ releases: [{ id: 3, tag_name: "pr-previews-2026-09", assets: [old, other] }] }); + const { review } = await reconcile(github, runEvent(), { app: artifact() }); assert.deepEqual(review, { state: "pending", description: "1 screen changed: a maintainer adds visual-ok after looking.", removeLabel: false }); const release = state.releases.find((entry) => entry.tag_name === releaseTag(new Date())); assert.deepEqual([release.prerelease, release.make_latest, release.target_commitish], [true, "false", "main"]); assert.deepEqual(state.uploads.map((upload) => [upload.name, upload.type]).sort(), ["after", "before", "diff"].map((kind) => [`pr7-aaaaaaa-docs~hero-dark-${kind}.png`, "image/png"])); - const [comment] = state.comments; - assert.match(comment.body, /src="https:\/\/example\.test\/download\/pr7-aaaaaaa-docs\.hero-dark-after\.png"/); - const data = readData(comment.body); - assert.deepEqual([data.head, data.base, data.ranScreens, data.total, data.changed, data.approved], [head, base, true, 2, 1, null]); + assert.match(comment(state).body, /src="https:\/\/example\.test\/download\/pr7-aaaaaaa-docs\.hero-dark-after\.png"/); + const data = readData(comment(state).body); + assert.deepEqual([data.head, data.base, data.baseRef, data.runId, data.runAttempt, data.total, data.changed, data.shownAt], + [head, base, "main", 99, 1, 2, 1, "2026-10-31T10:00:00.000Z"]); assert.deepEqual(state.statuses, [{ ...repo, sha: head, state: "pending", context: "Visual review", description: review.description, - target_url: comment.html_url }]); + target_url: comment(state).html_url }]); assert.deepEqual(state.deleted, [old.name]); + // Reconciling again uploads nothing new. + await reconcile(github, runEvent(), { app: artifact() }); + assert.equal(state.uploads.length, 3); }); -test("visual-ok approves the screens shown, survives a rerun of them and comes off for others", async () => { - const { github, state } = fakeGitHub({ pulls: [pr()] }); - await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); - const shown = readData(state.comments[0].body).fingerprint; - state.labels = ["visual-ok"]; - assert.deepEqual(await updateReview({ github, context: labelContext("labeled") }), - { state: "success", description: "1 changed screen approved.", removeLabel: false }); - assert.equal(readData(state.comments[0].body).approved, shown); +test("visual-ok approves the screens shown before it, survives a rerun of them, and comes off for others", async () => { + const { github, state, label, run } = fakeGitHub(); + await reconcile(github, runEvent(), { app: artifact() }); + label(); + assert.deepEqual((await reconcile(github, prEvent("labeled"))).review, { state: "success", description: "1 changed screen approved.", removeLabel: false }); assert.equal(state.statuses.at(-1).state, "success"); - const rerun = await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); - assert.deepEqual([rerun.state, state.labels], ["success", ["visual-ok"]]); - const changed = await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact([255, 0, 0, 255]) } }); - assert.deepEqual([changed.state, state.labels, readData(state.comments[0].body).approved], ["pending", [], null]); - assert.match(state.comments[0].body, /`visual-ok` was taken off/); + run(); + assert.deepEqual([(await reconcile(github, runEvent(), { app: artifact() })).review.state, state.labels], ["success", ["visual-ok"]]); + run(); + const changed = (await reconcile(github, runEvent(), { app: artifact([255, 0, 0, 255]) })).review; + assert.deepEqual([changed.state, state.labels], ["pending", []]); + assert.match(comment(state).body, /`visual-ok` was taken off/); assert.equal(state.comments.length, 1); assert.equal(state.statuses.at(-1).state, "pending"); }); -test("removing visual-ok sends the review back to pending", async () => { - const { github, state } = fakeGitHub({ pulls: [pr()], labels: ["visual-ok"] }); - await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); - await updateReview({ github, context: labelContext("labeled") }); - state.labels = []; - const review = await updateReview({ github, context: labelContext("unlabeled") }); - assert.equal(review.state, "pending"); - assert.equal(readData(state.comments[0].body).approved, null); +test("an approval holds even when the event of its label never ran", async () => { + const { github, state, label, run } = fakeGitHub(); + await reconcile(github, runEvent(), { app: artifact() }); + label(); + // GitHub dropped the label's reconcile; the next run's reconcile still sees the label came after the screens. + run(); + assert.equal((await reconcile(github, runEvent(), { app: artifact() })).review.state, "success"); + assert.deepEqual(state.labels, ["visual-ok"]); }); -test("updateReview ignores lookalike comments, and leaves a newer head's status to its run", async () => { - const lookalike = { id: 1, user: { login: "someone" }, body: `\n` }; - const { github, state } = fakeGitHub({ pulls: [pr()], comments: [lookalike], labels: ["visual-ok"] }); - assert.equal(await updateReview({ github, context: labelContext("labeled") }), null); - assert.equal(state.statuses.length, 0); - await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); - // The label came before any screens were drawn, so it approved none, and comes off. +test("visual-ok added before the screens appeared comes off, and removing it sends the review back to pending", async () => { + const { github, state, label } = fakeGitHub(); + label(); + assert.equal((await reconcile(github, runEvent(), { app: artifact() })).review.state, "pending"); assert.deepEqual(state.labels, []); - state.labels = ["visual-ok"]; + label(); + assert.equal((await reconcile(github, prEvent("labeled"))).review.state, "success"); + state.labels = []; + assert.equal((await reconcile(github, prEvent("unlabeled"))).review.state, "pending"); +}); + +test("a label added while a newer head is drawn leaves the status to that head's run", async () => { + const { github, state, label } = fakeGitHub(); + await reconcile(github, runEvent(), { app: artifact() }); + state.pr.head = { ...state.pr.head, sha: oldHead }; + label(); const statuses = state.statuses.length; - await updateReview({ github, context: labelContext("labeled", oldHead) }); + assert.equal((await reconcile(github, prEvent("labeled"))).action, "review"); assert.equal(state.statuses.length, statuses); - assert.equal(readData(state.comments[1].body).approved, readData(state.comments[1].body).fingerprint); }); -test("publishRun answers a run that drew no screens", async () => { - const skipped = ran({ jobs: { current: "success", build: "skipped", app: "skipped", site: "skipped" }, artifacts: { app: false, site: false }, - expected: { app: false, site: false } }); - const { github, state } = fakeGitHub({ pulls: [pr()] }); - assert.deepEqual(await publishRun({ github, context: runContext(), run: skipped, folders: {} }), - { state: "success", description: "No screens to compare.", removeLabel: false }); +test("a changed base sends the review back to pending until a new run", async () => { + const { github, state, label } = fakeGitHub(); + await reconcile(github, runEvent(), { app: artifact() }); + label(); + await reconcile(github, prEvent("labeled")); + state.pr.base = { sha: "d".repeat(40), ref: "develop" }; + const { review } = await reconcile(github, prEvent("edited", { changes: { base: { ref: { from: "main" } } } })); + assert.deepEqual(review, { state: "pending", description: "The base changed: push a commit to compare the screens with it.", removeLabel: false }); + assert.match(comment(state).body, /\*\*Visual review:\*\* The base changed/); +}); + +test("lookalike comments are ignored", async () => { + const lookalike = { id: 1, user: { login: "someone" }, body: `\n` }; + const { github, state, label } = fakeGitHub({ comments: [lookalike] }); + label(); + assert.equal((await reconcile(github, prEvent("labeled"))).action, "publish"); + assert.equal(state.comments.length, 2); + assert.equal(state.statuses.at(-1).state, "failure"); +}); + +test("a run that drew no screens passes without a comment, or rewrites the one an earlier head left", async () => { + const { github, state, run } = fakeGitHub(); + state.jobs[99] = jobsOf({ "Check revision": "success", "Build release": "skipped", "Draw screens": "skipped", "Draw website screens": "skipped" }); + state.artifacts[99] = []; + state.files = ["docs/README.md"]; + assert.deepEqual((await reconcile(github, runEvent())).review, { state: "success", description: "No screens to compare.", removeLabel: false }); assert.deepEqual([state.comments.length, state.statuses[0].state], [0, "success"]); - await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); - await publishRun({ github, context: runContext(), run: skipped, folders: {} }); - assert.match(state.comments[0].body, /changes neither the app nor the website/); - assert.equal(readData(state.comments[0].body).ranScreens, false); + state.files = ["Sources/StableBar/BarView.swift"]; + run(); + await reconcile(github, runEvent(), { app: artifact() }); + run({ jobs: { "Check revision": "success", "Build release": "skipped", "Draw screens": "skipped", "Draw website screens": "skipped" }, artifacts: [] }); + state.files = ["docs/README.md"]; + await reconcile(github, runEvent()); + assert.match(comment(state).body, /changes neither the app nor the website/); + assert.equal(readData(comment(state).body).ranScreens, false); }); -test("publishRun reports failed jobs and refuses unusable artifacts", async () => { - const { github, state } = fakeGitHub({ pulls: [pr()] }); - const blocked = await publishRun({ github, context: runContext(), folders: {}, - run: ran({ jobs: { current: "success", build: "failure", app: "skipped", site: "skipped" }, artifacts: { app: false, site: false } }) }); - assert.equal(blocked.state, "failure"); - assert.match(state.comments[0].body, /- `app`: `Build release ended with failure, so Draw screens didn't run\.`/); - const folder = artifact(); - symlinkSync("/etc/hosts", join(folder, "hosts.png")); - const refused = await publishRun({ github, context: runContext(), run: ran(), folders: { app: folder } }); - assert.equal(refused.state, "failure"); - assert.match(state.comments[0].body, /The screens-app artifact can't be used: 'hosts\.png': not a regular file\./); - const site = await publishRun({ github, context: runContext(), folders: { site: artifact() }, - run: ran({ jobs: { current: "success", build: "skipped", app: "skipped", site: "success" }, artifacts: { app: false, site: true }, - expected: { app: false, site: true } }) }); - assert.equal(site.state, "failure"); - assert.match(state.comments[0].body, /its id isn't site\/ followed by a plain name/); - // Screens the changed files call for, skipped or missing from the run, fail rather than pass as none. - const missing = await publishRun({ github, context: runContext(), folders: {}, - run: ran({ jobs: { current: "success", build: "success", app: "missing", site: "skipped" }, artifacts: { app: false, site: false } }) }); - assert.equal(missing.state, "failure"); - assert.match(state.comments[0].body, /- `app`: `Draw screens is missing from this CI run\.`/); - const skipped = await publishRun({ github, context: runContext(), folders: {}, - run: ran({ jobs: { current: "success", build: "success", app: "skipped", site: "skipped" }, artifacts: { app: false, site: false } }) }); - assert.equal(skipped.state, "failure"); - assert.match(state.comments[0].body, /- `app`: `Draw screens was skipped, but this pull request changes what it draws\.`/); - assert.equal(state.uploads.length, 0); +test("failed and missing jobs, and screens the changed files call for, fail the review", async () => { + const cases = [ + [{ "Check revision": "success", "Build release": "failure", "Draw screens": "skipped", "Draw website screens": "skipped" }, [], + "Build release ended with failure, so Draw screens didn't run."], + [{ "Check revision": "success", "Build release": "success", "Draw website screens": "skipped" }, [], "Draw screens is missing from this CI run."], + [{ "Check revision": "success", "Build release": "success", "Draw screens": "skipped", "Draw website screens": "skipped" }, [], + "Draw screens was skipped, but this pull request changes what it draws."], + [{ ...conclusions, "Draw screens": "cancelled" }, [], "Draw screens ended with cancelled."], + ]; + for (const [jobs, artifacts, error] of cases) { + const { github, state } = fakeGitHub(); + state.jobs[99] = jobsOf(jobs); + state.artifacts[99] = artifacts; + assert.equal((await reconcile(github, runEvent())).review.state, "failure", error); + assert.ok(comment(state).body.includes(`- \`app\`: \`${error}\``), error); + } +}); + +test("unusable artifacts are refused whole", async () => { + const refused = async (folder, expected, kind = "app") => { + const { github, state } = fakeGitHub(); + if (kind === "site") { + state.jobs[99] = jobsOf({ "Check revision": "success", "Build release": "skipped", "Draw screens": "skipped", "Draw website screens": "success" }); + state.artifacts[99] = [{ name: "screens-site" }]; + state.files = ["site/index.html"]; + } + assert.equal((await reconcile(github, runEvent(), { [kind]: folder })).review.state, "failure"); + assert.match(comment(state).body, expected); + assert.equal(state.uploads.length, 0); + }; + const linked = artifact(); + symlinkSync("/etc/hosts", join(linked, "hosts.png")); + await refused(linked, /The screens-app artifact can't be used: 'hosts\.png': not a regular file\./); + await refused(artifact(), /its id isn't site\/ followed by a plain name/, "site"); + const unreadable = artifact(); + writeFileSync(join(unreadable, "docs~hero-dark-diff.png"), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])); + await refused(unreadable, /'docs~hero-dark-diff\.png' isn't a PNG the bot can read/); + const empty = mkdtempSync(join(tmpdir(), "artifact-")); + writeFileSync(join(empty, "manifest.json"), JSON.stringify({ screens: [], failed: [] })); + await refused(empty, /The screens-app artifact can't be used: It holds no screens\./); + const scenes = mkdtempSync(join(tmpdir(), "artifact-")); + writeFileSync(join(scenes, "manifest.json"), JSON.stringify({ screens: [{ id: "scenes/bar-dark", status: "unchanged", changedPixels: 0, + region: null, images: {} }], failed: [] })); + await refused(scenes, /It holds no docs images\./); }); /** `file` with a text chunk after its header: other bytes, the same pixels. */ @@ -281,72 +335,73 @@ function withText(file) { writeFileSync(file, Buffer.concat([png.subarray(0, 33), length, body, crc, png.subarray(33)])); } -test("publishRun uploads its own encoding of the pixels, and fingerprints the pixels", async () => { - const { github, state } = fakeGitHub({ pulls: [pr()] }); +test("a publication uploads its own encoding of the pixels, and fingerprints the pixels", async () => { + const { github, state, run } = fakeGitHub(); const plain = artifact(); - await publishRun({ github, context: runContext(), run: ran(), folders: { app: plain } }); - const first = readData(state.comments[0].body).fingerprint; + await reconcile(github, runEvent(), { app: plain }); + const first = readData(comment(state).body).fingerprint; const uploaded = state.uploads.find((upload) => upload.name.endsWith("after.png")); assert.deepEqual(uploaded.data, encodePNG(decodePNG(readFileSync(join(plain, "docs~hero-dark-after.png"))))); const texted = artifact(); for (const kind of ["before", "after", "diff"]) withText(join(texted, `docs~hero-dark-${kind}.png`)); - await publishRun({ github, context: runContext(), run: ran(), folders: { app: texted } }); - assert.equal(readData(state.comments[0].body).fingerprint, first); + run(); + await reconcile(github, runEvent(), { app: texted }); + assert.equal(readData(comment(state).body).fingerprint, first); assert.ok(!state.uploads.at(-1).data.includes("from a pull request")); }); -test("publishRun refuses an image it can't decode", async () => { - const { github, state } = fakeGitHub({ pulls: [pr()] }); - const folder = artifact(); - writeFileSync(join(folder, "docs~hero-dark-diff.png"), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])); - const review = await publishRun({ github, context: runContext(), run: ran(), folders: { app: folder } }); - assert.equal(review.state, "failure"); - assert.match(state.comments[0].body, /'docs~hero-dark-diff\.png' isn't a PNG the bot can read/); - assert.equal(state.uploads.length, 0); -}); - -test("publishRun fails an artifact without the screens it always has", async () => { - const { github, state } = fakeGitHub({ pulls: [pr()] }); - const empty = mkdtempSync(join(tmpdir(), "artifact-")); - writeFileSync(join(empty, "manifest.json"), JSON.stringify({ screens: [], failed: [] })); - assert.equal((await publishRun({ github, context: runContext(), run: ran(), folders: { app: empty } })).state, "failure"); - assert.match(state.comments[0].body, /The screens-app artifact can't be used: It holds no screens\./); - // Draw screens always draws the docs images. - const scenes = mkdtempSync(join(tmpdir(), "artifact-")); - writeFileSync(join(scenes, "manifest.json"), JSON.stringify({ screens: [{ id: "scenes/bar-dark", status: "unchanged", changedPixels: 0, - region: null, images: {} }], failed: [] })); - assert.equal((await publishRun({ github, context: runContext(), run: ran(), folders: { app: scenes } })).state, "failure"); - assert.match(state.comments[0].body, /It holds no docs images\./); -}); - -test("publishRun moves on to the month's next release when one is full", async () => { +test("a publication moves on to the month's next release when one is full", async () => { const month = releaseTag(new Date()); const full = { id: 3, tag_name: month, assets: Array.from({ length: 999 }, (_, index) => ({ id: 10000 + index, name: `pr1-1234567-${index}.png` })) }; - const { github, state } = fakeGitHub({ pulls: [pr()], releases: [full] }); - await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); + const { github, state } = fakeGitHub({ releases: [full] }); + await reconcile(github, runEvent(), { app: artifact() }); const next = state.releases.find((release) => release.tag_name === `${month}-2`); assert.deepEqual([full.assets.length, next?.assets.length, next?.prerelease], [999, 3, true]); - assert.match(state.comments[0].body, /download\/pr7-aaaaaaa-docs\.hero-dark-after\.png/); + assert.match(comment(state).body, /download\/pr7-aaaaaaa-docs\.hero-dark-after\.png/); +}); + +test("closing unmerged deletes the images from every month; merging keeps them; a closed pull request publishes nothing", async () => { + const { github, state } = fakeGitHub({ releases: [ + { id: 1, tag_name: "pr-previews-2026-09", assets: [{ id: 11, name: "pr8-aaaaaaa-a.png" }] }, + { id: 2, tag_name: "pr-previews-2026-10", assets: [{ id: 21, name: "pr7-bbbbbbb-b.png" }] }, + { id: 3, tag_name: "v1.0.0", assets: [{ id: 31, name: "pr7-ccccccc-StableBar.dmg" }] }] }); + await reconcile(github, runEvent(), { app: artifact() }); + state.pr.state = "closed"; + assert.deepEqual(await reconcile(github, prEvent("closed")), { action: "close", deleted: 3 }); + assert.deepEqual(state.releases.slice(0, 3).map((release) => [release.tag_name, release.assets.map((asset) => asset.name)]), + [["pr-previews-2026-09", ["pr8-aaaaaaa-a.png"]], ["pr-previews-2026-10", []], ["v1.0.0", ["pr7-ccccccc-StableBar.dmg"]]]); + assert.match(comment(state).body, /closed without merging, so the images of its screens were deleted/); + assert.deepEqual(await reconcile(github, runEvent(), { app: artifact() }), { action: "none" }); + assert.equal(state.uploads.length, 3); + const merged = fakeGitHub(); + await reconcile(merged.github, runEvent(), { app: artifact() }); + Object.assign(merged.state.pr, { state: "closed", merged: true }); + assert.deepEqual(await reconcile(merged.github, prEvent("closed")), { action: "close", deleted: 0 }); + assert.equal(merged.state.deleted.length, 0); }); -test("failRun marks the head when the bot itself fails", async () => { +test("failRun marks an open pull request's head when the bot itself fails", async () => { const { github, state } = fakeGitHub(); - await failRun({ github, context: runContext(), run: ran() }); + await failRun({ github, context: runEvent(), number: 7 }); assert.deepEqual(state.statuses, [{ ...repo, sha: head, state: "error", context: "Visual review", description: "The screens bot failed: see the Screens workflow run.", target_url: "https://github.com/Seb-GRAF/StableBar/actions/runs/5" }]); + await failRun({ github, context: runEvent(), number: undefined }); + assert.equal(state.statuses.length, 1); }); -test("deleteScreens deletes an unmerged pull request's images from every month", async () => { - const { github, state } = fakeGitHub({ pulls: [pr()], releases: [ - { id: 1, tag_name: "pr-previews-2026-09", assets: [{ id: 11, name: "pr7-aaaaaaa-a.png" }, { id: 12, name: "pr8-aaaaaaa-a.png" }] }, - { id: 2, tag_name: "pr-previews-2026-10", assets: [{ id: 21, name: "pr7-bbbbbbb-b.png" }] }, - { id: 3, tag_name: "v1.0.0", assets: [{ id: 31, name: "pr7-ccccccc-StableBar.dmg" }] }] }); - await publishRun({ github, context: runContext(), run: ran(), folders: { app: artifact() } }); - const close = { repo, payload: { action: "closed", pull_request: pr({ state: "closed", merged: false }) } }; - // Publishing replaced this head's earlier image and dropped the other head's; closing takes the three it uploaded. - assert.equal(await deleteScreens({ github, context: close }), 3); - assert.deepEqual(state.releases.map((release) => [release.tag_name, release.assets.map((asset) => asset.name)]).slice(0, 3), - [["pr-previews-2026-09", ["pr8-aaaaaaa-a.png"]], ["pr-previews-2026-10", []], ["v1.0.0", ["pr7-ccccccc-StableBar.dmg"]]]); - assert.match(state.comments[0].body, /closed without merging, so the images of its screens were deleted/); - assert.notEqual(readData(state.comments[0].body), null); +test("changePatterns call for screens when the files the scenes read change", () => { + for (const file of ["Sources/StableBar/BarView.swift", "Resources/config.json", "scenes/starter-bar.json", "examples/essentials.json"]) { + assert.equal(changePatterns.app.test(file), true, file); + } + assert.equal(changePatterns.app.test("docs/README.md"), false); + assert.equal(changePatterns.site.test("site/index.html"), true); +}); + +test("changePatterns match the files CI's Check revision job draws screens for", () => { + const workflow = readFileSync(new URL("../workflows/ci.yml", import.meta.url), "utf8"); + const found = Object.fromEntries([...workflow.matchAll(/grep -qE '([^']+)' <<< "\$changed"; then\n\s+echo "(\w+)=true"/g)] + .map(([, pattern, output]) => [output, pattern])); + // A JavaScript pattern's source writes "/" as "\/". + const grep = (pattern) => pattern.source.replaceAll("\\/", "/"); + assert.deepEqual({ app: grep(changePatterns.app), site: grep(changePatterns.site) }, { app: found.screens, site: found.site }); }); diff --git a/.github/scripts/screens-comment.mjs b/.github/scripts/screens-comment.mjs index 1a7721e..7fcd3a7 100644 --- a/.github/scripts/screens-comment.mjs +++ b/.github/scripts/screens-comment.mjs @@ -165,14 +165,18 @@ export function summary(screens, failed) { } /** - * The Visual review status, from a run's summary: success when nothing needs a look or a maintainer - * approved this exact fingerprint with visual-ok, pending while changed screens wait for one, failure - * when drawing failed. `removeLabel` asks to take visual-ok off when it approved something else. + * The Visual review status, from a run's summary: success when nothing needs a look or visual-ok was + * added after the comment began showing these screens (`labeledAt` after `shownAt`, both ISO times + * from GitHub), pending while changed screens wait for it, failure when drawing failed. Screens + * compared with a base the pull request no longer has wait for a new run. `removeLabel` asks to take + * visual-ok off when it came before these screens. */ -export function reviewState({ ranScreens, total = 0, changed = 0, failed = 0, failure = null, labels = [], approved = null, fingerprint: current = null }) { +export function reviewState({ ranScreens, total = 0, changed = 0, failed = 0, failure = null, labels = [], labeledAt = null, shownAt = null, + baseChanged = false }) { if (!ranScreens) return { state: "success", description: "No screens to compare.", removeLabel: false }; + if (baseChanged) return { state: "pending", description: "The base changed: push a commit to compare the screens with it.", removeLabel: false }; const labelled = labels.includes("visual-ok"); - const valid = labelled && approved !== null && approved === current; + const valid = labelled && labeledAt !== null && shownAt !== null && Date.parse(labeledAt) > Date.parse(shownAt); if (failed > 0) { if (valid) return { state: "success", description: `Approved with visual-ok although ${plural(failed, "screen")} failed to draw.`, removeLabel: false }; return { state: "failure", description: cut(`Drawing the screens failed: ${failure ?? "see the comment."}`, 140), removeLabel: labelled }; @@ -274,10 +278,16 @@ export function withReview(body, review, data) { return `${start >= 0 ? body.slice(0, start + 1) : `${body.trimEnd()}\n\n`}${reviewSection(review, data)}`; } +const isTime = (value) => typeof value === "string" && value.length <= 40 && !Number.isNaN(Date.parse(value)); + +/** The comment's data: what was drawn for which head and CI run, against which base, and since when it's shown. */ function isData(data) { - return hasOnly(data, ["head", "base", "ranScreens", "fingerprint", "approved", "total", "changed", "failed", "failure"]) - && isHex(data.head, 40) && (data.base === null || isHex(data.base, 40)) && typeof data.ranScreens === "boolean" - && (data.fingerprint === null || isHex(data.fingerprint, 64)) && (data.approved === null || isHex(data.approved, 64)) + return hasOnly(data, ["head", "base", "baseRef", "runId", "runAttempt", "ranScreens", "fingerprint", "shownAt", "total", "changed", "failed", "failure"]) + && isHex(data.head, 40) && (data.base === null || isHex(data.base, 40)) + && (data.baseRef === null || (typeof data.baseRef === "string" && data.baseRef.length > 0 && data.baseRef.length <= 255)) + && (data.runId === null || isCount(data.runId, 1)) && (data.runAttempt === null || isCount(data.runAttempt, 1)) + && typeof data.ranScreens === "boolean" + && (data.fingerprint === null || isHex(data.fingerprint, 64)) && (data.shownAt === null || isTime(data.shownAt)) && isCount(data.total) && isCount(data.changed) && isCount(data.failed) && (data.failure === null || (typeof data.failure === "string" && data.failure.length <= 300)); } diff --git a/.github/scripts/screens-comment.test.mjs b/.github/scripts/screens-comment.test.mjs index 9bec1ce..4090a55 100644 --- a/.github/scripts/screens-comment.test.mjs +++ b/.github/scripts/screens-comment.test.mjs @@ -9,7 +9,8 @@ const digest = (character) => character.repeat(64); const screen = (id, status, images = {}, extra = {}) => ({ id, status, changedPixels: status === "changed" ? 120 : 0, region: status === "changed" ? { x: 10, y: 20, width: 30, height: 40 } : null, images, ...(status === "changed" || status === "resized" ? { afterDigest: "e".repeat(64) } : {}), ...extra }); -const data = (extra = {}) => ({ head: sha("a"), base: sha("b"), ranScreens: true, fingerprint: digest("f"), approved: null, +const data = (extra = {}) => ({ head: sha("a"), base: sha("b"), baseRef: "main", runId: 41, runAttempt: 1, ranScreens: true, + fingerprint: digest("f"), shownAt: "2026-10-31T10:00:00.000Z", total: 2, changed: 1, failed: 0, failure: null, ...extra }); const urls = (file) => `https://example.test/${file}`; // What a pull request's CI job could put in a name or a message. @@ -128,33 +129,32 @@ test("summary counts the screens and keeps the first failure short", () => { }); test("reviewState follows the visual review rules", () => { - const run = { ranScreens: true, total: 2, changed: 1, failed: 0, failure: null }; + const shownAt = "2026-10-31T10:00:00.000Z", later = "2026-10-31T10:05:00Z", earlier = "2026-10-31T09:55:00Z"; + const run = { ranScreens: true, total: 2, changed: 1, failed: 0, failure: null, shownAt }; assert.deepEqual(reviewState({ ranScreens: false }), { state: "success", description: "No screens to compare.", removeLabel: false }); assert.deepEqual(reviewState({ ...run, changed: 0, labels: [] }), { state: "success", description: "2 screens, none changed.", removeLabel: false }); - assert.deepEqual(reviewState({ ...run, labels: [], fingerprint: digest("1") }), + assert.deepEqual(reviewState({ ...run, labels: [] }), { state: "pending", description: "1 screen changed: a maintainer adds visual-ok after looking.", removeLabel: false }); - assert.deepEqual(reviewState({ ...run, labels: ["visual-ok"], approved: digest("1"), fingerprint: digest("1") }), + // visual-ok approves the screens shown before it was added. + assert.deepEqual(reviewState({ ...run, labels: ["visual-ok"], labeledAt: later }), { state: "success", description: "1 changed screen approved.", removeLabel: false }); - const stale = reviewState({ ...run, labels: ["visual-ok"], approved: digest("0"), fingerprint: digest("1") }); - assert.deepEqual([stale.state, stale.removeLabel], ["pending", true]); - const early = reviewState({ ...run, labels: ["visual-ok"], approved: null, fingerprint: digest("1") }); - assert.deepEqual([early.state, early.removeLabel], ["pending", true]); + for (const labeledAt of [earlier, "2026-10-31T10:00:00Z", null]) { + const stale = reviewState({ ...run, labels: ["visual-ok"], labeledAt }); + assert.deepEqual([stale.state, stale.removeLabel], ["pending", true], String(labeledAt)); + } const failing = { ...run, failed: 1, failure: `scenes/x: draw: Unknown value ${"y".repeat(200)}` }; - const broken = reviewState({ ...failing, labels: [], fingerprint: digest("1") }); + const broken = reviewState({ ...failing, labels: [] }); assert.equal(broken.state, "failure"); assert.match(broken.description, /^Drawing the screens failed: scenes\/x/); assert.ok(Array.from(broken.description).length <= 140); - assert.deepEqual(reviewState({ ...failing, labels: ["visual-ok"], approved: digest("1"), fingerprint: digest("1") }), + assert.deepEqual(reviewState({ ...failing, labels: ["visual-ok"], labeledAt: later }), { state: "success", description: "Approved with visual-ok although 1 screen failed to draw.", removeLabel: false }); -}); - -test("an approval doesn't cover a failure that came after it", () => { - const screens = [screen("docs/a", "changed", { before: "b.png", after: "a.png", diff: "d.png" })]; - const failed = [{ id: "scenes/x", error: "draw: Unknown value" }]; - const approved = fingerprint({ screens, failed: [] }, () => "11"); - const review = reviewState({ ranScreens: true, ...summary(screens, failed), labels: ["visual-ok"], approved, - fingerprint: fingerprint({ screens, failed }, () => "11") }); - assert.deepEqual([review.state, review.removeLabel], ["failure", true]); + assert.deepEqual([reviewState({ ...failing, labels: ["visual-ok"], labeledAt: earlier }).state, reviewState({ ...failing, labels: ["visual-ok"], labeledAt: earlier }).removeLabel], + ["failure", true]); + // Screens compared with another base wait for a new run, approved or not. + assert.deepEqual(reviewState({ ...run, labels: ["visual-ok"], labeledAt: later, baseChanged: true }), + { state: "pending", description: "The base changed: push a commit to compare the screens with it.", removeLabel: false }); + assert.equal(reviewState({ ranScreens: false, baseChanged: true }).state, "success"); }); test("commentBody shows changed screens, folds the rest and keeps its data", () => { @@ -218,8 +218,8 @@ test("withReview replaces only the review line and the data", () => { const approved = { state: "success", description: "1 changed screen approved.", removeLabel: false }; const body = make(taken); assert.match(body, /`visual-ok` was taken off/); - const updated = withReview(body, approved, data({ approved: digest("f") })); - assert.equal(updated, make(approved, { approved: digest("f") })); + const updated = withReview(body, approved, data({ shownAt: "2026-10-31T11:00:00.000Z" })); + assert.equal(updated, make(approved, { shownAt: "2026-10-31T11:00:00.000Z" })); assert.doesNotMatch(updated, /taken off/); }); @@ -236,9 +236,12 @@ test("readData reads only the data on the comment's last line", () => { const line = (value) => ``; assert.deepEqual(readData(`\ntext\n${line(data())}`), data()); assert.deepEqual(readData(`\r\ntext\r\n${line(data())}\r\n`), data()); - assert.deepEqual(readData(`\n${line(data({ approved: digest("e") }))}\n${line(data())}`), data()); + assert.deepEqual(readData(`\n${line(data({ runId: 42 }))}\n${line(data())}`), data()); assert.equal(readData(`\n${line(data())}\nmore text`), null); - assert.equal(readData(`x\n${line({ ...data(), approved: "yes" })}`), null); + assert.equal(readData(`x\n${line({ ...data(), shownAt: "yesterday" })}`), null); + assert.equal(readData(`x\n${line({ ...data(), approved: null })}`), null); + assert.deepEqual(readData(`x\n${line(data({ ranScreens: false, fingerprint: null, shownAt: null, runId: null, runAttempt: null }))}`), + data({ ranScreens: false, fingerprint: null, shownAt: null, runId: null, runAttempt: null })); assert.equal(readData(`x\n${line({ ...data(), extra: 1 })}`), null); assert.equal(readData("no data here"), null); }); diff --git a/.github/workflows/screens.yml b/.github/workflows/screens.yml index ea115be..d65defa 100644 --- a/.github/workflows/screens.yml +++ b/.github/workflows/screens.yml @@ -3,15 +3,17 @@ name: Screens # Posts the before/after screens of a pull request and sets its Visual review status. CI's Draw # screens and Draw website screens jobs run the pull request's code and upload what they drew. This # workflow runs the default branch's copy of itself and of .github/scripts, reads those artifacts -# only as data, and holds the write token: it uploads the images to a monthly pre-release, writes -# the comment, follows the visual-ok label, and deletes an unmerged pull request's images when it -# closes. The logic and its tests are in .github/scripts/screens-bot.mjs and screens-comment.mjs. +# only as data, and holds the write token. Each event reconciles its pull request, one at a time: +# it publishes the newest finished CI run's screens to a monthly pre-release and the comment, sets +# the status from the visual-ok label's history, and deletes an unmerged pull request's images when +# it closes. Running it again for a pull request only brings everything up to date. The logic and +# its tests are in .github/scripts/screens-bot.mjs and screens-comment.mjs. on: workflow_run: workflows: [CI] types: [completed] pull_request_target: - types: [labeled, unlabeled, closed] + types: [labeled, unlabeled, edited, closed, reopened] permissions: actions: read @@ -20,16 +22,20 @@ permissions: pull-requests: write statuses: write -# One queue per pull request for CI runs and one for its label and close events: GitHub keeps only -# the newest pending run in a queue, so a label event never replaces a run's pending publication. -concurrency: - group: screens-${{ github.event_name }}-${{ github.event.workflow_run.head_repository.full_name || github.event.pull_request.head.repo.full_name }}-${{ github.event.workflow_run.head_branch || github.event.pull_request.head.ref }} - cancel-in-progress: false - jobs: - publish: - name: Publish screens - if: github.event_name == 'workflow_run' && github.event.workflow_run.event == 'pull_request' + reconcile: + name: Reconcile screens + if: >- + (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'pull_request') + || (github.event_name == 'pull_request_target' && ( + github.event.action == 'closed' || github.event.action == 'reopened' + || (github.event.action == 'edited' && github.event.changes.base) + || github.event.label.name == 'visual-ok')) + # One reconcile of a pull request at a time. GitHub keeps only the newest event waiting, but + # each reconcile reads everything afresh, so a dropped one loses nothing. + concurrency: + group: screens-${{ github.event.workflow_run.head_repository.full_name || github.event.pull_request.head.repo.full_name }}-${{ github.event.workflow_run.head_branch || github.event.pull_request.head.ref }} + cancel-in-progress: false runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -38,83 +44,48 @@ jobs: ref: ${{ github.event.repository.default_branch }} sparse-checkout: .github/scripts persist-credentials: false - - name: Find the pull request - id: run + - name: Plan + id: plan uses: actions/github-script@v8 with: script: | - const { findRun } = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); - const run = await findRun({ github, context }); - if (!run) return core.info("This run isn't for an open pull request's current head."); - core.setOutput("run", JSON.stringify(run)); - core.setOutput("app", run.artifacts.app ? "true" : ""); - core.setOutput("site", run.artifacts.site ? "true" : ""); - # A download that fails leaves its folder missing, which the next step reports. + const { plan } = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); + const chosen = await plan({ github, context }); + core.info(`Plan: ${chosen.action}${chosen.number ? ` for #${chosen.number}` : ""}${chosen.run ? ` from run ${chosen.run.id}` : ""}`); + core.setOutput("plan", JSON.stringify(chosen)); + core.setOutput("run-id", chosen.run?.id ?? ""); + core.setOutput("app", chosen.run?.artifacts.app ? "true" : ""); + core.setOutput("site", chosen.run?.artifacts.site ? "true" : ""); + # A download that fails leaves its folder missing, which Apply reports. - uses: actions/download-artifact@v8 - if: steps.run.outputs.app + if: steps.plan.outputs.app continue-on-error: true with: name: screens-app path: ${{ runner.temp }}/screens-app - run-id: ${{ github.event.workflow_run.id }} + run-id: ${{ steps.plan.outputs.run-id }} github-token: ${{ github.token }} - uses: actions/download-artifact@v8 - if: steps.run.outputs.site + if: steps.plan.outputs.site continue-on-error: true with: name: screens-site path: ${{ runner.temp }}/screens-site - run-id: ${{ github.event.workflow_run.id }} + run-id: ${{ steps.plan.outputs.run-id }} github-token: ${{ github.token }} - - name: Publish - if: steps.run.outputs.run + - name: Apply uses: actions/github-script@v8 env: - RUN: ${{ steps.run.outputs.run }} + PLAN: ${{ steps.plan.outputs.plan }} with: script: | const bot = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); - const run = JSON.parse(process.env.RUN); + const chosen = JSON.parse(process.env.PLAN); const folders = { app: `${process.env.RUNNER_TEMP}/screens-app`, site: `${process.env.RUNNER_TEMP}/screens-site` }; try { - const review = await bot.publishRun({ github, context, run, folders }); - core.info(`Visual review: ${review.state}, ${review.description}`); + const result = await bot.apply({ github, context, plan: chosen, folders }); + core.info(result.review ? `Visual review: ${result.review.state}, ${result.review.description}` : JSON.stringify(result)); } catch (error) { - await bot.failRun({ github, context, run }); + await bot.failRun({ github, context, number: chosen.number }).catch((failure) => core.warning(failure.message)); throw error; } - - review: - name: Follow visual-ok - if: github.event_name == 'pull_request_target' && github.event.action != 'closed' && github.event.label.name == 'visual-ok' - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ github.event.repository.default_branch }} - sparse-checkout: .github/scripts - persist-credentials: false - - uses: actions/github-script@v8 - with: - script: | - const { updateReview } = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); - const review = await updateReview({ github, context }); - core.info(review ? `Visual review: ${review.state}, ${review.description}` : "There's no screens comment yet."); - - clean-up: - name: Delete unmerged screens - if: github.event_name == 'pull_request_target' && github.event.action == 'closed' && !github.event.pull_request.merged - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ github.event.repository.default_branch }} - sparse-checkout: .github/scripts - persist-credentials: false - - uses: actions/github-script@v8 - with: - script: | - const { deleteScreens } = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); - core.info(`Deleted ${await deleteScreens({ github, context })} images.`); From 8932b9b1196e69e34c28c19f39191cd2041d02e1 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 17:21:56 +0100 Subject: [PATCH 19/25] docs: describe the screens reconcile, crops of new heights and full releases The Before and after screens section now says when Screens runs and that running it again only brings a pull request up to date, how screens of a new height are cropped, that the images move on to -2 and on when a month's release is full, when visual-ok approves, and that a changed base sends the review back to pending. --- CONTRIBUTING.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73d73c3..84fe225 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -148,12 +148,12 @@ On a pull request, CI draws the screens twice, from the pull request and from it - **Draw screens** draws the [docs images](#docs-images) and the [scenes](#scenes) when the app, the scenes, the examples they read or the comparison scripts change. Both sides draw the pull request's scene files, each beside its own resources and examples, so a new scene shows how `main` draws it too. When the app didn't change, the pull request's app draws both sides. - **Draw website screens** captures the four pages of `site` at desktop (1440×900) and phone (390×844) size, in light and dark mode, when the website changes. The capture answers requests from the folder and fetches nothing else, seeds `Math.random` and stops the page's clock at the scenes' date, so two captures of one commit match. -Both sides are drawn in one job on one machine. A pixel counts as changed when a color channel moves by more than 1 and a pixel next to it changed too. +Both sides are drawn in one job on one machine. A pixel counts as changed when a color channel moves by more than 1 and a pixel next to it changed too. A screen whose height changed, such as a website page that grew, is compared from the top: its crop starts at the first change and runs across the full width. A crop shows at most 1,600 rows of a change, and the comment says so when it stops short; the approval still covers the whole image. -These jobs run the pull request's code with a read-only token. The **Screens** workflow runs from `main` when CI finishes, reads what they uploaded only as data, and: +These jobs run the pull request's code with a read-only token. The **Screens** workflow runs from `main` when CI finishes, and when a pull request gets or loses `visual-ok`, changes its base, closes or reopens. Each time it reads where the pull request stands, one pull request at a time, so running it again only brings everything up to date. It reads what the jobs uploaded only as data, and: -- comments once on the pull request and rewrites that comment on every push: a table of the changed screens, their before, after and difference images cropped to the change, the unchanged screens folded, and what couldn't be drawn; -- keeps the images as assets of a monthly prerelease, `pr-previews-YYYY-MM`, and deletes a head's images once a newer head replaces them. Closing a pull request without merging deletes all of its images; a merged one keeps its last; +- comments once on the pull request and rewrites that comment for each new CI run: a table of the changed screens, their before, after and difference images cropped to the change, the unchanged screens folded, and what couldn't be drawn; +- keeps the images as assets of a monthly prerelease, `pr-previews-YYYY-MM`, then `-2` and on when one is full, and deletes a head's images once a newer head replaces them. Closing a pull request without merging deletes all of its images; a merged one keeps its last; - sets the **Visual review** status on the pull request's head. | Visual review | When | @@ -161,10 +161,11 @@ These jobs run the pull request's code with a read-only token. The **Screens** w | Success: "No screens to compare." | The pull request changes neither the app nor the website. | | Success: "N screens, none changed." | Nothing looks different. | | Pending | Screens changed. A maintainer looks at them, then adds the `visual-ok` label. | -| Success: "M changed screens approved." | `visual-ok` approved exactly these screens. | +| Success: "M changed screens approved." | `visual-ok` was added after the comment began showing these screens. | +| Pending: "The base changed…" | The pull request's base changed since its screens were drawn. Push a commit to draw them again. | | Failure | Drawing failed; the comment says what. A `visual-ok` added while it showed the failure still approves it. | -`visual-ok` approves the screens the comment shows when it's added, so reload the pull request first if a push just landed. Any maintainer can add it, the author too. A later push that changes those screens, or what fails to draw, takes the label off and the review goes back to pending: look again, and add it again. Removing the label also sends the review back to pending. +`visual-ok` approves the screens the comment showed when the label was added, so reload the pull request first if a push just landed. Any maintainer can add it, the author too. A later run that shows the same screens keeps the approval. One that shows other screens, or other failures, takes the label off and the review goes back to pending: look again, and add it again. Removing the label also sends the review back to pending. Pull requests from forks get the same comment and status: the drawing runs in the fork's pull request, and **Screens** finds the pull request by its branch and head. The comparison runs in the pull request's own CI, so a pull request that changes the CI workflow or the drawing scripts decides what its comparison shows. **Screens** reads the pull request's changed files from GitHub, so screens they call for that weren't drawn fail the review, but review such a change itself before you trust its screens. From 962ad2654dc426d4bd8df0757d373c4f0f193ab4 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 17:28:06 +0100 Subject: [PATCH 20/25] ci: tie published screens to their own run, base and moment Three ways the reconcile could approve or publish the wrong screens: - A run that started before the base changed, published after, recorded the new base, and kept an approval given against the old one. The base change now comes from GitHub's base_ref_changed events: a run created before the latest one waits for a new run, and an approval doesn't carry across it. - For a fork, the newest run at the pull request's head commit counted, though another fork's pull request at the same commit has runs of its own. Only runs from the pull request's own head repository and branch count now. - shownAt was read before the uploads, so a label added during them approved screens the comment didn't show yet. New screens now take the time the comment was written, from GitHub's reply, and a publication cut short before that is repaired by the next reconcile from the comment's last write. --- .github/scripts/screens-bot.mjs | 68 +++++++++++------ .github/scripts/screens-bot.test.mjs | 94 ++++++++++++++++++++---- .github/scripts/screens-comment.mjs | 6 +- .github/scripts/screens-comment.test.mjs | 3 +- CONTRIBUTING.md | 2 +- 5 files changed, 129 insertions(+), 44 deletions(-) diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs index af32f92..8c7ce18 100644 --- a/.github/scripts/screens-bot.mjs +++ b/.github/scripts/screens-bot.mjs @@ -43,13 +43,16 @@ async function pullRequestOfRun({ github, owner, repo, run }) { return (pulls.find((pr) => pr.head.sha === run.head_sha) ?? pulls[0])?.number ?? null; } -/** The newest finished CI run for a head, preferring the runs GitHub lists the pull request on. */ -async function latestRun({ github, owner, repo, number, sha }) { +/** + * The newest finished CI run for the pull request's head, from its own head repository and branch: a + * pull request from another fork at the same commit has runs of its own, and they don't count. + */ +async function latestRun({ github, owner, repo, pr }) { const runs = (await github.paginate(github.rest.actions.listWorkflowRuns, - { owner, repo, workflow_id: "ci.yml", head_sha: sha, event: "pull_request", status: "completed", per_page: 100 })) - .filter((run) => run.path === workflowPath); - const listed = runs.filter((run) => run.pull_requests.some((pr) => pr.number === number)); - return [...(listed.length > 0 ? listed : runs)].sort((a, b) => b.run_number - a.run_number)[0] ?? null; + { owner, repo, workflow_id: "ci.yml", head_sha: pr.head.sha, event: "pull_request", status: "completed", per_page: 100 })) + .filter((run) => run.path === workflowPath && run.head_branch === pr.head.ref + && pr.head.repo?.full_name !== undefined && run.head_repository?.full_name === pr.head.repo.full_name); + return runs.sort((a, b) => b.run_number - a.run_number)[0] ?? null; } /** @@ -65,7 +68,7 @@ export async function plan({ github, context }) { if (!number) return { action: "none" }; const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: number }); if (pr.state !== "open") return { action: "close", number }; - const run = await latestRun({ github, owner, repo, number, sha: pr.head.sha }); + const run = await latestRun({ github, owner, repo, pr }); const existing = findScreensComment(await comments(github, owner, repo, number)); const data = existing ? readData(existing.body) : null; if (!run || (data?.runId === run.id && data?.runAttempt === run.run_attempt)) return { action: "review", number }; @@ -77,7 +80,7 @@ export async function plan({ github, context }) { return { action: "publish", number, run: { - id: run.id, attempt: run.run_attempt, head: pr.head.sha, + id: run.id, attempt: run.run_attempt, head: pr.head.sha, createdAt: run.created_at, jobs: Object.fromEntries(Object.entries(jobNames).map(([key, name]) => [key, jobs.find((job) => job.name === name)?.conclusion ?? "missing"])), artifacts: Object.fromEntries(Object.entries(kinds).map(([kind, { artifact }]) => [kind, uploaded.includes(artifact)])), expected: Object.fromEntries(Object.keys(kinds).map((kind) => [kind, files.some((file) => changePatterns[kind].test(file))])), @@ -134,12 +137,19 @@ export function readArtifact(folder, prefixes, always) { const comments = (github, owner, repo, number) => github.paginate(github.rest.issues.listComments, { owner, repo, issue_number: number, per_page: 100 }); -/** When visual-ok was last added to the pull request, as GitHub's ISO time, or null. */ -async function lastLabeled({ github, owner, repo, number }) { +/** When visual-ok was last added to the pull request, and when its base last changed, as GitHub's ISO times, or null. */ +async function history({ github, owner, repo, number }) { const events = await github.paginate(github.rest.issues.listEvents, { owner, repo, issue_number: number, per_page: 100 }); - return events.filter((event) => event.event === "labeled" && event.label?.name === approval).map((event) => event.created_at).sort().at(-1) ?? null; + const last = (matches) => events.filter(matches).map((event) => event.created_at).sort().at(-1) ?? null; + return { + labeledAt: last((event) => event.event === "labeled" && event.label?.name === approval), + baseChangedAt: last((event) => event.event === "base_ref_changed"), + }; } +/** Whether the pull request's base changed after `time`, an ISO time. */ +const changedSince = (baseChangedAt, time) => baseChangedAt !== null && time !== null && Date.parse(baseChangedAt) > Date.parse(time); + async function removeApproval({ github, owner, repo, number }) { try { await github.rest.issues.removeLabel({ owner, repo, issue_number: number, name: approval }); @@ -264,27 +274,32 @@ function gather(run, folders) { * Publishes a run from plan(): uploads the images of its changed screens, writes or rewrites the * comment, sets the status on the head, takes visual-ok off when it came before these screens, and then * deletes the images of the pull request's other heads. Screens the comment already showed keep the - * time they first appeared, and so their approval. + * time they first appeared, and so their approval, unless the base changed since. New screens appear + * when the comment is written: that write's time becomes `shownAt`, so a label added during the uploads + * approves nothing. A run that started before the base last changed compared the old base: its review + * waits for a new run. */ -async function publish({ github, owner, repo, pr, existing, data: previous, now, labels, labeledAt, run, folders, target }) { +async function publish({ github, owner, repo, pr, existing, data: previous, labels, labeledAt, baseChangedAt, run, folders, target }) { const number = pr.number, head = pr.head.sha; const { screens, failed, files, ranScreens } = gather(run, folders); - const drawn = { head, base: pr.base.sha, baseRef: pr.base.ref, runId: run.id, runAttempt: run.attempt }; - let review, body; + const drawn = { head, base: pr.base.sha, baseRef: pr.base.ref, runId: run.id, runAttempt: run.attempt, runCreatedAt: run.createdAt ?? null }; + const baseChanged = changedSince(baseChangedAt, drawn.runCreatedAt); + let review, body, data, shownLater = false; if (!ranScreens) { review = reviewState({ ranScreens: false }); - const data = { ...drawn, ranScreens: false, fingerprint: null, shownAt: null, total: 0, changed: 0, failed: 0, failure: null }; + data = { ...drawn, ranScreens: false, fingerprint: null, shownAt: null, total: 0, changed: 0, failed: 0, failure: null }; body = noScreensBody({ reason: `This pull request at ${head.slice(0, 7)} changes neither the app nor the website, so there are no screens to compare.`, review, data }); } else { const shown = screens.filter((screen) => screen.status !== "unchanged").flatMap((screen) => Object.values(screen.images)); const urls = await upload({ github, owner, repo, number, head, target, images: shown.map((file) => [file, files.get(file).data]) }); const print = fingerprint({ screens, failed }, (file) => files.get(file).digest); - const shownAt = previous?.fingerprint === print && previous.shownAt ? previous.shownAt : now; + const carried = previous?.fingerprint === print && previous.shownAt !== null && !baseChanged && !changedSince(baseChangedAt, previous.shownAt); const counts = summary(screens, failed); - review = reviewState({ ranScreens: true, ...counts, labels, labeledAt, shownAt }); + data = { ...drawn, ranScreens: true, fingerprint: print, shownAt: carried ? previous.shownAt : null, ...counts }; + shownLater = !carried; + review = reviewState({ ...data, labels, labeledAt, baseChanged }); if (review.removeLabel) await removeApproval({ github, owner, repo, number }); - const data = { ...drawn, ranScreens: true, fingerprint: print, shownAt, ...counts }; body = commentBody({ baseRef: pr.base.ref, base: pr.base.sha, head, screens, failed, imageURL: (file) => urls.get(file), review, data }); } let comment = existing; @@ -293,6 +308,10 @@ async function publish({ github, owner, repo, pr, existing, data: previous, now, } else if (ranScreens) { comment = (await github.rest.issues.createComment({ owner, repo, issue_number: number, body })).data; } + if (shownLater && comment) { + comment = (await github.rest.issues.updateComment({ owner, repo, comment_id: comment.id, + body: withReview(body, review, { ...data, shownAt: comment.updated_at }) })).data; + } await setStatus({ github, owner, repo, sha: head, review, url: comment?.html_url }); await deleteAssets({ github, owner, repo, pick: (assets) => staleAssets(assets, number, head) }); return review; @@ -303,9 +322,11 @@ async function publish({ github, owner, repo, pr, existing, data: previous, now, * whether the pull request still has the base its screens were compared with. Sets the status when * the comment is for the current head; a newer head's run sets its own. */ -async function review({ github, owner, repo, pr, existing, data, labels, labeledAt }) { - if (!data) return null; - const baseChanged = data.baseRef !== null && data.baseRef !== pr.base.ref; +async function review({ github, owner, repo, pr, existing, data: stored, labels, labeledAt, baseChangedAt }) { + if (!stored) return null; + // A publication cut short before it recorded when its screens appeared: the comment's last write is no earlier. + const data = stored.ranScreens && stored.shownAt === null ? { ...stored, shownAt: existing.updated_at } : stored; + const baseChanged = changedSince(baseChangedAt, data.runCreatedAt); const result = reviewState({ ...data, labels, labeledAt, baseChanged }); if (result.removeLabel) await removeApproval({ github, owner, repo, number: pr.number }); const body = withReview(existing.body, result, data); @@ -343,8 +364,7 @@ export async function apply({ github, context, plan: chosen, folders = {} }) { const existing = findScreensComment(await comments(github, owner, repo, number)); const state = { github, owner, repo, pr, existing, data: existing ? readData(existing.body) : null, - now: new Date(response.headers?.date ?? Date.now()).toISOString(), - labels: pr.labels.map((label) => label.name), labeledAt: await lastLabeled({ github, owner, repo, number }), + labels: pr.labels.map((label) => label.name), ...await history({ github, owner, repo, number }), }; if (chosen.action === "publish" && chosen.run.head === pr.head.sha) { return { action: "publish", review: await publish({ ...state, run: chosen.run, folders, target: context.payload.repository?.default_branch }) }; diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs index 219bc99..8403c4c 100644 --- a/.github/scripts/screens-bot.test.mjs +++ b/.github/scripts/screens-bot.test.mjs @@ -49,8 +49,11 @@ function artifact(color = [255, 255, 255, 255]) { function fakeGitHub({ pull = {}, comments = [], labels = [], releases = [], files = ["Sources/StableBar/BarView.swift"] } = {}) { const state = { now: new Date("2026-10-31T10:00:00Z"), comments, labels, releases, files, events: [], statuses: [], uploads: [], deleted: [], next: 1000, - pr: { number: 7, state: "open", merged: false, head: { sha: head, ref: "feat/x", repo: { owner: { login: "someone" } } }, base: { sha: base, ref: "main" }, ...pull }, - runs: [{ id: 99, run_number: 50, run_attempt: 1, head_sha: head, path: ".github/workflows/ci.yml", pull_requests: [] }], + pr: { number: 7, state: "open", merged: false, head: { sha: head, ref: "feat/x", repo: { owner: { login: "someone" }, full_name: "someone/StableBar" } }, + base: { sha: base, ref: "main" }, ...pull }, + runs: [{ id: 99, run_number: 50, run_attempt: 1, head_sha: head, path: ".github/workflows/ci.yml", pull_requests: [], created_at: "2026-10-31T09:50:00Z", + head_branch: "feat/x", head_repository: { full_name: "someone/StableBar" } }], + duringUpload: null, jobs: { 99: jobsOf(conclusions) }, artifacts: { 99: [{ name: "screens-app" }] }, }; const at = () => state.now.toISOString(); @@ -66,13 +69,14 @@ function fakeGitHub({ pull = {}, comments = [], labels = [], releases = [], file issues: { listComments: async () => ({ data: state.comments }), createComment: async ({ body }) => { - const comment = { id: state.next++, user: bot, body, html_url: `https://example.test/comment/${state.next}` }; + const comment = { id: state.next++, user: bot, body, html_url: `https://example.test/comment/${state.next}`, updated_at: at() }; state.comments.push(comment); return { data: comment }; }, updateComment: async ({ comment_id, body }) => { const comment = state.comments.find((entry) => entry.id === comment_id); comment.body = body; + comment.updated_at = at(); return { data: comment }; }, listEvents: async () => ({ data: state.events }), @@ -101,6 +105,13 @@ function fakeGitHub({ pull = {}, comments = [], labels = [], releases = [], file listReleases: async () => ({ data: state.releases }), listReleaseAssets: async ({ release_id }) => ({ data: state.releases.find((release) => release.id === release_id).assets }), uploadReleaseAsset: async ({ release_id, name, data, headers }) => { + // Uploads take a while: time passes, and a maintainer may act meanwhile. + state.now = new Date(state.now.getTime() + 10_000); + if (state.duringUpload) { + const act = state.duringUpload; + state.duringUpload = null; + act(); + } const renamed = name.replaceAll("~", "."); const asset = { id: state.next++, name: renamed, browser_download_url: `https://example.test/download/${renamed}` }; state.releases.find((release) => release.id === release_id).assets.push(asset); @@ -130,16 +141,23 @@ function fakeGitHub({ pull = {}, comments = [], labels = [], releases = [], file state.labels.push(name); state.events.push({ event: "labeled", label: { name }, created_at: at() }); }; - /** A push or a rerun: a new finished CI run for `sha`, with these jobs and artifacts. */ - const run = ({ sha = head, jobs = conclusions, artifacts = ["screens-app"] } = {}) => { + /** A push or a rerun: a new finished CI run for `sha`, with these jobs and artifacts, from the pull request's own fork unless `from` says otherwise. */ + const run = ({ sha = head, jobs = conclusions, artifacts = ["screens-app"], from = "someone/StableBar", created = null } = {}) => { state.now = new Date(state.now.getTime() + 60_000); const id = state.next++; - state.runs.push({ id, run_number: id, run_attempt: 1, head_sha: sha, path: ".github/workflows/ci.yml", pull_requests: [] }); + state.runs.push({ id, run_number: id, run_attempt: 1, head_sha: sha, path: ".github/workflows/ci.yml", pull_requests: [], + created_at: created ?? at(), head_branch: "feat/x", head_repository: { full_name: from } }); state.jobs[id] = jobsOf(jobs); state.artifacts[id] = artifacts.map((name) => ({ name })); return id; }; - return { github: { rest, paginate }, state, label, run }; + /** Time passes; then the pull request's base changes. */ + const retarget = (ref = "develop") => { + state.now = new Date(state.now.getTime() + 60_000); + state.pr.base = { sha: "d".repeat(40), ref }; + state.events.push({ event: "base_ref_changed", created_at: at() }); + }; + return { github: { rest, paginate }, state, label, run, retarget }; } const runEvent = (extra = {}) => ({ repo, serverUrl: "https://github.com", runId: 5, payload: { @@ -154,7 +172,7 @@ const comment = (state) => state.comments.find((entry) => entry.user.login === b test("plan finds a fork's pull request, and the newest finished CI run for its head", async () => { const { github, state, run } = fakeGitHub({ files: ["Sources/StableBar/BarView.swift", "README.md"] }); - assert.deepEqual(await plan({ github, context: runEvent() }), { action: "publish", number: 7, run: { id: 99, attempt: 1, head, + assert.deepEqual(await plan({ github, context: runEvent() }), { action: "publish", number: 7, run: { id: 99, attempt: 1, head, createdAt: "2026-10-31T09:50:00Z", jobs: { current: "success", build: "success", app: "success", site: "skipped" }, artifacts: { app: true, site: false }, expected: { app: true, site: false } } }); // A newer run for the head, a rerun or a run another event started, is the one to publish. const newer = run({ jobs: { "Check revision": "success" }, artifacts: [] }); @@ -182,8 +200,9 @@ test("a publication uploads the changed screens, comments, sets the status and d ["after", "before", "diff"].map((kind) => [`pr7-aaaaaaa-docs~hero-dark-${kind}.png`, "image/png"])); assert.match(comment(state).body, /src="https:\/\/example\.test\/download\/pr7-aaaaaaa-docs\.hero-dark-after\.png"/); const data = readData(comment(state).body); - assert.deepEqual([data.head, data.base, data.baseRef, data.runId, data.runAttempt, data.total, data.changed, data.shownAt], - [head, base, "main", 99, 1, 2, 1, "2026-10-31T10:00:00.000Z"]); + // The screens appeared when the comment was written, after the uploads. + assert.deepEqual([data.head, data.base, data.baseRef, data.runId, data.runAttempt, data.runCreatedAt, data.total, data.changed, data.shownAt], + [head, base, "main", 99, 1, "2026-10-31T09:50:00Z", 2, 1, "2026-10-31T10:00:30.000Z"]); assert.deepEqual(state.statuses, [{ ...repo, sha: head, state: "pending", context: "Visual review", description: review.description, target_url: comment(state).html_url }]); assert.deepEqual(state.deleted, [old.name]); @@ -239,20 +258,63 @@ test("a label added while a newer head is drawn leaves the status to that head's assert.equal(state.statuses.length, statuses); }); -test("a changed base sends the review back to pending until a new run", async () => { - const { github, state, label } = fakeGitHub(); +test("a changed base sends the review back to pending until a run started after it", async () => { + const { github, state, label, run, retarget } = fakeGitHub(); await reconcile(github, runEvent(), { app: artifact() }); label(); await reconcile(github, prEvent("labeled")); - state.pr.base = { sha: "d".repeat(40), ref: "develop" }; - const { review } = await reconcile(github, prEvent("edited", { changes: { base: { ref: { from: "main" } } } })); - assert.deepEqual(review, { state: "pending", description: "The base changed: push a commit to compare the screens with it.", removeLabel: false }); + const before = state.now.toISOString(); + retarget(); + const pending = { state: "pending", description: "The base changed: push a commit to compare the screens with it.", removeLabel: false }; + const edited = await reconcile(github, prEvent("edited", { changes: { base: { ref: { from: "main" } } } })); + assert.deepEqual([edited.action, edited.review], ["review", pending]); assert.match(comment(state).body, /\*\*Visual review:\*\* The base changed/); + // A run that started before the change, and finished after it, compared the old base: still pending. + const early = run({ created: before }); + const stale = await reconcile(github, runEvent(), { app: artifact() }); + assert.deepEqual([stale.action, readData(comment(state).body).runId, stale.review], ["publish", early, pending]); + // A run started after the change publishes, and the approval from before it doesn't count. + run(); + assert.deepEqual([(await reconcile(github, runEvent(), { app: artifact() })).review.state, state.labels], ["pending", []]); +}); + +test("a publication cut short before it recorded when its screens appeared is repaired by the next reconcile", async () => { + const { github, state, label } = fakeGitHub(); + const update = github.rest.issues.updateComment; + github.rest.issues.updateComment = async () => { + throw Object.assign(new Error("Server Error"), { status: 502 }); + }; + await assert.rejects(reconcile(github, runEvent(), { app: artifact() }), /Server Error/); + github.rest.issues.updateComment = update; + assert.equal(readData(comment(state).body).shownAt, null); + label(); + assert.equal((await reconcile(github, prEvent("labeled"))).review.state, "success"); +}); + +test("only the pull request's own head repository and branch publish screens", async () => { + const { github, state, run } = fakeGitHub(); + // Someone else's fork, at the same commit, with a newer run. + run({ from: "somebody-else/StableBar" }); + const chosen = await plan({ github, context: runEvent() }); + assert.equal(chosen.run.id, 99); + state.runs = state.runs.filter((entry) => entry.id !== 99); + assert.deepEqual(await plan({ github, context: runEvent() }), { action: "review", number: 7 }); +}); + +test("a label added while the screens upload doesn't approve them", async () => { + const { github, state, label } = fakeGitHub(); + state.duringUpload = () => label(); + assert.equal((await reconcile(github, runEvent(), { app: artifact() })).review.state, "pending"); + // The label's own reconcile follows: it came before the screens appeared, so it comes off. + assert.deepEqual([(await reconcile(github, prEvent("labeled"))).review.state, state.labels], ["pending", []]); + label(); + assert.equal((await reconcile(github, prEvent("labeled"))).review.state, "success"); }); test("lookalike comments are ignored", async () => { const lookalike = { id: 1, user: { login: "someone" }, body: `\n` }; const { github, state, label } = fakeGitHub({ comments: [lookalike] }); label(); diff --git a/.github/scripts/screens-comment.mjs b/.github/scripts/screens-comment.mjs index 7fcd3a7..2ebd28f 100644 --- a/.github/scripts/screens-comment.mjs +++ b/.github/scripts/screens-comment.mjs @@ -280,12 +280,14 @@ export function withReview(body, review, data) { const isTime = (value) => typeof value === "string" && value.length <= 40 && !Number.isNaN(Date.parse(value)); -/** The comment's data: what was drawn for which head and CI run, against which base, and since when it's shown. */ +/** The comment's data: what was drawn for which head and CI run, when that run started, and since when it's shown. */ function isData(data) { - return hasOnly(data, ["head", "base", "baseRef", "runId", "runAttempt", "ranScreens", "fingerprint", "shownAt", "total", "changed", "failed", "failure"]) + return hasOnly(data, ["head", "base", "baseRef", "runId", "runAttempt", "runCreatedAt", "ranScreens", "fingerprint", "shownAt", "total", "changed", + "failed", "failure"]) && isHex(data.head, 40) && (data.base === null || isHex(data.base, 40)) && (data.baseRef === null || (typeof data.baseRef === "string" && data.baseRef.length > 0 && data.baseRef.length <= 255)) && (data.runId === null || isCount(data.runId, 1)) && (data.runAttempt === null || isCount(data.runAttempt, 1)) + && (data.runCreatedAt === null || isTime(data.runCreatedAt)) && typeof data.ranScreens === "boolean" && (data.fingerprint === null || isHex(data.fingerprint, 64)) && (data.shownAt === null || isTime(data.shownAt)) && isCount(data.total) && isCount(data.changed) && isCount(data.failed) diff --git a/.github/scripts/screens-comment.test.mjs b/.github/scripts/screens-comment.test.mjs index 4090a55..a6ac803 100644 --- a/.github/scripts/screens-comment.test.mjs +++ b/.github/scripts/screens-comment.test.mjs @@ -9,7 +9,7 @@ const digest = (character) => character.repeat(64); const screen = (id, status, images = {}, extra = {}) => ({ id, status, changedPixels: status === "changed" ? 120 : 0, region: status === "changed" ? { x: 10, y: 20, width: 30, height: 40 } : null, images, ...(status === "changed" || status === "resized" ? { afterDigest: "e".repeat(64) } : {}), ...extra }); -const data = (extra = {}) => ({ head: sha("a"), base: sha("b"), baseRef: "main", runId: 41, runAttempt: 1, ranScreens: true, +const data = (extra = {}) => ({ head: sha("a"), base: sha("b"), baseRef: "main", runId: 41, runAttempt: 1, runCreatedAt: "2026-10-31T09:50:00Z", ranScreens: true, fingerprint: digest("f"), shownAt: "2026-10-31T10:00:00.000Z", total: 2, changed: 1, failed: 0, failure: null, ...extra }); const urls = (file) => `https://example.test/${file}`; @@ -242,6 +242,7 @@ test("readData reads only the data on the comment's last line", () => { assert.equal(readData(`x\n${line({ ...data(), approved: null })}`), null); assert.deepEqual(readData(`x\n${line(data({ ranScreens: false, fingerprint: null, shownAt: null, runId: null, runAttempt: null }))}`), data({ ranScreens: false, fingerprint: null, shownAt: null, runId: null, runAttempt: null })); + assert.equal(readData(`x\n${line({ ...data(), runCreatedAt: "soon" })}`), null); assert.equal(readData(`x\n${line({ ...data(), extra: 1 })}`), null); assert.equal(readData("no data here"), null); }); diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 84fe225..080bc77 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -165,7 +165,7 @@ These jobs run the pull request's code with a read-only token. The **Screens** w | Pending: "The base changed…" | The pull request's base changed since its screens were drawn. Push a commit to draw them again. | | Failure | Drawing failed; the comment says what. A `visual-ok` added while it showed the failure still approves it. | -`visual-ok` approves the screens the comment showed when the label was added, so reload the pull request first if a push just landed. Any maintainer can add it, the author too. A later run that shows the same screens keeps the approval. One that shows other screens, or other failures, takes the label off and the review goes back to pending: look again, and add it again. Removing the label also sends the review back to pending. +`visual-ok` approves the screens the comment showed when the label was added, so reload the pull request first if a push just landed. Any maintainer can add it, the author too. A later run that shows the same screens against the same base keeps the approval. One that shows other screens, or other failures, takes the label off and the review goes back to pending: look again, and add it again. Removing the label also sends the review back to pending. Pull requests from forks get the same comment and status: the drawing runs in the fork's pull request, and **Screens** finds the pull request by its branch and head. The comparison runs in the pull request's own CI, so a pull request that changes the CI workflow or the drawing scripts decides what its comparison shows. **Screens** reads the pull request's changed files from GitHub, so screens they call for that weren't drawn fail the review, but review such a change itself before you trust its screens. From 5d8436e530bdd87a4da3b83d1609011933e42a9e Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 17:31:26 +0100 Subject: [PATCH 21/25] ci: compare no screens for a branch that several pull requests share Two open pull requests from one branch, against two bases, get CI runs with the same head repository, branch and commit, and GitHub lists every pull request of the branch on each run, so one pull request's reconcile could publish the other's run, compared against the other base. The bot now fails the review of both, saying why, until one of them closes. A failed review's line in the comment now just points above. --- .github/scripts/screens-bot.mjs | 24 ++++++++++++++++++++++-- .github/scripts/screens-bot.test.mjs | 20 ++++++++++++++++++-- .github/scripts/screens-comment.mjs | 2 +- CONTRIBUTING.md | 2 +- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs index 8c7ce18..46f11eb 100644 --- a/.github/scripts/screens-bot.mjs +++ b/.github/scripts/screens-bot.mjs @@ -58,8 +58,8 @@ async function latestRun({ github, owner, repo, pr }) { /** * What a reconcile of the event's pull request does, decided before the workflow downloads anything: * "none" for a run that isn't CI's for an open pull request, "close" for a closed pull request, - * "publish" with the newest finished CI run for its head when the comment doesn't show that run yet, - * and "review" otherwise. A publication carries what the run's jobs did, which artifacts it uploaded + * "ambiguous" when another open pull request comes from the same branch, "publish" with the newest + * finished CI run for its head when the comment doesn't show that run yet, and "review" otherwise. A publication carries what the run's jobs did, which artifacts it uploaded * and which screens the pull request's files call for. A job the run doesn't have is "missing". */ export async function plan({ github, context }) { @@ -68,6 +68,10 @@ export async function plan({ github, context }) { if (!number) return { action: "none" }; const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: number }); if (pr.state !== "open") return { action: "close", number }; + // Pull requests from one branch share its CI runs' head, branch and commit, so their runs can't be told apart. + const siblings = pr.head.repo ? await github.paginate(github.rest.pulls.list, + { owner, repo, state: "open", head: `${pr.head.repo.owner.login}:${pr.head.ref}`, per_page: 100 }) : []; + if (siblings.some((other) => other.number !== number)) return { action: "ambiguous", number }; const run = await latestRun({ github, owner, repo, pr }); const existing = findScreensComment(await comments(github, owner, repo, number)); const data = existing ? readData(existing.body) : null; @@ -335,6 +339,21 @@ async function review({ github, owner, repo, pr, existing, data: stored, labels, return result; } +const sharedBranch = "Several open pull requests come from this branch, and the bot can't tell their CI runs apart."; + +/** Fails the review of a pull request whose branch another open pull request shares, and says why in its comment. */ +async function ambiguous({ github, owner, repo, pr, existing }) { + const data = { head: pr.head.sha, base: pr.base.sha, baseRef: pr.base.ref, runId: null, runAttempt: null, runCreatedAt: null, + ranScreens: true, fingerprint: null, shownAt: null, total: 0, changed: 0, failed: 1, failure: sharedBranch }; + const result = reviewState(data); + const body = noScreensBody({ reason: sharedBranch, review: result, data }); + const comment = existing + ? (await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body })).data + : (await github.rest.issues.createComment({ owner, repo, issue_number: pr.number, body })).data; + await setStatus({ github, owner, repo, sha: pr.head.sha, review: result, url: comment.html_url }); + return result; +} + /** Deletes an unmerged, closed pull request's images from every month, says so in its comment, and counts them. */ async function closeUnmerged({ github, owner, repo, number }) { const deleted = await deleteAssets({ github, owner, repo, pick: (assets) => pullRequestAssets(assets, number) }); @@ -366,6 +385,7 @@ export async function apply({ github, context, plan: chosen, folders = {} }) { github, owner, repo, pr, existing, data: existing ? readData(existing.body) : null, labels: pr.labels.map((label) => label.name), ...await history({ github, owner, repo, number }), }; + if (chosen.action === "ambiguous") return { action: "ambiguous", review: await ambiguous(state) }; if (chosen.action === "publish" && chosen.run.head === pr.head.sha) { return { action: "publish", review: await publish({ ...state, run: chosen.run, folders, target: context.payload.repository?.default_branch }) }; } diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs index 8403c4c..8a55cff 100644 --- a/.github/scripts/screens-bot.test.mjs +++ b/.github/scripts/screens-bot.test.mjs @@ -53,7 +53,7 @@ function fakeGitHub({ pull = {}, comments = [], labels = [], releases = [], file base: { sha: base, ref: "main" }, ...pull }, runs: [{ id: 99, run_number: 50, run_attempt: 1, head_sha: head, path: ".github/workflows/ci.yml", pull_requests: [], created_at: "2026-10-31T09:50:00Z", head_branch: "feat/x", head_repository: { full_name: "someone/StableBar" } }], - duringUpload: null, + duringUpload: null, others: [], jobs: { 99: jobsOf(conclusions) }, artifacts: { 99: [{ name: "screens-app" }] }, }; const at = () => state.now.toISOString(); @@ -63,7 +63,8 @@ function fakeGitHub({ pull = {}, comments = [], labels = [], releases = [], file if (pull_number !== state.pr.number) throw notFound(); return { data: { ...state.pr, labels: state.labels.map((name) => ({ name })) }, headers: { date: state.now.toUTCString() } }; }, - list: async ({ head: filter, state: wanted }) => ({ data: [state.pr].filter((pr) => pr.state === wanted && `${pr.head.repo.owner.login}:${pr.head.ref}` === filter) }), + list: async ({ head: filter, state: wanted }) => + ({ data: [state.pr, ...state.others].filter((pr) => pr.state === wanted && `${pr.head.repo.owner.login}:${pr.head.ref}` === filter) }), listFiles: async () => ({ data: state.files.map((filename) => ({ filename })) }), }, issues: { @@ -301,6 +302,21 @@ test("only the pull request's own head repository and branch publish screens", a assert.deepEqual(await plan({ github, context: runEvent() }), { action: "review", number: 7 }); }); +test("a branch with several open pull requests compares none, and says why", async () => { + const { github, state } = fakeGitHub(); + // The same branch, opened again against another base: its CI runs look like this pull request's. + state.others.push({ ...state.pr, number: 8, base: { sha: "d".repeat(40), ref: "develop" } }); + const { review } = await reconcile(github, runEvent(), { app: artifact() }); + const reason = "Several open pull requests come from this branch, and the bot can't tell their CI runs apart."; + assert.deepEqual(review, { state: "failure", description: `Drawing the screens failed: ${reason}`, removeLabel: false }); + assert.ok(comment(state).body.includes(reason)); + assert.equal(state.uploads.length, 0); + // Once the other one closes, the screens are compared again. + state.others[0].state = "closed"; + assert.equal((await reconcile(github, prEvent("labeled"), { app: artifact() })).action, "publish"); + assert.equal(state.uploads.length, 3); +}); + test("a label added while the screens upload doesn't approve them", async () => { const { github, state, label } = fakeGitHub(); state.duringUpload = () => label(); diff --git a/.github/scripts/screens-comment.mjs b/.github/scripts/screens-comment.mjs index 2ebd28f..c4d6386 100644 --- a/.github/scripts/screens-comment.mjs +++ b/.github/scripts/screens-comment.mjs @@ -223,7 +223,7 @@ function section(screen, imageURL) { /** The review line, which shows no untrusted text, and the data. */ function reviewSection(review, data) { - const lines = [`${reviewHeading} ${review.state === "failure" ? "Drawing the screens failed, see “Couldn't draw” above." : review.description}`]; + const lines = [`${reviewHeading} ${review.state === "failure" ? "Drawing the screens failed, see above." : review.description}`]; if (review.removeLabel) lines.push("", "`visual-ok` was taken off: it approved other screens than these."); lines.push("", ``); return lines.join("\n"); diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 080bc77..db2177e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -167,7 +167,7 @@ These jobs run the pull request's code with a read-only token. The **Screens** w `visual-ok` approves the screens the comment showed when the label was added, so reload the pull request first if a push just landed. Any maintainer can add it, the author too. A later run that shows the same screens against the same base keeps the approval. One that shows other screens, or other failures, takes the label off and the review goes back to pending: look again, and add it again. Removing the label also sends the review back to pending. -Pull requests from forks get the same comment and status: the drawing runs in the fork's pull request, and **Screens** finds the pull request by its branch and head. The comparison runs in the pull request's own CI, so a pull request that changes the CI workflow or the drawing scripts decides what its comparison shows. **Screens** reads the pull request's changed files from GitHub, so screens they call for that weren't drawn fail the review, but review such a change itself before you trust its screens. +Pull requests from forks get the same comment and status: the drawing runs in the fork's pull request, and **Screens** finds the pull request by its branch and head. Two open pull requests from one branch share its CI runs, which the bot can't tell apart, so it compares neither and fails their review until one closes. The comparison runs in the pull request's own CI, so a pull request that changes the CI workflow or the drawing scripts decides what its comparison shows. **Screens** reads the pull request's changed files from GitHub, so screens they call for that weren't drawn fail the review, but review such a change itself before you trust its screens. ## Agent skills From 7c7b0e4c004912aefe53249b0258da8c7fd1415e Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 17:35:20 +0100 Subject: [PATCH 22/25] ci: keep a closed sibling's CI runs out of a pull request's screens Once the other pull request from a shared branch closed, the bot compared screens again, and its newest run at the shared commit could be the closed one's, drawn against the other base. The bot now stops whenever another pull request from the branch, open or closed, is at the same commit, which a new commit ends. A branch reused for one pull request after another, such as a fork's main, isn't affected: its old pull requests are at older commits. The comment now says visual-ok was taken off because it was added before these screens appeared, which covers a label added too early as well as one for other screens. --- .github/scripts/screens-bot.mjs | 18 ++++++++++-------- .github/scripts/screens-bot.test.mjs | 19 ++++++++++++------- .github/scripts/screens-comment.mjs | 2 +- .github/scripts/screens-comment.test.mjs | 2 +- CONTRIBUTING.md | 2 +- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs index 46f11eb..df61acc 100644 --- a/.github/scripts/screens-bot.mjs +++ b/.github/scripts/screens-bot.mjs @@ -58,7 +58,7 @@ async function latestRun({ github, owner, repo, pr }) { /** * What a reconcile of the event's pull request does, decided before the workflow downloads anything: * "none" for a run that isn't CI's for an open pull request, "close" for a closed pull request, - * "ambiguous" when another open pull request comes from the same branch, "publish" with the newest + * "ambiguous" when another pull request from its branch is at its commit, "publish" with the newest * finished CI run for its head when the comment doesn't show that run yet, and "review" otherwise. A publication carries what the run's jobs did, which artifacts it uploaded * and which screens the pull request's files call for. A job the run doesn't have is "missing". */ @@ -68,10 +68,11 @@ export async function plan({ github, context }) { if (!number) return { action: "none" }; const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: number }); if (pr.state !== "open") return { action: "close", number }; - // Pull requests from one branch share its CI runs' head, branch and commit, so their runs can't be told apart. + // Another pull request from this branch at this commit, open or closed, has CI runs with the same head, branch and + // commit, and GitHub lists every pull request of a branch on each run, so their runs can't be told apart. const siblings = pr.head.repo ? await github.paginate(github.rest.pulls.list, - { owner, repo, state: "open", head: `${pr.head.repo.owner.login}:${pr.head.ref}`, per_page: 100 }) : []; - if (siblings.some((other) => other.number !== number)) return { action: "ambiguous", number }; + { owner, repo, state: "all", head: `${pr.head.repo.owner.login}:${pr.head.ref}`, per_page: 100 }) : []; + if (siblings.some((other) => other.number !== number && other.head.sha === pr.head.sha)) return { action: "ambiguous", number }; const run = await latestRun({ github, owner, repo, pr }); const existing = findScreensComment(await comments(github, owner, repo, number)); const data = existing ? readData(existing.body) : null; @@ -339,14 +340,15 @@ async function review({ github, owner, repo, pr, existing, data: stored, labels, return result; } -const sharedBranch = "Several open pull requests come from this branch, and the bot can't tell their CI runs apart."; +const sharedCommit = "Another pull request is at this branch's commit: close it if it's open, then push a commit."; -/** Fails the review of a pull request whose branch another open pull request shares, and says why in its comment. */ +/** Fails the review of a pull request that another pull request from its branch shares a commit with, and says why. */ async function ambiguous({ github, owner, repo, pr, existing }) { const data = { head: pr.head.sha, base: pr.base.sha, baseRef: pr.base.ref, runId: null, runAttempt: null, runCreatedAt: null, - ranScreens: true, fingerprint: null, shownAt: null, total: 0, changed: 0, failed: 1, failure: sharedBranch }; + ranScreens: true, fingerprint: null, shownAt: null, total: 0, changed: 0, failed: 1, failure: sharedCommit }; const result = reviewState(data); - const body = noScreensBody({ reason: sharedBranch, review: result, data }); + const body = noScreensBody({ review: result, data, reason: "Another pull request from this branch is at this commit, open or closed, so the bot " + + "can't tell their CI runs apart and compares no screens. Close it if it's open, then push a commit." }); const comment = existing ? (await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body })).data : (await github.rest.issues.createComment({ owner, repo, issue_number: pr.number, body })).data; diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs index 8a55cff..c5aa9ad 100644 --- a/.github/scripts/screens-bot.test.mjs +++ b/.github/scripts/screens-bot.test.mjs @@ -64,7 +64,7 @@ function fakeGitHub({ pull = {}, comments = [], labels = [], releases = [], file return { data: { ...state.pr, labels: state.labels.map((name) => ({ name })) }, headers: { date: state.now.toUTCString() } }; }, list: async ({ head: filter, state: wanted }) => - ({ data: [state.pr, ...state.others].filter((pr) => pr.state === wanted && `${pr.head.repo.owner.login}:${pr.head.ref}` === filter) }), + ({ data: [state.pr, ...state.others].filter((pr) => (wanted === "all" || pr.state === wanted) && `${pr.head.repo.owner.login}:${pr.head.ref}` === filter) }), listFiles: async () => ({ data: state.files.map((filename) => ({ filename })) }), }, issues: { @@ -302,18 +302,23 @@ test("only the pull request's own head repository and branch publish screens", a assert.deepEqual(await plan({ github, context: runEvent() }), { action: "review", number: 7 }); }); -test("a branch with several open pull requests compares none, and says why", async () => { - const { github, state } = fakeGitHub(); +test("another pull request at the same branch and commit, open or closed, stops the comparison until a new commit", async () => { + const { github, state, run } = fakeGitHub(); // The same branch, opened again against another base: its CI runs look like this pull request's. state.others.push({ ...state.pr, number: 8, base: { sha: "d".repeat(40), ref: "develop" } }); const { review } = await reconcile(github, runEvent(), { app: artifact() }); - const reason = "Several open pull requests come from this branch, and the bot can't tell their CI runs apart."; + const reason = "Another pull request is at this branch's commit: close it if it's open, then push a commit."; assert.deepEqual(review, { state: "failure", description: `Drawing the screens failed: ${reason}`, removeLabel: false }); - assert.ok(comment(state).body.includes(reason)); + assert.ok(comment(state).body.includes("can't tell their CI runs apart")); assert.equal(state.uploads.length, 0); - // Once the other one closes, the screens are compared again. + // Closed, it still has runs at this commit. state.others[0].state = "closed"; - assert.equal((await reconcile(github, prEvent("labeled"), { app: artifact() })).action, "publish"); + assert.equal((await reconcile(github, runEvent(), { app: artifact() })).action, "ambiguous"); + // A new commit is this pull request's alone. + const pushed = "e".repeat(40); + state.pr.head = { ...state.pr.head, sha: pushed }; + run({ sha: pushed }); + assert.equal((await reconcile(github, runEvent({ head_sha: pushed }), { app: artifact() })).action, "publish"); assert.equal(state.uploads.length, 3); }); diff --git a/.github/scripts/screens-comment.mjs b/.github/scripts/screens-comment.mjs index c4d6386..f9d9959 100644 --- a/.github/scripts/screens-comment.mjs +++ b/.github/scripts/screens-comment.mjs @@ -224,7 +224,7 @@ function section(screen, imageURL) { /** The review line, which shows no untrusted text, and the data. */ function reviewSection(review, data) { const lines = [`${reviewHeading} ${review.state === "failure" ? "Drawing the screens failed, see above." : review.description}`]; - if (review.removeLabel) lines.push("", "`visual-ok` was taken off: it approved other screens than these."); + if (review.removeLabel) lines.push("", "`visual-ok` was taken off: it was added before these screens appeared."); lines.push("", ``); return lines.join("\n"); } diff --git a/.github/scripts/screens-comment.test.mjs b/.github/scripts/screens-comment.test.mjs index a6ac803..01e7057 100644 --- a/.github/scripts/screens-comment.test.mjs +++ b/.github/scripts/screens-comment.test.mjs @@ -217,7 +217,7 @@ test("withReview replaces only the review line and the data", () => { const taken = { state: "pending", description: "1 screen changed: a maintainer adds visual-ok after looking.", removeLabel: true }; const approved = { state: "success", description: "1 changed screen approved.", removeLabel: false }; const body = make(taken); - assert.match(body, /`visual-ok` was taken off/); + assert.match(body, /`visual-ok` was taken off: it was added before these screens appeared\./); const updated = withReview(body, approved, data({ shownAt: "2026-10-31T11:00:00.000Z" })); assert.equal(updated, make(approved, { shownAt: "2026-10-31T11:00:00.000Z" })); assert.doesNotMatch(updated, /taken off/); diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db2177e..4cb1077 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -167,7 +167,7 @@ These jobs run the pull request's code with a read-only token. The **Screens** w `visual-ok` approves the screens the comment showed when the label was added, so reload the pull request first if a push just landed. Any maintainer can add it, the author too. A later run that shows the same screens against the same base keeps the approval. One that shows other screens, or other failures, takes the label off and the review goes back to pending: look again, and add it again. Removing the label also sends the review back to pending. -Pull requests from forks get the same comment and status: the drawing runs in the fork's pull request, and **Screens** finds the pull request by its branch and head. Two open pull requests from one branch share its CI runs, which the bot can't tell apart, so it compares neither and fails their review until one closes. The comparison runs in the pull request's own CI, so a pull request that changes the CI workflow or the drawing scripts decides what its comparison shows. **Screens** reads the pull request's changed files from GitHub, so screens they call for that weren't drawn fail the review, but review such a change itself before you trust its screens. +Pull requests from forks get the same comment and status: the drawing runs in the fork's pull request, and **Screens** finds the pull request by its branch and head. Two pull requests from one branch at one commit, open or closed, share CI runs the bot can't tell apart, so it compares no screens and fails the review: close the other one if it's open, then push a commit. The comparison runs in the pull request's own CI, so a pull request that changes the CI workflow or the drawing scripts decides what its comparison shows. **Screens** reads the pull request's changed files from GitHub, so screens they call for that weren't drawn fail the review, but review such a change itself before you trust its screens. ## Agent skills From 4b36ebbec2573248f85309b8bbcfa535d95d02c7 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 17:37:42 +0100 Subject: [PATCH 23/25] ci: show a reopened pull request's screens again at once Closing deletes a pull request's images, but its comment still named the run it showed, so on reopening the bot saw nothing new to publish until the reopened pull request's CI was done. Meanwhile the comment said its images were deleted while the review said approved. Closing now forgets the run, so the reopen publishes the last run's screens again, and the same screens keep their approval. --- .github/scripts/screens-bot.mjs | 10 +++++++--- .github/scripts/screens-bot.test.mjs | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs index df61acc..f14da05 100644 --- a/.github/scripts/screens-bot.mjs +++ b/.github/scripts/screens-bot.mjs @@ -356,12 +356,16 @@ async function ambiguous({ github, owner, repo, pr, existing }) { return result; } -/** Deletes an unmerged, closed pull request's images from every month, says so in its comment, and counts them. */ +/** + * Deletes an unmerged, closed pull request's images from every month, says so in its comment, and counts + * them. The comment forgets which run it showed, so a reopened pull request shows its screens again. + */ async function closeUnmerged({ github, owner, repo, number }) { const deleted = await deleteAssets({ github, owner, repo, pick: (assets) => pullRequestAssets(assets, number) }); const existing = findScreensComment(await comments(github, owner, repo, number)); - const data = existing ? readData(existing.body) : null; - if (data) { + const stored = existing ? readData(existing.body) : null; + if (stored) { + const data = { ...stored, runId: null, runAttempt: null }; const review = { state: "success", description: "Nothing to review: the pull request closed.", removeLabel: false }; const body = noScreensBody({ reason: "This pull request closed without merging, so the images of its screens were deleted.", review, data }); if (body !== existing.body) await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs index c5aa9ad..6c492ff 100644 --- a/.github/scripts/screens-bot.test.mjs +++ b/.github/scripts/screens-bot.test.mjs @@ -456,6 +456,11 @@ test("closing unmerged deletes the images from every month; merging keeps them; assert.match(comment(state).body, /closed without merging, so the images of its screens were deleted/); assert.deepEqual(await reconcile(github, runEvent(), { app: artifact() }), { action: "none" }); assert.equal(state.uploads.length, 3); + // Reopened, it shows its last run's screens again at once. + state.pr.state = "open"; + assert.equal((await reconcile(github, prEvent("reopened"), { app: artifact() })).action, "publish"); + assert.equal(state.uploads.length, 6); + assert.doesNotMatch(comment(state).body, /closed without merging/); const merged = fakeGitHub(); await reconcile(merged.github, runEvent(), { app: artifact() }); Object.assign(merged.state.pr, { state: "closed", merged: true }); From 6fb68bfeceddc51b72cc7c465fa94bf14a8fc467 Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 18:23:36 +0100 Subject: [PATCH 24/25] ci: mark the head errored when the screens plan fails, and pin Playwright From CodeRabbit's review: - Only Apply marked a head errored when the bot failed, so a Plan step that failed on one of its GitHub calls left the head without a Visual review status, which a required check would wait for. The Plan step now finds the pull request as plan() does, for a fork's run by its branch, marks its head errored, and fails. - Playwright installed the newest 1.x, so a rerun weeks later could capture a page with a newer Chromium and report text a pixel off as a change, taking visual-ok off. It's pinned to 1.64.1, to be raised on purpose. --- .github/scripts/screens-bot.mjs | 11 ++++++++++- .github/scripts/screens-bot.test.mjs | 16 +++++++++++++++- .github/workflows/ci.yml | 4 +++- .github/workflows/screens.yml | 12 ++++++++++-- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs index f14da05..27a71e5 100644 --- a/.github/scripts/screens-bot.mjs +++ b/.github/scripts/screens-bot.mjs @@ -43,6 +43,15 @@ async function pullRequestOfRun({ github, owner, repo, run }) { return (pulls.find((pr) => pr.head.sha === run.head_sha) ?? pulls[0])?.number ?? null; } +/** + * The pull request an event is about: in a label, edit or close event's payload, or for a CI run, the + * open one it was for. The workflow also asks for it when plan() fails, to mark that head errored. + */ +export async function pullRequestNumber({ github, context }) { + const { owner, repo } = context.repo; + return context.payload.pull_request?.number ?? await pullRequestOfRun({ github, owner, repo, run: context.payload.workflow_run }); +} + /** * The newest finished CI run for the pull request's head, from its own head repository and branch: a * pull request from another fork at the same commit has runs of its own, and they don't count. @@ -64,7 +73,7 @@ async function latestRun({ github, owner, repo, pr }) { */ export async function plan({ github, context }) { const { owner, repo } = context.repo; - const number = context.payload.pull_request?.number ?? await pullRequestOfRun({ github, owner, repo, run: context.payload.workflow_run }); + const number = await pullRequestNumber({ github, context }); if (!number) return { action: "none" }; const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: number }); if (pr.state !== "open") return { action: "close", number }; diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs index 6c492ff..c2e9e69 100644 --- a/.github/scripts/screens-bot.test.mjs +++ b/.github/scripts/screens-bot.test.mjs @@ -7,7 +7,7 @@ import zlib from "node:zlib"; import { decodePNG, encodePNG } from "./png.mjs"; import { main as compare } from "./compare-screens.mjs"; import { readData, releaseTag } from "./screens-comment.mjs"; -import { apply, changePatterns, failRun, plan } from "./screens-bot.mjs"; +import { apply, changePatterns, failRun, plan, pullRequestNumber } from "./screens-bot.mjs"; const head = "a".repeat(40), oldHead = "c".repeat(40), base = "b".repeat(40); const repo = { owner: "Seb-GRAF", repo: "StableBar" }; @@ -468,6 +468,20 @@ test("closing unmerged deletes the images from every month; merging keeps them; assert.equal(merged.state.deleted.length, 0); }); +test("a reconcile that can't plan still finds its pull request, so its head is marked errored", async () => { + const { github, state } = fakeGitHub(); + github.rest.actions.listWorkflowRuns = async () => { + throw Object.assign(new Error("Server Error"), { status: 502 }); + }; + // As the Plan step does: a fork's CI run lists no pull request, so the number comes from its branch. + await assert.rejects(plan({ github, context: runEvent() }), /Server Error/); + const number = await pullRequestNumber({ github, context: runEvent() }); + assert.equal(number, 7); + await failRun({ github, context: runEvent(), number }); + assert.deepEqual(state.statuses.map((status) => [status.sha, status.state]), [[head, "error"]]); + assert.equal(await pullRequestNumber({ github, context: prEvent("labeled") }), 7); +}); + test("failRun marks an open pull request's head when the bot itself fails", async () => { const { github, state } = fakeGitHub(); await failRun({ github, context: runEvent(), number: 7 }); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 155d73b..1efe461 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -368,7 +368,9 @@ jobs: mkdir playwright cd playwright npm init --yes > /dev/null - npm install --no-audit --no-fund playwright@1 + # One exact version: a newer Chromium can draw text a pixel differently, which a rerun would + # report as a change. Raise it on purpose. + npm install --no-audit --no-fund playwright@1.64.1 npx playwright install --with-deps chromium - name: Capture the website run: | diff --git a/.github/workflows/screens.yml b/.github/workflows/screens.yml index d65defa..0aece2c 100644 --- a/.github/workflows/screens.yml +++ b/.github/workflows/screens.yml @@ -48,9 +48,17 @@ jobs: id: plan uses: actions/github-script@v8 with: + # A plan that fails marks the head errored, as Apply does, so a required Visual review doesn't wait forever. script: | - const { plan } = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); - const chosen = await plan({ github, context }); + const bot = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/screens-bot.mjs`); + let chosen; + try { + chosen = await bot.plan({ github, context }); + } catch (error) { + const number = await bot.pullRequestNumber({ github, context }).catch(() => null); + await bot.failRun({ github, context, number }).catch((failure) => core.warning(failure.message)); + throw error; + } core.info(`Plan: ${chosen.action}${chosen.number ? ` for #${chosen.number}` : ""}${chosen.run ? ` from run ${chosen.run.id}` : ""}`); core.setOutput("plan", JSON.stringify(chosen)); core.setOutput("run-id", chosen.run?.id ?? ""); From 6b52545ff9d3d182ce45ca1f2cc721e3a05721bf Mon Sep 17 00:00:00 2001 From: Steve Mendes Reis Date: Sat, 31 Oct 2026 18:35:15 +0100 Subject: [PATCH 25/25] ci: mark the event's head errored when GitHub can't say where it stands From CodeRabbit's review: when a reconcile failed reading the pull request, failRun read it again, and if that failed too, no error status was set. After a base change, the head then kept the old comparison's status. A label, edit or close event carries its pull request, so failRun now falls back on it. A CI run carries none, and still fails loudly. --- .github/scripts/screens-bot.mjs | 14 ++++++++++++-- .github/scripts/screens-bot.test.mjs | 13 +++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/scripts/screens-bot.mjs b/.github/scripts/screens-bot.mjs index 27a71e5..80d652a 100644 --- a/.github/scripts/screens-bot.mjs +++ b/.github/scripts/screens-bot.mjs @@ -407,11 +407,21 @@ export async function apply({ github, context, plan: chosen, folders = {} }) { return { action: "review", review: await review(state) }; } -/** Marks an open pull request's head as errored when the bot itself fails, so a required Visual review doesn't wait forever. */ +/** + * Marks an open pull request's head as errored when the bot itself fails, so a required Visual review doesn't + * wait forever. When GitHub can't say where the pull request stands, a label, edit or close event's own pull + * request does: a base change whose reconcile failed must not keep the old comparison's status. + */ export async function failRun({ github, context, number }) { if (!number) return; const { owner, repo } = context.repo; - const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: number }); + let pr; + try { + pr = (await github.rest.pulls.get({ owner, repo, pull_number: number })).data; + } catch (error) { + if (context.payload.pull_request?.number !== number) throw error; + pr = context.payload.pull_request; + } if (pr.state !== "open") return; await github.rest.repos.createCommitStatus({ owner, repo, sha: pr.head.sha, state: "error", context: statusContext, description: "The screens bot failed: see the Screens workflow run.", target_url: `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}` }); diff --git a/.github/scripts/screens-bot.test.mjs b/.github/scripts/screens-bot.test.mjs index c2e9e69..c195c36 100644 --- a/.github/scripts/screens-bot.test.mjs +++ b/.github/scripts/screens-bot.test.mjs @@ -482,6 +482,19 @@ test("a reconcile that can't plan still finds its pull request, so its head is m assert.equal(await pullRequestNumber({ github, context: prEvent("labeled") }), 7); }); +test("failRun falls back on the event's own pull request when GitHub can't answer", async () => { + const { github, state } = fakeGitHub(); + github.rest.pulls.get = async () => { + throw Object.assign(new Error("Server Error"), { status: 502 }); + }; + // A base change whose plan failed on reading the pull request: the event carries its head. + const edited = prEvent("edited", { pull_request: { number: 7, state: "open", head: { sha: head } } }); + await failRun({ github, context: edited, number: 7 }); + assert.deepEqual(state.statuses.map((status) => [status.sha, status.state]), [[head, "error"]]); + // A CI run carries no pull request to fall back on. + await assert.rejects(failRun({ github, context: runEvent(), number: 7 }), /Server Error/); +}); + test("failRun marks an open pull request's head when the bot itself fails", async () => { const { github, state } = fakeGitHub(); await failRun({ github, context: runEvent(), number: 7 });