From 5493614b7a29d4ecf99467a7f2e39702df04cc3f Mon Sep 17 00:00:00 2001 From: Jukka Kurkela Date: Fri, 18 Sep 2026 11:56:19 +0300 Subject: [PATCH] chore: add a tool for measuring fixture tolerances Fixture tolerances have been maintained by nudging a number upward whenever a browser update made a fixture fail. That calibrates from the one value in the failure report, which is a guess at the margin rather than a measurement of it, and it leaves the suite loose everywhere while still failing somewhere. `npm run measure-tolerances` measures instead. Every fixture is forced to `tolerance: 0` so it reports its real pixel delta, in four environments: the host's Chrome and Firefox, plus Linux Chrome and Firefox in a container built to match the CI image. Tolerances are then derived: - delta 0 in every environment -> 0 - otherwise -> max(1.5 * linux, 1.05 * host) Linux gets a real margin because CI is the gate; the host gets enough to keep local runs green without sizing every tolerance for a platform that renders text 3-4x further from the reference. The container runs linux/amd64 under emulation deliberately: Chrome is not published for linux/arm64, and native arm64 rendering differs (point/starShadow measures 1474px there against 1476px in CI). It was validated against three pixel counts from a real CI run, all reproduced exactly. Runs that end early are rejected rather than measured, since partial data would silently yield tolerances that are too tight. The fixtures it rewrites are restored afterwards, and it refuses to start if test/fixtures is dirty. Verified end to end: run against master's fixtures it reproduces all 226 values from the recalibration in #992 exactly. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 3 + package.json | 3 +- scripts/measure-tolerances/Dockerfile | 21 +++ scripts/measure-tolerances/README.md | 75 +++++++++ scripts/measure-tolerances/in-container.sh | 19 +++ scripts/measure-tolerances/measure.js | 174 +++++++++++++++++++++ scripts/measure-tolerances/prepare.js | 51 ++++++ 7 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 scripts/measure-tolerances/Dockerfile create mode 100644 scripts/measure-tolerances/README.md create mode 100755 scripts/measure-tolerances/in-container.sh create mode 100644 scripts/measure-tolerances/measure.js create mode 100644 scripts/measure-tolerances/prepare.js diff --git a/.gitignore b/.gitignore index e6fd2bc07..9b1fedeb3 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ npm-debug.log* *.log *.swp *.stackdump + +# output of scripts/measure-tolerances +/tolerances.report.json diff --git a/package.json b/package.json index 5cd621be7..b3ce3bcbe 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "test-karma": "karma start ./karma.conf.cjs --auto-watch --single-run", "test-lint": "npm run lint", "test-types": "tsc -p types/tests/", - "test-integration": "mocha --full-trace test/integration/*-test.js" + "test-integration": "mocha --full-trace test/integration/*-test.js", + "measure-tolerances": "node scripts/measure-tolerances/measure.js" }, "devDependencies": { "@rollup/plugin-json": "^6.0.0", diff --git a/scripts/measure-tolerances/Dockerfile b/scripts/measure-tolerances/Dockerfile new file mode 100644 index 000000000..5c6151c62 --- /dev/null +++ b/scripts/measure-tolerances/Dockerfile @@ -0,0 +1,21 @@ +# Reproduces the browsers CI runs (ubuntu-latest, x86_64) so fixture deltas can +# be measured against the environment that actually gates the build. +# +# linux/amd64 is deliberate: Chrome is not published for linux/arm64, and native +# arm64 rendering differs from x86_64 (point/starShadow measures 1474px there vs +# 1476px on x86_64 and in CI), so arm64 numbers would not be CI-faithful. +FROM --platform=linux/amd64 ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl ca-certificates gnupg xvfb xz-utils \ + fonts-liberation fonts-dejavu-core fontconfig \ + && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && curl -fsSL -o /tmp/chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \ + && apt-get install -y --no-install-recommends /tmp/chrome.deb \ + && curl -fsSL -o /tmp/ff.tar.xz "https://download.mozilla.org/?product=firefox-latest-ssl&os=linux64&lang=en-US" \ + && mkdir -p /opt && tar -xJf /tmp/ff.tar.xz -C /opt \ + && rm -rf /tmp/*.deb /tmp/*.tar.xz /var/lib/apt/lists/* +ENV CHROME_BIN=/usr/bin/google-chrome +ENV FIREFOX_BIN=/opt/firefox/firefox +WORKDIR /app diff --git a/scripts/measure-tolerances/README.md b/scripts/measure-tolerances/README.md new file mode 100644 index 000000000..63ed7a0be --- /dev/null +++ b/scripts/measure-tolerances/README.md @@ -0,0 +1,75 @@ +# Measuring fixture tolerances + +Fixture tests compare a rendered chart against a reference PNG and allow a +`tolerance` — the fraction of pixels that may differ. Browsers do not rasterise +text and shadows identically, so some difference is expected, and the tolerance +has to absorb it without hiding real regressions. + +Picking those numbers by hand does not work for long. Every browser update +shifts the deltas, a fixture starts failing, and the tolerance gets nudged up +based on the one number in the failure report — which is a guess about the +margin, not a measurement of it. Do that a few times and the suite is loose +everywhere and still failing somewhere. + +This tool measures instead. + +## Usage + +```sh +npm run measure-tolerances # report what would change +npm run measure-tolerances -- --apply # write the derived values +``` + +Requires Docker. `test/fixtures` must be clean — the fixtures are temporarily +rewritten and then restored with `git checkout`. + +Host browsers default to `chrome,firefox`; override with +`-- --host-browsers=firefox` if you only have one installed. + +## How it works + +Every fixture is forced to `tolerance: 0`, which makes each one report its real +pixel delta instead of just passing or failing. That runs in four environments: +your own Chrome and Firefox, and Linux Chrome and Firefox in a container built +to match the CI image. + +Tolerances are then derived: + +| condition | tolerance | +|---|---| +| delta is 0 in every environment | `0` | +| otherwise | `max(1.5 × linux, 1.05 × host)` | + +Linux gets a real margin because CI is what gates the build. The host gets only +enough to keep local runs green — macOS in particular renders text 3-4x further +from the reference than Linux does, and sizing every tolerance for it is what +makes them loose. + +The trade-off is deliberate: a host browser update will redden local runs +without affecting CI, and re-measuring is cheap. + +## Why the container is emulated + +It runs `linux/amd64` under emulation even on Apple Silicon. Chrome is not +published for `linux/arm64` at all, and native arm64 rendering genuinely +differs — `point/starShadow` measures 1474px there against 1476px on x86_64 and +in CI, and `line/labelShadowColors` differs by 74%. Faster, but wrong. + +Emulation was verified against three pixel counts observed in CI, all +reproduced exactly: + +| fixture | browser | CI | container | +|---|---|---|---| +| `point/crossShadow` | Firefox | 1565px | 1565px | +| `point/starShadow` | Firefox | 1476px | 1476px | +| `doughnutLabel/contentMultiline` | Chrome | 919px | 919px | + +If you change the image, re-check it against numbers from a real CI run before +trusting what it produces. + +Emulated Chrome needs two accommodations, both in `prepare.js`: karma's +timeouts are raised well past their defaults, and `--no-sandbox` is added +because the container does not grant the user namespaces Chrome's sandbox +wants. The flag was confirmed not to affect rendering — `contentMultiline` +still measures the same 919px with it. Without `--shm-size=2g` Chrome hangs +outright rather than running slowly. diff --git a/scripts/measure-tolerances/in-container.sh b/scripts/measure-tolerances/in-container.sh new file mode 100755 index 000000000..0867e234a --- /dev/null +++ b/scripts/measure-tolerances/in-container.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Runs inside the image built from the Dockerfile next to this file. +# Expects the repo mounted read-only at /src and a browser name as $1. +set -e +browser="$1" + +useradd -m runner 2>/dev/null || true +mkdir -p /app && cp -r /src/. /app/ && rm -rf /app/node_modules /app/dist +chown -R runner /app + +su runner -c "HOME=/home/runner bash -c ' +set -e +cd /app +npm ci --no-audit --no-fund >/dev/null 2>&1 +npm run build >/dev/null 2>&1 +node scripts/measure-tolerances/prepare.js +xvfb-run --auto-servernum npx karma start ./karma.conf.cjs \ + --single-run --no-auto-watch --browsers $browser 2>&1 +'" diff --git a/scripts/measure-tolerances/measure.js b/scripts/measure-tolerances/measure.js new file mode 100644 index 000000000..224589104 --- /dev/null +++ b/scripts/measure-tolerances/measure.js @@ -0,0 +1,174 @@ +// Measures every fixture's real pixel delta in each environment that matters, +// then derives tolerances from those measurements instead of guessing margins. +// +// npm run measure-tolerances report only +// npm run measure-tolerances -- --apply also write the derived tolerances +// +// Environments: the host's Chrome and Firefox, plus Linux Chrome and Firefox in +// a container reproducing the CI image. See README.md next to this file. +import fs from 'fs'; +import {execFileSync} from 'child_process'; +import path from 'path'; +import {fileURLToPath} from 'url'; +import {fixtureFiles, readTolerance, writeTolerance, forceZeroTolerance} from './prepare.js'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const IMAGE = 'chartjs-annotation-ci'; +const REPORT = 'tolerances.report.json'; + +// CI is the gate, so Linux gets a real margin. macOS renders text 3-4x further +// from the reference than Linux does; giving it the same margin is what made +// tolerances loose, so it gets only enough to keep local runs green. +const LINUX_MARGIN = 1.5; +const HOST_MARGIN = 1.05; +// A fixture that is pixel-perfect everywhere asserts exactly that. +const ZERO_STAYS_ZERO = true; + +const ceil = v => Math.ceil(v * 20000) / 20000; +const run = (cmd, args, opts = {}) => + execFileSync(cmd, args, {encoding: 'utf-8', maxBuffer: 64 * 1024 * 1024, ...opts}); + +function pixelCount(fixture) { + const buf = fs.readFileSync(`${fixture.replace(/\.js$/, '')}.png`); + return buf.readUInt32BE(16) * buf.readUInt32BE(20); // IHDR width * height +} + +// karma prints ` ... /base/test/fixtures/.js FAILED` and then +// `Difference: px`, so deltas are read back from its output. +function parseDeltas(output) { + const deltas = {}; + let fixture = null; + for (const line of output.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '').split('\n')) { + const failed = line.match(/\/base\/(test\/fixtures\/.+?\.js) FAILED/); + if (failed) { + fixture = failed[1]; + continue; + } + const diff = line.match(/Difference: (\d+)px/); + if (diff && fixture) { + deltas[fixture] = Math.max(deltas[fixture] || 0, Number(diff[1])); + fixture = null; + } + } + return deltas; +} + +function assertFixturesClean() { + if (run('git', ['status', '--porcelain', 'test/fixtures']).trim()) { + throw new Error('test/fixtures has uncommitted changes; commit or stash them first ' + + '(they would be lost when the measured fixtures are restored)'); + } +} + +// A run that dies partway (karma DISCONNECTED, a crashed browser) still yields +// deltas for the specs it reached. Deriving tolerances from those would silently +// produce values that are too tight, so incomplete runs are rejected outright. +function assertComplete(output, label) { + const clean = output.replace(/\x1b\[[0-9;]*[A-Za-z]/g, ''); + if (/DISCONNECTED/.test(clean)) { + throw new Error(`${label}: browser disconnected, measurement incomplete`); + } + const executed = [...clean.matchAll(/Executed (\d+) of (\d+)/g)].pop(); + if (!executed) { + throw new Error(`${label}: karma reported no specs`); + } + if (executed[1] !== executed[2]) { + throw new Error(`${label}: only ${executed[1]} of ${executed[2]} specs ran`); + } +} + +function measureHost(browsers) { + console.log(`measuring host (${browsers})...`); + forceZeroTolerance(); + try { + const out = run('npx', ['karma', 'start', './karma.conf.cjs', + '--single-run', '--no-auto-watch', '--browsers', browsers], {stdio: 'pipe'}); + assertComplete(out, 'host'); + return parseDeltas(out); + } catch (err) { + // karma exits non-zero because every spec fails at tolerance 0; that is the + // point of the run, so the deltas still have to be read out of its output. + if (err.stdout === undefined) { + throw err; + } + const out = String(err.stdout); + assertComplete(out, 'host'); + return parseDeltas(out); + } finally { + run('git', ['checkout', '--', 'test/fixtures']); + } +} + +function measureLinux(browser) { + console.log(`measuring linux ${browser} in docker (emulated; chrome takes a while)...`); + const args = ['run', '--rm', '--platform', 'linux/amd64', + // without this chrome hangs outright rather than running slowly + '--shm-size=2g', + '-v', `${process.cwd()}:/src:ro`, + IMAGE, 'bash', '/src/scripts/measure-tolerances/in-container.sh', browser]; + try { + return parseDeltas(run('docker', args)); + } catch (err) { + // karma exits non-zero because every spec fails at tolerance 0; that is the + // point of the run, so the deltas still have to be read out of its output. + if (err.stdout === undefined) { + throw err; + } + const out = String(err.stdout); + assertComplete(out, `linux ${browser}`); + return parseDeltas(out); + } +} + +function derive(measurements) { + return fixtureFiles().map(file => { + const area = pixelCount(file); + const ratio = env => (measurements[env][file] || 0) / area; + const linux = Math.max(ratio('linuxChrome'), ratio('linuxFirefox')); + const host = ratio('host'); + const current = readTolerance(file); + const proposed = (ZERO_STAYS_ZERO && linux === 0 && host === 0) + ? 0 + : Math.max(ceil(linux * LINUX_MARGIN), ceil(host * HOST_MARGIN)); + return {file, linux, host, current, proposed}; + }); +} + +const args = process.argv.slice(2); +const browsers = (args.find(a => a.startsWith('--host-browsers=')) || '').split('=')[1] || 'chrome,firefox'; + +// Fail fast on a dirty tree before spending an hour in docker. +assertFixturesClean(); +run('docker', ['build', '--platform', 'linux/amd64', '-t', IMAGE, HERE], {stdio: 'inherit'}); +const measurements = { + host: measureHost(browsers), + linuxChrome: measureLinux('chrome'), + linuxFirefox: measureLinux('firefox') +}; + +const rows = derive(measurements); +fs.writeFileSync(REPORT, JSON.stringify(rows, null, 2)); + +const changes = rows.filter(r => r.proposed !== r.current); +const pct = v => (v * 100).toFixed(3) + '%'; +console.log(`\n${changes.length} of ${rows.length} fixtures would change:\n`); +for (const r of changes.sort((a, b) => b.proposed - a.proposed)) { + const dir = r.proposed < r.current ? 'tighter' : 'looser '; + console.log(` ${dir} ${r.file.replace('test/fixtures/', '').padEnd(38)} ` + + `${pct(r.current).padStart(8)} -> ${pct(r.proposed).padStart(8)} ` + + `(linux ${pct(r.linux)}, host ${pct(r.host)})`); +} +const budget = rows.reduce((sum, r) => ({ + before: sum.before + r.current * pixelCount(r.file), + after: sum.after + r.proposed * pixelCount(r.file) +}), {before: 0, after: 0}); +console.log(`\ntotal allowed differing pixels: ${Math.round(budget.before).toLocaleString('en')} -> ` + + `${Math.round(budget.after).toLocaleString('en')}`); +console.log(`report written to ${REPORT}`); + +if (args.includes('--apply')) { + changes.forEach(r => writeTolerance(r.file, r.proposed)); + console.log(`\napplied to ${changes.length} fixtures; run the suite to confirm`); +} else if (changes.length) { + console.log('\nre-run with --apply to write these values'); +} diff --git a/scripts/measure-tolerances/prepare.js b/scripts/measure-tolerances/prepare.js new file mode 100644 index 000000000..d3f809249 --- /dev/null +++ b/scripts/measure-tolerances/prepare.js @@ -0,0 +1,51 @@ +// Forces every fixture to `tolerance: 0` so each one reports its real pixel +// delta instead of just pass/fail, and relaxes karma's timeouts because an +// emulated Chrome is far slower than the 2 minute default allows. +// +// Destructive: only run directly inside the container, which works on a throwaway +// copy. measure.js imports the helpers and handles restoring the host checkout. +import fs from 'fs'; +import path from 'path'; +import {pathToFileURL} from 'url'; + +export function fixtureFiles(dir = 'test/fixtures') { + return fs.readdirSync(dir, {withFileTypes: true}).flatMap(e => { + const p = path.join(dir, e.name); + return e.isDirectory() ? fixtureFiles(p) : (e.name.endsWith('.js') ? [p] : []); + }); +} + +export function readTolerance(file) { + const m = fs.readFileSync(file, 'utf-8').match(/^ {2}tolerance: ([\d.]+),$/m); + return m ? parseFloat(m[1]) : 0.001; // chartjs-test-utils default +} + +export function writeTolerance(file, value) { + const src = fs.readFileSync(file, 'utf-8'); + const line = ` tolerance: ${String(value)},`; + fs.writeFileSync(file, /^ {2}tolerance: [\d.]+,$/m.test(src) + ? src.replace(/^ {2}tolerance: [\d.]+,$/m, line) + : src.replace(/^module\.exports = \{$/m, `module.exports = {\n${line}`)); +} + +export function forceZeroTolerance() { + fixtureFiles().forEach(file => writeTolerance(file, 0)); +} + +export function relaxKarmaTimeouts() { + const conf = 'karma.conf.cjs'; + fs.writeFileSync(conf, fs.readFileSync(conf, 'utf-8') + .replace(/browserNoActivityTimeout: \d+/, 'browserNoActivityTimeout: 3600000') + .replace(/browserDisconnectTimeout: \d+/, 'browserDisconnectTimeout: 3600000') + .replace(/captureTimeout: \d+/, 'captureTimeout: 3600000') + // Chrome's sandbox needs user namespaces the container does not grant. + // Verified not to affect rendering: with it, doughnutLabel/contentMultiline + // still measures the 919px observed in CI. + .replace("'--disable-accelerated-2d-canvas'", "'--disable-accelerated-2d-canvas', '--no-sandbox'")); +} + +// Self-executes only when run as a script, never when imported. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + forceZeroTolerance(); + relaxKarmaTimeouts(); +}