Show where a capture differs from its golden

The diff a failing run wrote was a raw absolute difference: near-black
wherever the change was subtle, which is most of the ones worth looking
at. Mark the counted pixels magenta over a dimmed copy of the capture
instead, and dim the ignored status strip further so telemetry is never
mistaken for a change.

Add `just goldens-diff <case>`, which reports every shot rather than only
the failures. A difference inside its tolerance is invisible during a
run, and that is exactly what you want to see before deciding whether a
golden has gone stale. It reads each shot's tolerance and ignored strip
from the run's events, which now record both, and says so plainly when
asked about an older run that does not.

Author: gajop

justfile

@@ -165,6 +165,12 @@ e2e-report *args:
 goldens-status:
     @PYTHONPATH="{{tool_pythonpath}}" uv run --locked sbc-e2e goldens-status
 
+# Where a case's last captures differ from its goldens, marked in magenta.
+# Reports the within-tolerance differences a passing run never shows.
+[group('test')]
+goldens-diff case *args:
+    @PYTHONPATH="{{tool_pythonpath}}" uv run --locked sbc-e2e goldens-diff "{{case}}" {{args}}
+
 # Approve a case's reference images (the human OK): `just goldens-approve rotation`.
 # Optionally name individual shots. Only a human runs this.
 [group('test')]

tools/e2e/cli_e2e.py

@@ -1,3 +1,4 @@
+import json
 from collections.abc import Iterator
 from dataclasses import dataclass
 from pathlib import Path
@@ -7,7 +8,15 @@
 
 from .driver.cases import TARGETS, Case, select_cases, target_cases
 from .driver.utils.paths import ARTIFACT_ROOT, create_suite_artifact_dir
-from .fixtures.golden import GOLDEN_ROOT, STATUS_APPROVED, approve_review, load_review
+from .fixtures.golden import (
+    GOLDEN_ROOT,
+    STATUS_APPROVED,
+    approve_review,
+    differing_pixels,
+    golden_path,
+    load_review,
+    write_diff,
+)
 from .runner import E2ERun
 from .suite_report import artifact_paths, default_report_path, write_report
 
@@ -121,6 +130,63 @@ def goldens_status() -> None:
         typer.echo(f"\n{pending} image(s) awaiting approval.")
 
 
+@app.command("goldens-diff")
+def goldens_diff(
+    case: Annotated[str, typer.Argument(help="Golden case directory.")],
+    run: Annotated[Path | None, typer.Option(help="Run directory; default is the latest for the case.")] = None,
+    out: Annotated[Path | None, typer.Option(help="Where to write the overlays.")] = None,
+) -> None:
+    """Show where a case's captures differ from its goldens, as magenta overlays.
+
+    Reports every shot, including the ones whose difference stays inside their
+    tolerance -- those are invisible in a run, and they are exactly what you want
+    to see before deciding whether a golden is stale.
+    """
+    run_dir = run or _latest_run_dir(case)
+    if run_dir is None:
+        raise typer.BadParameter(f"no run artifacts for {case}; run `just test-e2e {case}` first")
+    checks = _golden_checks(run_dir)
+    if not checks:
+        raise typer.BadParameter(f"{run_dir} recorded no goldens")
+    destination = out or run_dir / "golden-diffs"
+    destination.mkdir(parents=True, exist_ok=True)
+    if any("ignored_bottom" not in check for check in checks.values()):
+        typer.echo(
+            f"warning: {run_dir.name} predates recorded golden metadata, so the "
+            "ignored status strip and each shot's tolerance are unknown here; "
+            "counts include regions the run itself does not read. Re-run the case."
+        )
+    for name, check in sorted(checks.items()):
+        actual = Path(check["path"])
+        golden = golden_path(case, name)
+        if not actual.is_file() or not golden.is_file():
+            typer.echo(f"{name:32} missing capture or golden")
+            continue
+        ignored_bottom = int(check.get("ignored_bottom", 0))
+        channel_tolerance = int(check.get("channel_tolerance", 1))
+        tolerance = int(check.get("tolerance", 0))
+        differing = differing_pixels(
+            golden,
+            actual,
+            channel_tolerance=channel_tolerance,
+            ignored_bottom=ignored_bottom,
+        )
+        if not differing:
+            typer.echo(f"{name:32} identical")
+            continue
+        target = destination / f"{name}.diff.png"
+        write_diff(
+            golden,
+            actual,
+            target,
+            channel_tolerance=channel_tolerance,
+            ignored_bottom=ignored_bottom,
+        )
+        verdict = "over tolerance" if differing > tolerance else f"within tolerance {tolerance}"
+        typer.echo(f"{name:32} {differing:>8} px  {verdict:<22} {target}")
+    typer.echo(f"\ndiffs: {destination}")
+
+
 @app.command("approve-goldens")
 def approve_goldens(
     case: Annotated[str, typer.Argument(help="Golden case directory.")],
@@ -256,3 +322,28 @@ def _case_batches(cases: list[Case], reuse_sessions: bool) -> Iterator[list[Case
 
 def _batch_key(case: Case) -> tuple[tuple[tuple[str, str], ...], tuple[tuple[str, str], ...]]:
     return tuple(sorted(case.flags.items())), tuple(sorted(case.env.items()))
+
+
+def _latest_run_dir(case: str) -> Path | None:
+    """The newest artifact directory holding this case, standalone or in a suite."""
+    if not ARTIFACT_ROOT.is_dir():
+        return None
+    candidates = [path for path in ARTIFACT_ROOT.glob(f"*-{case}") if path.is_dir()]
+    candidates += [path / case for path in ARTIFACT_ROOT.glob("*-suite") if (path / case).is_dir()]
+    return max(candidates, key=lambda path: path.stat().st_mtime, default=None)
+
+
+def _golden_checks(run_dir: Path) -> dict[str, dict[str, object]]:
+    """The `golden` events of a run, newest entry per shot."""
+    events = run_dir / "events.jsonl"
+    if not events.is_file():
+        return {}
+    checks: dict[str, dict[str, object]] = {}
+    for line in events.read_text().splitlines():
+        try:
+            event = cast("dict[str, object]", json.loads(line))
+        except json.JSONDecodeError:
+            continue
+        if event.get("kind") == "golden" and isinstance(event.get("name"), str):
+            checks[cast("str", event["name"])] = event
+    return checks

tools/e2e/driver/_capture.py

@@ -195,7 +195,9 @@ def _finish_screenshots(self) -> list[str]:
                 status=status,
                 path=str(check.path),
                 capture_ms=check.capture_ms,
+                tolerance=check.tolerance,
                 channel_tolerance=check.channel_tolerance,
+                ignored_bottom=check.ignored_bottom,
                 conversion_ms=conversion.elapsed_ms,
                 compare_ms=int((monotonic() - started) * 1000),
             )

tools/e2e/fixtures/golden.py

@@ -28,6 +28,10 @@
 STATUS_AI = "ai-reviewed"
 STATUS_APPROVED = "approved"
 
+# Magenta: the UI theme has no such colour, so a marked pixel cannot be mistaken
+# for one of its own.
+DIFF_COLOR = (255, 0, 255)
+
 
 class ReviewEntry(TypedDict):
     status: str
@@ -115,9 +119,38 @@ def differing_pixels(
         return _count_pixels_over(difference, channel_tolerance)
 
 
-def write_diff(golden: Path, actual: Path, out: Path) -> None:
+def write_diff(
+    golden: Path,
+    actual: Path,
+    out: Path,
+    *,
+    channel_tolerance: int = 1,
+    ignored_bottom: int = 0,
+) -> None:
+    """Paint the changed pixels magenta over a dimmed copy of the capture.
+
+    A raw absolute difference is nearly black wherever the change is subtle,
+    which is most of the interesting ones. Locating the change matters more than
+    measuring it here, so the base image stays recognisable and every pixel the
+    comparison counts is made the one colour nothing in the UI uses.
+
+    Rows inside `ignored_bottom` are dimmed further and never marked: they carry
+    live telemetry that no check reads.
+    """
     with Image.open(golden) as expected, Image.open(actual) as observed:
-        ImageChops.difference(expected.convert("RGBA"), observed.convert("RGBA")).save(out)
+        base = observed.convert("RGB")
+        difference = ImageChops.difference(expected.convert("RGBA"), base.convert("RGBA"))
+        changed = _over_tolerance_mask(difference, channel_tolerance)
+        if ignored_bottom:
+            keep = max(0, base.height - ignored_bottom)
+            changed.paste(0, (0, keep, base.width, base.height))
+        canvas = Image.blend(base, Image.new("RGB", base.size, (0, 0, 0)), 0.65)
+        if ignored_bottom:
+            keep = max(0, base.height - ignored_bottom)
+            strip = canvas.crop((0, keep, base.width, base.height))
+            canvas.paste(Image.blend(strip, Image.new("RGB", strip.size, (0, 0, 0)), 0.6), (0, keep))
+        canvas.paste(Image.new("RGB", base.size, DIFF_COLOR), mask=changed)
+        canvas.save(out)
 
 
 def compare(
@@ -182,6 +215,11 @@ def review_path(case_name: str) -> Path:
     return GOLDEN_ROOT / case_name / "review.json"
 
 
+def _over_tolerance_mask(image: Image.Image, threshold: int) -> Image.Image:
+    """A 1-bit mask of the pixels `_count_pixels_over` counts."""
+    return _channel_maximum(image).point(lambda value: 255 if value > threshold else 0).convert("1")
+
+
 def _count_pixels_over(image: Image.Image, threshold: int) -> int:
     """Count pixels whose largest channel exceeds ``threshold``.
 
@@ -190,8 +228,12 @@ def _count_pixels_over(image: Image.Image, threshold: int) -> int:
     `max(channel) > threshold` predicate while making large golden comparisons
     cheap enough to keep the suite's visual coverage.
     """
+    return sum(_channel_maximum(image).histogram()[threshold + 1 :])
+
+
+def _channel_maximum(image: Image.Image) -> Image.Image:
     channels = image.split()
     maximum = channels[0]
     for channel in channels[1:]:
         maximum = ImageChops.lighter(maximum, channel)
-    return sum(maximum.histogram()[threshold + 1 :])
+    return maximum