From 4c53eb2f114e3a56b2c8c956843f9668e1bbc75f Mon Sep 17 00:00:00 2001 From: Vitor Mattos Date: Mon, 21 Sep 2026 21:54:55 -0300 Subject: [PATCH] ci: improve README screenshot failure report Signed-off-by: Vitor Mattos --- .../workflows/screenshot-failure-comment.yml | 105 ++++++++++++++++-- scripts/check-screenshots.mjs | 27 ++++- 2 files changed, 118 insertions(+), 14 deletions(-) diff --git a/.github/workflows/screenshot-failure-comment.yml b/.github/workflows/screenshot-failure-comment.yml index b51b681..5c748b4 100644 --- a/.github/workflows/screenshot-failure-comment.yml +++ b/.github/workflows/screenshot-failure-comment.yml @@ -43,6 +43,7 @@ jobs: GH_REPO: ${{ github.repository }} PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} RUN_ID: ${{ github.event.workflow_run.id }} + RUN_URL: ${{ github.event.workflow_run.html_url }} REPORT_BRANCH: screenshot-reports shell: bash run: | @@ -105,33 +106,117 @@ jobs: fi raw_base="https://raw.githubusercontent.com/$GH_REPO/$REPORT_BRANCH/$report_path" + marker='' + + summary_line="Screenshot output differs from the committed README image." + if [[ -f screenshot-artifacts/demo-screenshot-report.json ]]; then + report_type="$(jq -r '.type' screenshot-artifacts/demo-screenshot-report.json)" + if [[ "$report_type" == "pixels" ]]; then + changed_pixels="$(jq -r '.changedPixels' screenshot-artifacts/demo-screenshot-report.json)" + total_pixels="$(jq -r '.totalPixels' screenshot-artifacts/demo-screenshot-report.json)" + ratio="$(jq -r '.ratio * 100' screenshot-artifacts/demo-screenshot-report.json)" + max_ratio="$(jq -r '.maxDiffRatio * 100' screenshot-artifacts/demo-screenshot-report.json)" + summary_line="**Difference:** $(printf '%.4f' "$ratio")% · **Allowed:** $(printf '%.4f' "$max_ratio")% +$changed_pixels of $total_pixels pixels differ." + elif [[ "$report_type" == "size" ]]; then + committed_size="$(jq -r '"\(.committed.width)x\(.committed.height)"' screenshot-artifacts/demo-screenshot-report.json)" + generated_size="$(jq -r '"\(.generated.width)x\(.generated.height)"' screenshot-artifacts/demo-screenshot-report.json)" + summary_line="**Size changed:** $committed_size → $generated_size" + fi + fi cat > screenshot-comment.md <> screenshot-comment.md <> screenshot-comment.md <<'EOF' + + A pixel diff is not available because the image dimensions changed. + EOF + fi + + cat >> screenshot-comment.md < + Expected screenshot ![Expected screenshot]($raw_base/expected.png?run=$RUN_ID) - ### Generated + + +
+ Generated screenshot ![Generated screenshot]($raw_base/generated.png?run=$RUN_ID) + +
+ + If the visual change is expected, run: + + `npm run screenshots:update` EOF - if [[ "$has_diff" == "true" ]]; then - cat >> screenshot-comment.md <")) | .id' | + head -n 1 + )" - ### Diff + if [[ -n "$existing_comment_id" ]]; then + jq -n --rawfile body screenshot-comment.md '{body: $body}' | + gh api --method PATCH "repos/$GH_REPO/issues/comments/$existing_comment_id" --input - >/dev/null + else + gh pr comment "$PR_NUMBER" --repo "$GH_REPO" --body-file screenshot-comment.md + fi + + cat >> "$GITHUB_STEP_SUMMARY" <> "$GITHUB_STEP_SUMMARY" <> "$GITHUB_STEP_SUMMARY" <<'EOF' + + A pixel diff is not available because the image dimensions changed. + EOF fi - cat >> screenshot-comment.md <<'EOF' + cat >> "$GITHUB_STEP_SUMMARY" < + Expected screenshot + + ![Expected screenshot]($raw_base/expected.png?run=$RUN_ID) - gh pr comment "$PR_NUMBER" --repo "$GH_REPO" --body-file screenshot-comment.md + + +
+ Generated screenshot + + ![Generated screenshot]($raw_base/generated.png?run=$RUN_ID) + +
+ EOF diff --git a/scripts/check-screenshots.mjs b/scripts/check-screenshots.mjs index 21b2d31..8b78f7b 100644 --- a/scripts/check-screenshots.mjs +++ b/scripts/check-screenshots.mjs @@ -11,16 +11,23 @@ import { generateScreenshot, packageRoot, screenshotPath } from './generate-scre const maxDiffRatio = 0.001 -const diffPath = path.join(packageRoot, 'test-results', 'demo-screenshot-diff.png') -const currentPath = path.join(packageRoot, 'test-results', 'demo-screenshot-current.png') -const expectedPath = path.join(packageRoot, 'test-results', 'demo-screenshot-expected.png') +const resultsDir = path.join(packageRoot, 'test-results') +const diffPath = path.join(resultsDir, 'demo-screenshot-diff.png') +const currentPath = path.join(resultsDir, 'demo-screenshot-current.png') +const expectedPath = path.join(resultsDir, 'demo-screenshot-expected.png') +const reportPath = path.join(resultsDir, 'demo-screenshot-report.json') async function writeComparisonImages(committedImage, currentImage) { - await mkdir(path.dirname(currentPath), { recursive: true }) + await mkdir(resultsDir, { recursive: true }) await writeFile(expectedPath, committedImage) await writeFile(currentPath, currentImage) } +async function writeReport(report) { + await mkdir(resultsDir, { recursive: true }) + await writeFile(reportPath, JSON.stringify(report, null, 2)) +} + async function main() { const committedImage = await readFile(screenshotPath) const committed = PNG.sync.read(committedImage) @@ -29,6 +36,11 @@ async function main() { if (committed.width !== current.width || committed.height !== current.height) { await writeComparisonImages(committedImage, currentImage) + await writeReport({ + type: 'size', + committed: { width: committed.width, height: committed.height }, + generated: { width: current.width, height: current.height }, + }) throw new Error( `Screenshot size changed: committed ${committed.width}x${committed.height}, ` + @@ -56,6 +68,13 @@ async function main() { await writeComparisonImages(committedImage, currentImage) await writeFile(diffPath, PNG.sync.write(diff)) + await writeReport({ + type: 'pixels', + changedPixels, + totalPixels, + ratio, + maxDiffRatio, + }) throw new Error( `${report}, above the allowed ${(maxDiffRatio * 100).toFixed(4)}%.\n` +