From 1c00c6cd052451a136794f72bd98f51cab9cfa08 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 30 Oct 2026 11:35:12 +0000 Subject: [PATCH 1/2] Check every recipe screenshot for overflow and broken images Agents kept screenshots as evidence without opening them; in October one shipped a broken LaunchIgniter badge in its own screenshot. Each evidence.shot() now soft-asserts that the page is no wider than the screen and that every visible image loaded, so the recipe reports every bad screen instead of relying on someone to look. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-debatelab/SKILL.md | 2 +- .../verify-debatelab/recipes/fixtures.ts | 22 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.claude/skills/verify-debatelab/SKILL.md b/.claude/skills/verify-debatelab/SKILL.md index b231a9b37..d0ca7b8c9 100644 --- a/.claude/skills/verify-debatelab/SKILL.md +++ b/.claude/skills/verify-debatelab/SKILL.md @@ -86,7 +86,7 @@ It also prints the commit the stack started at and whether there are uncommitted - `followSystemTheme(page)`: new accounts start on the light theme. A recipe that compares light and dark calls this first; it picks System in Settings, so `page.emulateMedia({ colorScheme })` then drives the theme. - `expectStreamed([...])`: every provider call for those features asked the model to stream, which proves the page showed the answer live. - `expectProviderCalls([...])`: what reached the provider. Each test writes `provider-calls.json`, and in live mode the fixture fails the test if any chat call was answered by a model other than the pin. -- `evidence`: `step(line)`, `aria(name, locator?)`, `shot(name)` and `json(name, value)`. +- `evidence`: `step(line)`, `aria(name, locator?)`, `shot(name)` and `json(name, value)`. Every `shot` also fails the recipe (softly, so all bad screens are reported) when the page is wider than the screen or a visible image failed to load. Still open the screenshots: this check can't see clipped text, overlaps or wrong colours. - Automatic failure on any uncaught page error, horizontal overflow, or remote Supabase request. Every result, source and practice round belongs to a Current Debate or is explicitly one-off. A tool given a motion when no debate exists opens `Start a debate with this motion?`; a motion that differs from the Current Debate's opens `This is a different motion`. Both open after the submit click and before the AI request, so create the `aiResponse()` promise, click, answer with `chooseForMotion`, then await. Or call `startDebate` first, after which tools prefill its motion and open no dialog. The Library's `Add source` is disabled until a debate exists. See `features/current-debate.md`. diff --git a/.claude/skills/verify-debatelab/recipes/fixtures.ts b/.claude/skills/verify-debatelab/recipes/fixtures.ts index 49bf24c38..05aef773d 100644 --- a/.claude/skills/verify-debatelab/recipes/fixtures.ts +++ b/.claude/skills/verify-debatelab/recipes/fixtures.ts @@ -90,9 +90,28 @@ export async function expectProviderCalls(features: string[]) { return calls; } +/** + * What no screenshot should show: a page wider than the screen, or an image + * that failed to load. Agents keep evidence they never open (one shipped a + * broken badge image in its own screenshot), so every `shot` checks for both. + * Soft, so one recipe reports every bad screen. Lazy images not yet fetched + * are skipped; an image that tried and got nothing is not. + */ +export async function expectSoundScreen(target: Page, name: string) { + const { overflow, brokenImages } = await target.evaluate(() => ({ + overflow: document.documentElement.scrollWidth - document.documentElement.clientWidth, + brokenImages: [...document.images] + .filter((img) => img.currentSrc && img.complete && img.naturalWidth === 0) + .filter((img) => img.getClientRects().length > 0 && getComputedStyle(img).visibility !== "hidden") + .map((img) => img.currentSrc), + })); + expect.soft(overflow, `${name}: horizontal page overflow in px`).toBeLessThanOrEqual(1); + expect.soft(brokenImages, `${name}: images that failed to load`).toEqual([]); +} + export type Evidence = { dir: string; - /** Full-page screenshot, `.png`. */ + /** Full-page screenshot, `.png`, after `expectSoundScreen`. */ shot(name: string, target?: Page): Promise; /** ARIA snapshot of a region (default: whole page), `.aria.yml`. */ aria(name: string, locator?: Locator): Promise; @@ -133,6 +152,7 @@ export const test = stackTest.extend<{ evidence: Evidence }>({ async shot(name, target = page) { const path = file(`${name}.png`); await target.screenshot({ path, fullPage: true }); + await expectSoundScreen(target, name); return path; }, async aria(name, locator = page.locator("body")) { From bb3408b581fb3048f957660ee51a7d988e777839 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 30 Oct 2026 13:27:02 +0000 Subject: [PATCH 2/2] Expect the translated not-found message as the h1 in loc-switcher The not-found page now shows "404" as an eyebrow and the translated message as its h1, so the recipe failed looking for a "404" heading. #279 fixed the same assertion in e2e/localization.spec.ts. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-debatelab/recipes/localization.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.claude/skills/verify-debatelab/recipes/localization.spec.ts b/.claude/skills/verify-debatelab/recipes/localization.spec.ts index 786a4b87b..d59eddd2a 100644 --- a/.claude/skills/verify-debatelab/recipes/localization.spec.ts +++ b/.claude/skills/verify-debatelab/recipes/localization.spec.ts @@ -101,12 +101,12 @@ test("loc-switcher: sign-in and not-found pages carry the language switcher", as await evidence.shot("02-login-hu"); await page.goto("/no-such-page"); - await expect(page.getByRole("heading", { level: 1, name: hu.notFound.title })).toBeVisible(); + await expect(page.getByRole("heading", { level: 1, name: hu.notFound.message })).toBeVisible(); await expect(page).toHaveTitle(hu.pageHead.notFound.title.replace("{{brand}}", BRAND)); await evidence.shot("03-not-found-hu"); await chooseLanguage(page, "es"); await expect(page).toHaveTitle(es.pageHead.notFound.title.replace("{{brand}}", BRAND)); - await expect(page.getByText(es.notFound.message)).toBeVisible(); + await expect(page.getByRole("heading", { level: 1, name: es.notFound.message })).toBeVisible(); await evidence.shot("04-not-found-es"); });