From 120d8ad3af3b6e281cd44ed983a9e44e1a902b03 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 05:59:41 +0000 Subject: [PATCH 1/2] test(android-e2e): reveal automation canaries by visibility probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `smoke:automation-system` revealed its canaries with a single blind `scroll down ` and then acted on them immediately. `scroll` is a gesture, not an offset — the command's own contract says "app scroll physics determine the final content offset" — so one amount cannot guarantee the target is on screen, least of all right after the landscape/portrait round-trip has relaid the list out. Both open Android smoke failures sit on that seam: - step 34 `press id="automation-press"` → `selector_not_found`, right after `scroll down 0.7`. Reproduced on #2360 and #2361 (disjoint diffs), on `main` at bd08e6e, and on #2356 at the identical line, so the orientation fix does not cover it. - the post-alert canary after `scroll down 1`, which #2356 records as still open. The iOS scenario already reveals the same canaries correctly, probing `is visible` and scrolling again until the element is on screen. That policy was iOS-only; this moves it to the shared live-device layer as `createVisibilityScroll` and gives the Android scenario the same reveal. Its unit test moves with it. No production code changes, and the press and diff assertions still have to pass against a real element — the scroll budget is bounded and a genuinely absent canary still fails. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Jqfa11D8QsCMuL17SsLvDz --- .../android-emulator-e2e/live-assertions.ts | 6 ++ .../live-automation-scenario.ts | 8 +- .../ios-simulator-e2e/live-assertions.ts | 59 +------------ ...live-device-e2e-visibility-scroll.test.ts} | 2 +- .../live-device-e2e/visibility-scroll.ts | 85 +++++++++++++++++++ 5 files changed, 101 insertions(+), 59 deletions(-) rename test/integration/{ios-simulator-e2e-visibility-scroll.test.ts => live-device-e2e-visibility-scroll.test.ts} (94%) create mode 100644 test/integration/live-device-e2e/visibility-scroll.ts diff --git a/test/integration/android-emulator-e2e/live-assertions.ts b/test/integration/android-emulator-e2e/live-assertions.ts index cb6ff84bc2..de2d6a9588 100644 --- a/test/integration/android-emulator-e2e/live-assertions.ts +++ b/test/integration/android-emulator-e2e/live-assertions.ts @@ -9,6 +9,7 @@ import { assertNonEmptyFile, createLiveDeviceAssertions, } from '../live-device-e2e/assertions.ts'; +import { createVisibilityScroll } from '../live-device-e2e/visibility-scroll.ts'; import type { CliJsonResult } from '../cli-json.ts'; import type { AndroidEmulatorBehaviorId } from './behavior-coverage.ts'; import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; @@ -22,6 +23,11 @@ export const { assertElementText, assertWaitSelector, assertWaitText, capturePng PUBLIC_COMMANDS.wait, ); +export const { scrollUntilVisible } = createVisibilityScroll< + AndroidEmulatorBehaviorId, + LiveContext +>(runStep); + export function assertDiffLine( result: CliJsonResult, kind: SnapshotDiffLine['kind'], diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.ts b/test/integration/android-emulator-e2e/live-automation-scenario.ts index 0dddb2824e..f99847c2b3 100644 --- a/test/integration/android-emulator-e2e/live-automation-scenario.ts +++ b/test/integration/android-emulator-e2e/live-automation-scenario.ts @@ -12,6 +12,7 @@ import { assertWaitText, capturePng, requireAndroidResourceId, + scrollUntilVisible, } from './live-assertions.ts'; import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; @@ -126,7 +127,10 @@ export async function assertAutomationSystem(context: LiveContext): Promise(runStep); export async function assertElementTextAfterScrolling( context: LiveContext, selector: string, expected: string, ): Promise { - await searchForVisibleElement( - selector, - (attempt) => - runStep( - context, - `check ${selector} visibility after scroll (attempt ${attempt})`, - ['is', 'visible', selector], - { allowFailure: true }, - ), - (attempt) => - runStep(context, `scroll toward ${selector} after attempt ${attempt}`, [ - 'scroll', - 'down', - '0.75', - ]).then(() => undefined), - ); + await scrollUntilVisible(context, selector); await assertElementText(context, selector, expected); } -/** - * Searches by semantic visibility rather than selector existence. An offscreen node can exist in - * the accessibility tree, so a successful `wait ` is not sufficient evidence to skip - * scrolling. The callbacks keep this live-device policy deterministic and unit-testable without a - * simulator. - */ -export async function searchForVisibleElement( - selector: string, - probeVisibility: (attempt: number) => Promise, - scrollAfterAttempt: (attempt: number) => Promise, -): Promise { - let stallRetriesLeft = SCROLL_SEARCH_STALL_RETRIES; - let lastFailure: CliJsonResult | undefined; - - for (let attempt = 1; attempt <= SCROLL_SEARCH_ATTEMPTS;) { - const probe = await probeVisibility(attempt); - if (probe.status === 0) return; - lastFailure = probe; - - // The snapshot never came back, so the surface was never read. Scrolling here would move the - // surface for a reason unrelated to visibility and spend an attempt on no evidence. - if (probe.json?.error?.details?.captureStalled === true && stallRetriesLeft > 0) { - stallRetriesLeft -= 1; - continue; - } - - attempt += 1; - if (attempt <= SCROLL_SEARCH_ATTEMPTS) { - await scrollAfterAttempt(attempt - 1); - } - } - assert.fail( - `${selector} did not become visible after scrolling\nlast visibility probe: ${JSON.stringify(lastFailure?.json ?? null)}`, - ); -} - function requireNode( result: CliJsonResult, identifier: string, diff --git a/test/integration/ios-simulator-e2e-visibility-scroll.test.ts b/test/integration/live-device-e2e-visibility-scroll.test.ts similarity index 94% rename from test/integration/ios-simulator-e2e-visibility-scroll.test.ts rename to test/integration/live-device-e2e-visibility-scroll.test.ts index 4824634682..b2022bc27d 100644 --- a/test/integration/ios-simulator-e2e-visibility-scroll.test.ts +++ b/test/integration/live-device-e2e-visibility-scroll.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import type { CliJsonResult } from './cli-json.ts'; -import { searchForVisibleElement } from './ios-simulator-e2e/live-assertions.ts'; +import { searchForVisibleElement } from './live-device-e2e/visibility-scroll.ts'; function result(status: number, details?: Record): CliJsonResult { return { diff --git a/test/integration/live-device-e2e/visibility-scroll.ts b/test/integration/live-device-e2e/visibility-scroll.ts new file mode 100644 index 0000000000..6e2811cff5 --- /dev/null +++ b/test/integration/live-device-e2e/visibility-scroll.ts @@ -0,0 +1,85 @@ +import assert from 'node:assert/strict'; + +import type { CliJsonResult } from '../cli-json.ts'; +import type { LiveDeviceContext } from './runtime.ts'; + +type RunStep = ( + context: Context, + step: string, + args: string[], + options?: { allowFailure?: boolean }, +) => Promise; + +const SCROLL_SEARCH_ATTEMPTS = 4; +// A stalled capture says nothing about where the element is, so it must not consume the scroll +// budget outright; a couple of retries absorb a slow runner without masking a real absence. +const SCROLL_SEARCH_STALL_RETRIES = 2; +// One finger path per attempt. `scroll` is a gesture, not an offset — app scroll physics decide +// where the content lands — so the search re-probes rather than trusting a single amount. +const SCROLL_SEARCH_AMOUNT = '0.75'; + +/** + * Searches by semantic visibility rather than selector existence. An offscreen node can exist in + * the accessibility tree, so a successful `wait ` is not sufficient evidence to skip + * scrolling. The callbacks keep this live-device policy deterministic and unit-testable without a + * device. + */ +export async function searchForVisibleElement( + selector: string, + probeVisibility: (attempt: number) => Promise, + scrollAfterAttempt: (attempt: number) => Promise, +): Promise { + let stallRetriesLeft = SCROLL_SEARCH_STALL_RETRIES; + let lastFailure: CliJsonResult | undefined; + + for (let attempt = 1; attempt <= SCROLL_SEARCH_ATTEMPTS;) { + const probe = await probeVisibility(attempt); + if (probe.status === 0) return; + lastFailure = probe; + + // The snapshot never came back, so the surface was never read. Scrolling here would move the + // surface for a reason unrelated to visibility and spend an attempt on no evidence. + if (probe.json?.error?.details?.captureStalled === true && stallRetriesLeft > 0) { + stallRetriesLeft -= 1; + continue; + } + + attempt += 1; + if (attempt <= SCROLL_SEARCH_ATTEMPTS) { + await scrollAfterAttempt(attempt - 1); + } + } + assert.fail( + `${selector} did not become visible after scrolling\nlast visibility probe: ${JSON.stringify(lastFailure?.json ?? null)}`, + ); +} + +/** + * Binds {@link searchForVisibleElement} to a platform's `runStep`, so every live scenario reveals + * a canary the same way: probe visibility, scroll one finger path, probe again. + */ +export function createVisibilityScroll< + BehaviorId extends string, + Context extends LiveDeviceContext, +>(runStep: RunStep) { + async function scrollUntilVisible(context: Context, selector: string): Promise { + await searchForVisibleElement( + selector, + (attempt) => + runStep( + context, + `check ${selector} visibility after scroll (attempt ${attempt})`, + ['is', 'visible', selector], + { allowFailure: true }, + ), + (attempt) => + runStep(context, `scroll toward ${selector} after attempt ${attempt}`, [ + 'scroll', + 'down', + SCROLL_SEARCH_AMOUNT, + ]).then(() => undefined), + ); + } + + return { scrollUntilVisible }; +} From a2c4368d494e295b4ff8f4b3b29d132af7bc7a94 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 06:16:15 +0000 Subject: [PATCH 2/2] test(android-e2e): anchor the canary search and restore failed-step evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first CI run of the visibility search refuted the scroll-position hypothesis it was built on. `id="automation-press"` failed with `selector_not_found` after four probes and three 0.75-viewport scrolls — the element was not in the accessibility tree at all, not merely off screen. Two consequences, both fixed here. The search only scrolls DOWN, so a canary the rotation round-trip left ABOVE the viewport is unreachable no matter how many attempts it spends. Both Android reveals now anchor with `scroll top` first, the idiom the scenario already uses after system UI perturbs the route, so the search starts from a known position and the target can only be below it. The probes run with `allowFailure`, so the budget-exhausted `assert.fail` never reached the harness's failed-step evidence capture: the Android artifact fell from 143,676 to 12,086 bytes and lost the screenshot, snapshot and device facts that name what was on screen instead. The search now spends one final probe as a real step before failing, which restores all three. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Jqfa11D8QsCMuL17SsLvDz --- .../live-automation-scenario.ts | 8 +++++++- .../live-device-e2e-visibility-scroll.test.ts | 19 +++++++++++++++++++ .../live-device-e2e/visibility-scroll.ts | 11 +++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.ts b/test/integration/android-emulator-e2e/live-automation-scenario.ts index f99847c2b3..2b25e95207 100644 --- a/test/integration/android-emulator-e2e/live-automation-scenario.ts +++ b/test/integration/android-emulator-e2e/live-automation-scenario.ts @@ -129,7 +129,9 @@ export async function assertAutomationSystem(context: LiveContext): Promise { + const evidenceProbes: number[] = []; + + await assert.rejects( + searchForVisibleElement( + 'id="automation-press"', + async () => result(1), + async () => {}, + async () => { + evidenceProbes.push(1); + return result(1); + }, + ), + /did not become visible after scrolling/, + ); + + assert.deepEqual(evidenceProbes, [1]); +}); + test('a stalled capture retries without scrolling or consuming an attempt', async () => { const probes = [result(1, { captureStalled: true }), result(0)]; const probeAttempts: number[] = []; diff --git a/test/integration/live-device-e2e/visibility-scroll.ts b/test/integration/live-device-e2e/visibility-scroll.ts index 6e2811cff5..5af5b39328 100644 --- a/test/integration/live-device-e2e/visibility-scroll.ts +++ b/test/integration/live-device-e2e/visibility-scroll.ts @@ -28,6 +28,7 @@ export async function searchForVisibleElement( selector: string, probeVisibility: (attempt: number) => Promise, scrollAfterAttempt: (attempt: number) => Promise, + probeForEvidence?: () => Promise, ): Promise { let stallRetriesLeft = SCROLL_SEARCH_STALL_RETRIES; let lastFailure: CliJsonResult | undefined; @@ -49,6 +50,10 @@ export async function searchForVisibleElement( await scrollAfterAttempt(attempt - 1); } } + // The probes above run with `allowFailure`, so none of them reached the harness's failed-step + // evidence capture. Spend one more as a real step: it fails the same way and writes the + // screenshot, snapshot and device facts that say what was on screen instead. + await probeForEvidence?.(); assert.fail( `${selector} did not become visible after scrolling\nlast visibility probe: ${JSON.stringify(lastFailure?.json ?? null)}`, ); @@ -78,6 +83,12 @@ export function createVisibilityScroll< 'down', SCROLL_SEARCH_AMOUNT, ]).then(() => undefined), + () => + runStep(context, `probe ${selector} after exhausting the scroll budget`, [ + 'is', + 'visible', + selector, + ]), ); }