Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
204 changes: 203 additions & 1 deletion packages/platform-android/src/__tests__/input-actions.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { test, vi } from 'vitest';
import assert from 'node:assert/strict';
import { GESTURE_SAMPLE_INTERVAL_MS } from '@agent-device/contracts/gesture-plan';
import { GESTURE_DURATION_MAX_MS } from '@agent-device/contracts/gesture-plan-types';
import {
backAndroid,
homeAndroid,
Expand Down Expand Up @@ -79,7 +81,14 @@ test('scrollAndroid accepts sub-frame public durations at the Android planner mi
async () => {
const outputs: Record<string, unknown>[] = [];
for (const durationMs of [0, 15]) {
outputs.push(await scrollAndroid(ANDROID_EMULATOR, 'down', { durationMs }));
// 'inertial' keeps the injected plan's durationMs equal to the honored move time, so the
// flooring this test targets is not conflated with the 'controlled' release tail below.
outputs.push(
await scrollAndroid(ANDROID_EMULATOR, 'down', {
durationMs,
releaseBehavior: 'inertial',
}),
);
}
return outputs;
},
Expand All @@ -95,6 +104,199 @@ test('scrollAndroid accepts sub-frame public durations at the Android planner mi
);
});

test('scrollAndroid defaults to a controlled release: a quivering tail past the pan endpoint', async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
return { injected: true };
},
},
{ serial: ANDROID_EMULATOR.id },
async () => await scrollAndroid(ANDROID_EMULATOR, 'down', { pixels: 240, durationMs: 120 }),
);

assert.equal(touchCalls.length, 1);
const [touch] = touchCalls;
const samples = touch!.pointers[0]!.samples;
const endpoint = samples.find((sample) => sample.offsetMs === 120)!;
const tail = samples.filter((sample) => sample.offsetMs > 120);

// The plan carries a >=100ms tail past the honored 120ms move; the CLI-facing `durationMs` in
// the command result (asserted below) stays at the honored move time.
assert.equal(touch!.durationMs, 280);
assert.ok(tail.length >= 100 / GESTURE_SAMPLE_INTERVAL_MS);
for (const sample of tail) assert.equal(sample.point.y, endpoint.point.y);
const allPastEndpoint = [endpoint, ...tail];
for (let index = 1; index < allPastEndpoint.length; index += 1) {
assert.notEqual(allPastEndpoint[index]!.point.x, allPastEndpoint[index - 1]!.point.x);
}
});

test('scrollAndroid composes the duration floor with the default controlled-release tail', async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
},
},
{ serial: ANDROID_EMULATOR.id },
async () => await scrollAndroid(ANDROID_EMULATOR, 'down', { durationMs: 0 }),
);

// The move floors to the Android planner minimum (16ms) before the tail is appended, not after.
assert.equal(touchCalls[0]!.durationMs, GESTURE_SAMPLE_INTERVAL_MS + 160);
});

test('scrollAndroid runs the full controlled-release tail at the largest duration that still leaves it room', async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
const maxControlledMoveMs = GESTURE_DURATION_MAX_MS - 160;
const result = await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
},
},
{ serial: ANDROID_EMULATOR.id },
async () =>
await scrollAndroid(ANDROID_EMULATOR, 'down', {
pixels: 1800,
durationMs: maxControlledMoveMs,
}),
);

// The requested move is honored in full, and the tail always runs at its full length — the
// dispatched plan lands exactly at GESTURE_DURATION_MAX_MS, never past it.
assert.equal(result.durationMs, maxControlledMoveMs);
const [touch] = touchCalls;
assert.equal(touch!.durationMs, GESTURE_DURATION_MAX_MS);
assert.equal(touch!.pointers[0]!.samples.at(-1)!.offsetMs, GESTURE_DURATION_MAX_MS);
});

test('scrollAndroid rejects a controlled-release durationMs that would leave the release tail no room, without shortening the move', async () => {
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }),
touch: async () => {
throw new Error('touch must not run for a rejected request');
},
},
{ serial: ANDROID_EMULATOR.id },
async () => {
await assert.rejects(
scrollAndroid(ANDROID_EMULATOR, 'down', {
pixels: 1800,
durationMs: GESTURE_DURATION_MAX_MS - 159,
}),
/scroll durationMs must be at most 9840 for a controlled release/,
);
},
);
});

test("scrollAndroid accepts the full GESTURE_DURATION_MAX_MS for an 'inertial' release, which needs no tail", async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 0, y: 0, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
},
},
{ serial: ANDROID_EMULATOR.id },
async () =>
await scrollAndroid(ANDROID_EMULATOR, 'down', {
pixels: 1800,
durationMs: GESTURE_DURATION_MAX_MS,
releaseBehavior: 'inertial',
}),
);

const [touch] = touchCalls;
assert.equal(touch!.durationMs, GESTURE_DURATION_MAX_MS);
});

test('scrollAndroid jitters the axis orthogonal to a horizontal scroll, holding the scroll axis fixed', async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
return { injected: true };
},
},
{ serial: ANDROID_EMULATOR.id },
async () => await scrollAndroid(ANDROID_EMULATOR, 'left', { pixels: 240, durationMs: 120 }),
);

assert.equal(touchCalls.length, 1);
const [touch] = touchCalls;
const samples = touch!.pointers[0]!.samples;
const endpoint = samples.find((sample) => sample.offsetMs === 120)!;
const tail = samples.filter((sample) => sample.offsetMs > 120);

assert.ok(tail.length >= 100 / GESTURE_SAMPLE_INTERVAL_MS);
// The scroll axis (x, for a horizontal scroll) stays exactly at the endpoint — zero velocity
// there by construction; only the orthogonal axis (y) jitters to dodge the resampling quirk.
for (const sample of tail) assert.equal(sample.point.x, endpoint.point.x);
const allPastEndpoint = [endpoint, ...tail];
for (let index = 1; index < allPastEndpoint.length; index += 1) {
assert.notEqual(allPastEndpoint[index]!.point.y, allPastEndpoint[index - 1]!.point.y);
}
});

test('scrollAndroid honors an inertial release (scroll top/bottom): lifts at the pan endpoint', async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
await withAndroidAdbProvider(
{
exec: async () => {
throw new Error('adb must not run');
},
gestureViewport: async () => ({ x: 10, y: 20, width: 1080, height: 1920 }),
touch: async (request) => {
touchCalls.push(request);
return { injected: true };
},
},
{ serial: ANDROID_EMULATOR.id },
async () =>
await scrollAndroid(ANDROID_EMULATOR, 'down', {
pixels: 240,
durationMs: 120,
releaseBehavior: 'inertial',
}),
);

assert.equal(touchCalls.length, 1);
const [touch] = touchCalls;
assert.equal(touch!.durationMs, 120);
assert.equal(touch!.pointers[0]!.samples.at(-1)!.offsetMs, 120);
});

test('longPressAndroid sends a stationary semantic touch plan', async () => {
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
const result = await withAndroidAdbProvider(
Expand Down
131 changes: 114 additions & 17 deletions packages/platform-android/src/input-actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,28 @@
* IME, and the adb-shell writer — is `text-input.ts`.
*/
import { DEVICE_ROTATION_SURFACE_INDEX, type DeviceRotation } from '@agent-device/contracts/device';
import { buildGesturePlan } from '@agent-device/contracts/gesture-plan';
import { GESTURE_DURATION_MIN_MS } from '@agent-device/contracts/gesture-plan-types';
import { DEFAULT_MOBILE_SCROLL_DURATION_MS } from '@agent-device/contracts/scroll-command';
import { GESTURE_SAMPLE_INTERVAL_MS, buildGesturePlan } from '@agent-device/contracts/gesture-plan';
import {
GESTURE_DURATION_MAX_MS,
GESTURE_DURATION_MIN_MS,
} from '@agent-device/contracts/gesture-plan-types';
import type {
GesturePlan,
PointerTrajectorySample,
SinglePointerTrajectory,
} from '@agent-device/contracts/gesture-plan-types';
import {
DEFAULT_MOBILE_SCROLL_DURATION_MS,
type ScrollReleaseBehavior,
} from '@agent-device/contracts/scroll-command';
import {
type ScrollDirection,
buildScrollGesturePlan,
} from '@agent-device/contracts/scroll-gesture';
import { type TvRemoteButton, toAndroidTvRemoteKeyevent } from '@agent-device/contracts/tv-remote';
import type { DeviceInfo } from '@agent-device/kernel/device';
import { AppError } from '@agent-device/kernel/errors';
import type { Rect } from '@agent-device/kernel/snapshot';
import { sleep } from '@agent-device/host-kit/retry';
import { runAndroidAdb } from './adb.ts';
import { executeAndroidTouchPlan, readAndroidGestureViewport } from './touch-executor.ts';
Expand Down Expand Up @@ -163,7 +175,12 @@ export async function focusAndroid(device: DeviceInfo, x: number, y: number): Pr
export async function scrollAndroid(
device: DeviceInfo,
direction: ScrollDirection,
options?: { amount?: number; pixels?: number; durationMs?: number } & AndroidHelperSessionOptions,
options?: {
amount?: number;
pixels?: number;
durationMs?: number;
releaseBehavior?: ScrollReleaseBehavior;
} & AndroidHelperSessionOptions,
): Promise<Record<string, unknown>> {
// The viewport read and the gesture are two helper calls one command apart: giving the read the
// command's session scope keeps both on the same instrumentation.
Expand Down Expand Up @@ -192,21 +209,26 @@ export async function scrollAndroid(
options?.durationMs ?? DEFAULT_MOBILE_SCROLL_DURATION_MS,
GESTURE_DURATION_MIN_MS,
);
const releaseBehavior = options?.releaseBehavior ?? 'controlled';
if (releaseBehavior === 'controlled') assertRoomForControlledReleaseTail(durationMs);
const gesturePlan = buildGesturePlan(
{
intent: 'pan',
origin: { x: scrollPlan.x1, y: scrollPlan.y1 },
delta: {
x: scrollPlan.x2 - scrollPlan.x1,
y: scrollPlan.y2 - scrollPlan.y1,
},
durationMs,
},
viewport,
'android',
);
const backend = await executeAndroidTouchPlan(
device,
buildGesturePlan(
{
intent: 'pan',
origin: { x: scrollPlan.x1, y: scrollPlan.y1 },
delta: {
x: scrollPlan.x2 - scrollPlan.x1,
y: scrollPlan.y2 - scrollPlan.y1,
},
durationMs,
},
viewport,
'android',
),
releaseBehavior === 'controlled'
? withControlledReleaseTail(gesturePlan, viewport, direction)
: gesturePlan,
);

return {
Expand All @@ -216,6 +238,81 @@ export async function scrollAndroid(
};
}

// Kept an even multiple of GESTURE_SAMPLE_INTERVAL_MS so the tail's last sample lands back on the
// pan's exact endpoint (an odd multiple would still avoid the fling — every consecutive sample
// still differs — but would leave the release 1px off the requested endpoint).
const CONTROLLED_RELEASE_TAIL_MS = 160;

// The dispatched plan (move + tail) must never exceed GESTURE_DURATION_MAX_MS, the same ceiling
// every gesture plan is built under. Rather than silently dropping the tail for a move that
// leaves it no room, a controlled scroll's own accepted range stops short of the shared ceiling
// by the tail's length — the full tail runs for every accepted controlled scroll, and a request
// past this narrower range is rejected with the reason, not truncated.
const CONTROLLED_RELEASE_MAX_MOVE_MS = GESTURE_DURATION_MAX_MS - CONTROLLED_RELEASE_TAIL_MS;

function assertRoomForControlledReleaseTail(durationMs: number): void {
if (durationMs <= CONTROLLED_RELEASE_MAX_MOVE_MS) return;
throw new AppError(
'INVALID_ARGS',
`scroll durationMs must be at most ${CONTROLLED_RELEASE_MAX_MOVE_MS} for a controlled release ` +
`(leaves room for the ${CONTROLLED_RELEASE_TAIL_MS}ms release tail within the ` +
`${GESTURE_DURATION_MAX_MS}ms gesture ceiling)`,
{
hint: "Pass a shorter durationMs, or releaseBehavior 'inertial' if the fling is acceptable.",
},
);
}

/**
* A short, quivering tail appended after a 'controlled' scroll's endpoint, adding
* `CONTROLLED_RELEASE_TAIL_MS` of real time to the gesture. AOSP's `InputConsumer::rewriteMessage`
* collapses a MOVE that repeats the previous coordinates into a "resampled" sample, and
* `VelocityTracker` skips resampled samples — so a truly stationary tail never reaches the
* tracker, and `ScrollView.onTouchEvent` (which computes release velocity before applying UP)
* still flings at the pan's velocity. Nudging the axis orthogonal to the scroll by 1px every frame
* (holding the scroll axis exactly at the endpoint — zero velocity there by construction) keeps
* every sample distinct without adding net travel along either axis. Measured fling-free for
* vertical scrolls on a `RecyclerView` and an RN `ScrollView` (issue #2371); not independently
* verified against every OEM skin or a Compose `LazyColumn`. An 'inertial' release (the
* `scroll top`/`scroll bottom` edge passes) lifts at the pan's endpoint unchanged.
*
* Callers must have already checked `assertRoomForControlledReleaseTail` on the move duration —
* this always appends the full tail.
*/
function withControlledReleaseTail(
plan: GesturePlan,
viewport: Rect,
direction: ScrollDirection,
): GesturePlan {
if (plan.topology !== 'single') return plan;
const steps = CONTROLLED_RELEASE_TAIL_MS / GESTURE_SAMPLE_INTERVAL_MS;
const [pointer] = plan.pointers;
const end = pointer.samples.at(-1)!;
const horizontal = direction === 'left' || direction === 'right';
const jitterBase = horizontal ? end.point.y : end.point.x;
const jitterMin = (horizontal ? viewport.y : viewport.x) + 1;
const jitterMax = (horizontal ? viewport.y + viewport.height : viewport.x + viewport.width) - 1;
const nudged = jitterBase + 1 <= jitterMax ? jitterBase + 1 : Math.max(jitterMin, jitterBase - 1);
const tail: PointerTrajectorySample[] = Array.from({ length: steps }, (_, index) => {
const jitter = index % 2 === 0 ? nudged : jitterBase;
return {
offsetMs: plan.durationMs + (index + 1) * GESTURE_SAMPLE_INTERVAL_MS,
point: horizontal ? { x: end.point.x, y: jitter } : { x: jitter, y: end.point.y },
};
});
const samples: SinglePointerTrajectory['samples'] = [
pointer.samples[0],
pointer.samples[1],
...pointer.samples.slice(2),
...tail,
];
return {
...plan,
durationMs: plan.durationMs + CONTROLLED_RELEASE_TAIL_MS,
pointers: [{ ...pointer, samples }],
};
}

function resolveAndroidUserRotation(orientation: DeviceRotation): string {
const index = DEVICE_ROTATION_SURFACE_INDEX[orientation];
if (index === undefined) {
Expand Down
Loading
Loading