From 607fb679473a93386f626daec9f5ba920764cb16 Mon Sep 17 00:00:00 2001 From: Vaibhav P <52103891+Martian-dev@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:17:59 +0000 Subject: [PATCH 1/2] Add MCP server so agent harnesses can play browser games Runwave's CLI runs its own VLM loop and produces a playtest video. This adds an MCP surface where the connected agent is the player instead: observe a frame, send a timed input sequence, observe the next frame. The server embeds the controller rather than driving the daemon. That is possible because BrowserSession and runStep already take their dependencies as arguments, and it avoids the daemon's session files, unauthenticated loopback port, process.exit on stop, and cwd captured at module load. The normalize/timeline/execute chain, session factory, state reader, and grid overlay are reused unchanged. Recording is out of scope for interactive play, so Chromium runs headless and the gstreamer, PulseAudio, and Xvfb requirements are gone. Adaptations for an agent player rather than a playtest bot: - Frames return as MCP image blocks, downscaled by default. A 1280x720 PNG is ~1200 tokens against ~300 at half scale, which dominates cost over a long navigation. zoom covers detail without a full frame. - act reports whether the frame changed. A byte-identical frame means the input did not reach the game, not that the game ignored it. - Grid overlay off by default: it enlarges the PNG with a label margin, desyncing image coordinates from input coordinates. The offset is reported when it is on. - Grid cells resolve to the cell centre via a new opt-in markGridSampleMode. Playtest scatter varies footage but misses precise targets and makes runs unreproducible; the default is unchanged. - Per-turn state drops the WebGL probe and keeps the largest canvas as the game area. - Calls serialize per session, since one page and one step counter mean concurrent calls would interleave keypresses. - Signal handlers and a 30 minute idle timeout close detached Chromium and game processes. Tests: 12 unit tests, plus an integration test that drives real headless Chromium against a fixture game and skips when Chromium cannot launch. The integration path is unverified here: this machine is missing libnss3/libnspr4, so it has only been exercised via unit tests so far. Co-Authored-By: Claude --- mcp/Dockerfile | 33 + mcp/README.md | 131 +++ mcp/bin/runwave-mcp.js | 41 + mcp/src/config.js | 56 + mcp/src/diff.js | 29 + mcp/src/frame.js | 54 + mcp/src/image.js | 108 ++ mcp/src/registry.js | 60 + mcp/src/result.js | 50 + mcp/src/schema.js | 112 ++ mcp/src/server.js | 50 + mcp/src/session.js | 109 ++ mcp/src/state.js | 46 + mcp/src/tools-aux.js | 140 +++ mcp/src/tools-play.js | 178 +++ mcp/test/fixtures/game/index.html | 62 + mcp/test/integration.test.js | 131 +++ mcp/test/unit.test.js | 143 +++ package-lock.json | 1170 ++++++++++++++++++- package.json | 10 +- runwave/controller/src/action-normalizer.js | 4 +- runwave/protocol/src/mark-grid.js | 20 +- 22 files changed, 2730 insertions(+), 7 deletions(-) create mode 100644 mcp/Dockerfile create mode 100644 mcp/README.md create mode 100644 mcp/bin/runwave-mcp.js create mode 100644 mcp/src/config.js create mode 100644 mcp/src/diff.js create mode 100644 mcp/src/frame.js create mode 100644 mcp/src/image.js create mode 100644 mcp/src/registry.js create mode 100644 mcp/src/result.js create mode 100644 mcp/src/schema.js create mode 100644 mcp/src/server.js create mode 100644 mcp/src/session.js create mode 100644 mcp/src/state.js create mode 100644 mcp/src/tools-aux.js create mode 100644 mcp/src/tools-play.js create mode 100644 mcp/test/fixtures/game/index.html create mode 100644 mcp/test/integration.test.js create mode 100644 mcp/test/unit.test.js diff --git a/mcp/Dockerfile b/mcp/Dockerfile new file mode 100644 index 0000000..542fbcd --- /dev/null +++ b/mcp/Dockerfile @@ -0,0 +1,33 @@ +# Runwave MCP server: lets an agent harness play browser games. +# +# The Playwright base image already carries every system library Chromium needs +# (libnss3, libnspr4, libgbm, the X client libs), which is the whole reason to +# start from it rather than a plain node image. +# +# No recording means no gstreamer, no PulseAudio, and no Xvfb: Chromium runs +# headless, so this image stays far smaller than the playtest runner's. +FROM mcr.microsoft.com/playwright:v1.61.1-noble + +ENV NODE_ENV=production \ + RUNWAVE_MCP_WORKSPACE=/var/lib/runwave-mcp + +WORKDIR /opt/runwave + +# Dependencies are copied first so a source edit does not invalidate the +# install layer. +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev + +COPY runwave ./runwave +COPY mcp ./mcp + +RUN mkdir -p "$RUNWAVE_MCP_WORKSPACE" + +# Chromium's sandbox needs privileges a default container does not grant. The +# base image ships a non-root user with the right setup; use it rather than +# disabling the sandbox. +USER pwuser + +# stdio transport: the host speaks MCP over stdin/stdout, so nothing is exposed +# on the network and no port is published. +ENTRYPOINT ["node", "/opt/runwave/mcp/bin/runwave-mcp.js"] diff --git a/mcp/README.md b/mcp/README.md new file mode 100644 index 0000000..4d0e75f --- /dev/null +++ b/mcp/README.md @@ -0,0 +1,131 @@ +# Runwave MCP + +An MCP server that lets an agent harness such as Claude Code play a browser game +directly: look at a frame, send a timed sequence of inputs, look at the next +frame. + +This is the interactive counterpart to the `runwave` CLI. The CLI runs a VLM in a +loop by itself and produces a recorded video for playtesting. Here the connected +agent *is* the player, so the OpenRouter agent loop is not used and there is no +recording — which is also why this needs none of runwave's gstreamer, +PulseAudio, or Xvfb setup. + +## Requirements + +- Node 20+ +- Chromium's system libraries. On a bare Linux host: + ```sh + npx playwright install --with-deps chromium # needs sudo for the libs + ``` + In Docker, use `mcp/Dockerfile`, which starts from the Playwright base image + and already has them. + +No X server, no audio, no display. Chromium runs headless. + +## Run + +```sh +node mcp/bin/runwave-mcp.js +``` + +Artifacts (screenshots, per-step JSON) are written under +`RUNWAVE_MCP_WORKSPACE`, defaulting to a directory in the system temp dir. Set +it explicitly if you want to keep them: + +```sh +RUNWAVE_MCP_WORKSPACE=./artifacts node mcp/bin/runwave-mcp.js +``` + +Register it with Claude Code: + +```sh +claude mcp add runwave -- node /absolute/path/to/mcp/bin/runwave-mcp.js +``` + +Or in Docker: + +```sh +docker build -f mcp/Dockerfile -t runwave-mcp . +claude mcp add runwave -- docker run --rm -i runwave-mcp +``` + +## Tools + +| Tool | Purpose | +| --- | --- | +| `launch_game` | Start a session from a `url`, or a `game_dir` containing `start.sh` plus a `port`. Returns the first frame. | +| `observe` | Fresh frame and state, no input sent. | +| `act` | Send a timed input sequence, return the resulting frame. | +| `zoom` | Full-resolution crop of a region, to read small UI without a full frame. | +| `capture` | Save a clean, full-resolution, un-annotated screenshot. The deliverable. | +| `reset_game` | Reload at the launch URL. | +| `journal` | Text log of what has been tried this session. | +| `list_sessions` | Running sessions. | +| `end_game` | Close the browser and stop the game process. | + +## How `act` works + +An action sequence is timed, not a single keypress. Offsets are milliseconds +from the start of the sequence and actions may overlap, so one call can express +"hold right for 900ms and jump at 150ms": + +```json +{ + "session_id": "game-...", + "actions": [ + { "type": "key", "start": 0, "end": 900, "key": "ArrowRight" }, + { "type": "key", "start": 150, "end": 230, "key": "Space" } + ] +} +``` + +This matters because an agent turn costs seconds. Committing to a move beats +sending one tap per turn. + +Action types: `key`, `click`, `multi_click`, `drag`, `cursor_move`, `view_move`. +Pointer actions take either `x`/`y` in viewport pixels or an +`overlay_row`/`overlay_col` grid cell. The full schema is enforced on the tool +input, so a malformed sequence is rejected before any input is sent. + +## Notes on the design + +**Frames are downscaled by default.** A 1280x720 PNG is roughly 1200 tokens; at +half scale it is about 300. Over a long navigation that difference dominates +everything else. Pass `full_res: true` when detail genuinely matters, or use +`zoom` on the region you care about — usually cheaper than a full-resolution +frame. + +**One frame per turn.** Interval captures are off. Pass `captures` with explicit +offsets to see a trajectory within a sequence. + +**`act` reports whether the frame changed.** A byte-identical frame almost always +means the input never reached the game, rather than the game ignoring it. The +tool says so instead of leaving the agent to guess. + +**The grid overlay is off by default.** When enabled it enlarges the PNG with a +label margin on every side, so pixels read off the image no longer match pixels +sent back as `x`/`y`. The offset is reported in the response when the grid is on, +but exact coordinates or grid cells are the better targets. + +**Grid cells resolve to the cell centre.** Runwave's playtest path deliberately +scatters clicks inside a cell to vary footage; that is wrong when aiming at a +specific target, and it makes a run unreproducible. Playtest behaviour is +unchanged — this server opts into `markGridSampleMode: 'center'`. + +**Calls are serialized per session.** There is one Playwright page and one shared +step counter, so concurrent calls would interleave keypresses. Subagents may +share a `session_id` safely. + +**Sessions close on shutdown.** Chromium and any spawned game process are +detached children; `SIGINT`, `SIGTERM`, `SIGHUP`, and an uncaught exception all +close them. Sessions also close after 30 minutes idle, so a forgotten +`end_game` does not leak a browser. + +## Tests + +```sh +npm run test:mcp +``` + +The integration test drives a real headless Chromium against a fixture game and +skips itself when Chromium cannot launch. diff --git a/mcp/bin/runwave-mcp.js b/mcp/bin/runwave-mcp.js new file mode 100644 index 0000000..b814bef --- /dev/null +++ b/mcp/bin/runwave-mcp.js @@ -0,0 +1,41 @@ +#!/usr/bin/env node +'use strict'; + +const { StdioServerTransport } = require('@modelcontextprotocol/sdk/server/stdio.js'); +const { createServer } = require('../src/server'); + +async function main() { + const { server, registry, workspace } = createServer(); + // stdout is the MCP transport, so diagnostics go to stderr only. + process.stderr.write(`runwave-mcp workspace: ${workspace}\n`); + + // Chromium and any game process are detached children. Without this a host + // shutting the server down orphans the whole tree, leaving browsers running. + let shuttingDown = false; + const shutdown = async (signal) => { + if (shuttingDown) return; + shuttingDown = true; + process.stderr.write(`runwave-mcp shutting down on ${signal}\n`); + await registry.closeAll(); + await server.close().catch(() => {}); + process.exit(0); + }; + + for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) { + process.on(signal, () => { shutdown(signal); }); + } + process.on('uncaughtException', async (error) => { + process.stderr.write(`runwave-mcp uncaught: ${error.stack || error.message}\n`); + await shutdown('uncaughtException'); + }); + process.on('unhandledRejection', (reason) => { + process.stderr.write(`runwave-mcp unhandled rejection: ${reason}\n`); + }); + + await server.connect(new StdioServerTransport()); +} + +main().catch((error) => { + process.stderr.write(`${error.stack || error.message}\n`); + process.exit(1); +}); diff --git a/mcp/src/config.js b/mcp/src/config.js new file mode 100644 index 0000000..872aa3b --- /dev/null +++ b/mcp/src/config.js @@ -0,0 +1,56 @@ +'use strict'; + +const path = require('path'); + +const DEFAULT_VIEWPORT = { width: 1280, height: 720 }; + +// Runwave scatters clicks within a cell to vary playtest footage. An agent +// aiming at a target needs the opposite: the same cell must mean the same pixel +// so a saved action trace replays identically. +const CELL_SAMPLE_MODE = 'center'; + +function normalizeViewport(viewport) { + const width = Number(viewport && viewport.width); + const height = Number(viewport && viewport.height); + return { + width: Number.isFinite(width) && width > 0 ? Math.round(width) : DEFAULT_VIEWPORT.width, + height: Number.isFinite(height) && height > 0 ? Math.round(height) : DEFAULT_VIEWPORT.height, + }; +} + +// The daemon hands createSession its raw CLI input, which is why grid-cell +// actions fail there when no viewport was passed. Building the config +// explicitly closes that gap: viewport is always present and always numeric. +function buildSessionConfig(options = {}) { + const viewport = normalizeViewport(options.viewport); + return { + kind: 'web', + ...(options.url ? { url: options.url } : {}), + ...(options.gameDir ? { gameDir: path.resolve(options.gameDir) } : {}), + ...(options.port ? { port: Number(options.port) } : {}), + viewport, + deviceScaleFactor: 1, + // No recording: no gstreamer, no PulseAudio, no headed Chromium. + record: false, + headless: true, + // Overlay is opt-in per call. It enlarges the PNG by a margin per side, + // which desyncs image coordinates from input coordinates. + gridScreenshots: false, + fullPageScreenshots: false, + // The agent asks for frames explicitly; interval captures would spend + // context on frames nobody requested. + autoCaptures: false, + markGridSampleMode: CELL_SAMPLE_MODE, + ...(options.markGridRows ? { markGridRows: Number(options.markGridRows) } : {}), + ...(options.markGridCols ? { markGridCols: Number(options.markGridCols) } : {}), + ...(options.stateExpression ? { stateExpression: String(options.stateExpression) } : {}), + waitAfterLoad: Number(options.waitAfterLoad ?? 700), + }; +} + +module.exports = { + CELL_SAMPLE_MODE, + DEFAULT_VIEWPORT, + buildSessionConfig, + normalizeViewport, +}; diff --git a/mcp/src/diff.js b/mcp/src/diff.js new file mode 100644 index 0000000..b28e8c4 --- /dev/null +++ b/mcp/src/diff.js @@ -0,0 +1,29 @@ +'use strict'; + +const crypto = require('crypto'); +const fs = require('fs'); + +function hashFile(file) { + try { + return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); + } catch { + return null; + } +} + +// "Did anything happen?" is the single most useful signal after an input, and +// the cheapest: no second screenshot, no pixel walk. A false here usually means +// the input never reached the game, which is otherwise easy to misread as the +// game ignoring the move. +function changedSince(previousFile, nextFile) { + if (!previousFile || !nextFile) return null; + const before = hashFile(previousFile); + const after = hashFile(nextFile); + if (!before || !after) return null; + return before !== after; +} + +module.exports = { + changedSince, + hashFile, +}; diff --git a/mcp/src/frame.js b/mcp/src/frame.js new file mode 100644 index 0000000..a412532 --- /dev/null +++ b/mcp/src/frame.js @@ -0,0 +1,54 @@ +'use strict'; + +const { drawMarkGridOnScreenshot } = require('../../runwave/controller/src/grid-overlay'); +const { readPng } = require('./image'); +const { frameBlocks, gridNote, stateText, textBlock } = require('./result'); + +// The overlay writes the PNG larger than the capture by a fixed margin per +// side. gridLabelStyle is private, so the margin is recovered from the file +// itself rather than reimplementing the label metrics. +function overlayMargin(file, viewport) { + try { + const png = readPng(file); + const margin = Math.round((png.width - Number(viewport.width)) / 2); + return margin > 0 ? margin : 0; + } catch { + return 0; + } +} + +// Draws the overlay onto an existing capture and reports the margin it added. +function applyGrid(session, file) { + drawMarkGridOnScreenshot(file, session.config); + return overlayMargin(file, session.config.viewport); +} + +// Screenshots are taken clean. The grid is drawn only when a caller asks for +// it, so the deliverable frame is never annotated. +async function captureFrame(session, { name, grid = false }) { + const dir = session.actionDir(name); + const file = await session.browser.screenshot(dir, name); + if (!grid) return { file, margin: 0 }; + return { file, margin: applyGrid(session, file) }; +} + +// Assembles the per-turn payload: frame, trimmed state, and any coordinate +// caveat the model needs in order to aim correctly. +function frameResult({ file, margin, state, scale, fullRes, region, label, extra = [] }) { + const { blocks, image } = frameBlocks(file, { scale, fullRes, region, label }); + const notes = [stateText(state)]; + const caveat = gridNote(image, margin); + if (caveat) notes.push(caveat); + for (const note of extra) if (note) notes.push(note); + return { + content: [...blocks, textBlock(notes.join('\n'))], + image, + }; +} + +module.exports = { + applyGrid, + captureFrame, + frameResult, + overlayMargin, +}; diff --git a/mcp/src/image.js b/mcp/src/image.js new file mode 100644 index 0000000..5ddfe62 --- /dev/null +++ b/mcp/src/image.js @@ -0,0 +1,108 @@ +'use strict'; + +const fs = require('fs'); +const { PNG } = require('pngjs'); + +// Screenshots are the dominant context cost for an agent playing a game: a +// 1280x720 PNG is roughly 1200 tokens. Frames are downscaled by default so a +// long navigation stays affordable, and only widened on explicit request. +const DEFAULT_SCALE = 0.5; + +function readPng(file) { + return PNG.sync.read(fs.readFileSync(file)); +} + +function clampScale(scale) { + const value = Number(scale); + if (!Number.isFinite(value) || value <= 0) return DEFAULT_SCALE; + return Math.min(1, value); +} + +// Box filter. Averaging over the source rectangle keeps thin game sprites and +// small UI text legible, which nearest-neighbour sampling loses. +function resize(source, scale) { + const ratio = clampScale(scale); + if (ratio === 1) return source; + const width = Math.max(1, Math.round(source.width * ratio)); + const height = Math.max(1, Math.round(source.height * ratio)); + const target = new PNG({ width, height }); + + for (let y = 0; y < height; y += 1) { + const topEdge = Math.floor((y * source.height) / height); + const bottomEdge = Math.max(topEdge + 1, Math.floor(((y + 1) * source.height) / height)); + for (let x = 0; x < width; x += 1) { + const leftEdge = Math.floor((x * source.width) / width); + const rightEdge = Math.max(leftEdge + 1, Math.floor(((x + 1) * source.width) / width)); + let red = 0; + let green = 0; + let blue = 0; + let samples = 0; + for (let sourceY = topEdge; sourceY < bottomEdge; sourceY += 1) { + for (let sourceX = leftEdge; sourceX < rightEdge; sourceX += 1) { + const index = (source.width * sourceY + sourceX) << 2; + red += source.data[index]; + green += source.data[index + 1]; + blue += source.data[index + 2]; + samples += 1; + } + } + const out = (width * y + x) << 2; + target.data[out] = Math.round(red / samples); + target.data[out + 1] = Math.round(green / samples); + target.data[out + 2] = Math.round(blue / samples); + target.data[out + 3] = 255; + } + } + return target; +} + +// Clamped so a model-supplied region can never throw; an out-of-bounds ask +// yields the nearest valid rectangle instead of failing the turn. +function crop(source, region) { + const x = Math.max(0, Math.min(Math.round(Number(region.x) || 0), source.width - 1)); + const y = Math.max(0, Math.min(Math.round(Number(region.y) || 0), source.height - 1)); + const width = Math.max(1, Math.min(Math.round(Number(region.width) || 0), source.width - x)); + const height = Math.max(1, Math.min(Math.round(Number(region.height) || 0), source.height - y)); + const target = new PNG({ width, height }); + PNG.bitblt(source, target, x, y, width, height, 0, 0); + return { png: target, region: { x, y, width, height } }; +} + +function encode(png) { + return PNG.sync.write(png).toString('base64'); +} + +// Reads a screenshot off disk and returns an MCP image content block plus the +// dimensions actually sent, so a caller can map coordinates back if needed. +function imageBlock(file, options = {}) { + let png = readPng(file); + let region = null; + if (options.region) { + const cropped = crop(png, options.region); + png = cropped.png; + region = cropped.region; + } + const scale = options.fullRes ? 1 : clampScale(options.scale ?? DEFAULT_SCALE); + const sourceWidth = png.width; + const sourceHeight = png.height; + png = resize(png, scale); + return { + block: { type: 'image', data: encode(png), mimeType: 'image/png' }, + width: png.width, + height: png.height, + sourceWidth, + sourceHeight, + scale, + region, + }; +} + +module.exports = { + DEFAULT_SCALE, + clampScale, + crop, + encode, + imageBlock, + readPng, + resize, +}; diff --git a/mcp/src/registry.js b/mcp/src/registry.js new file mode 100644 index 0000000..1e6f74b --- /dev/null +++ b/mcp/src/registry.js @@ -0,0 +1,60 @@ +'use strict'; + +const { Session, newSessionId } = require('./session'); + +class SessionRegistry { + constructor({ workspace }) { + this.workspace = workspace; + this.sessions = new Map(); + } + + async create(options = {}) { + const id = options.sessionId ? String(options.sessionId) : newSessionId(); + if (this.sessions.has(id)) throw new Error(`session ${id} already exists`); + const session = new Session({ id, workspace: this.workspace, options }); + this.sessions.set(id, session); + try { + await session.start(); + } catch (error) { + this.sessions.delete(id); + // A game process or Chromium may already be up even though start threw. + await session.close().catch(() => {}); + throw error; + } + return session; + } + + get(id) { + const session = this.sessions.get(String(id)); + if (!session) { + const known = [...this.sessions.keys()]; + const hint = known.length ? ` known sessions: ${known.join(', ')}` : ' no sessions are running'; + throw new Error(`unknown session_id "${id}".${hint}`); + } + return session; + } + + async end(id) { + const session = this.get(id); + const summary = session.summary(); + await session.close(); + this.sessions.delete(session.id); + return { ...summary, closed: true }; + } + + list() { + return [...this.sessions.values()].map((session) => session.summary()); + } + + // Chromium and any spawned game process are detached children; without this + // a server shutdown orphans the whole tree. + async closeAll() { + const closing = [...this.sessions.values()].map((session) => session.close().catch(() => {})); + this.sessions.clear(); + await Promise.all(closing); + } +} + +module.exports = { + SessionRegistry, +}; diff --git a/mcp/src/result.js b/mcp/src/result.js new file mode 100644 index 0000000..83faefd --- /dev/null +++ b/mcp/src/result.js @@ -0,0 +1,50 @@ +'use strict'; + +const { imageBlock } = require('./image'); +const { compactState } = require('./state'); + +function textBlock(text) { + return { type: 'text', text }; +} + +function errorResult(error) { + return { + isError: true, + content: [textBlock(String((error && error.message) || error))], + }; +} + +// Every frame goes back as an image block for the model and a path for tooling +// that wants the original PNG on disk. +function frameBlocks(file, options = {}) { + const image = imageBlock(file, options); + const scaleNote = image.scale === 1 + ? `${image.width}x${image.height}` + : `${image.width}x${image.height}, downscaled ${image.scale}x from ${image.sourceWidth}x${image.sourceHeight}`; + const label = options.label ? `${options.label} ` : ''; + return { + blocks: [image.block, textBlock(`${label}frame (${scaleNote})\npath: ${file}`)], + image, + }; +} + +function stateText(raw) { + const state = compactState(raw); + return Object.keys(state).length ? `state: ${JSON.stringify(state)}` : 'state: {}'; +} + +// Coordinate space warning matters: with the grid on, the saved PNG is larger +// than the viewport by a margin per side, so pixels read off the image do not +// match pixels sent back as x/y. +function gridNote(image, margin) { + if (!margin) return null; + return `grid overlay is on. The image includes a ${margin}px label margin on every side, so image pixel (px, py) is viewport (px - ${margin}, py - ${margin}). Prefer overlay_row/overlay_col targets while the grid is on.`; +} + +module.exports = { + errorResult, + frameBlocks, + gridNote, + stateText, + textBlock, +}; diff --git a/mcp/src/schema.js b/mcp/src/schema.js new file mode 100644 index 0000000..e51803d --- /dev/null +++ b/mcp/src/schema.js @@ -0,0 +1,112 @@ +'use strict'; + +const { z } = require('zod'); +const { MAX_ACTION_SPAN_MS } = require('../../runwave/protocol/src/action'); +const { DEFAULT_MARK_GRID } = require('../../runwave/protocol/src/mark-grid'); + +// Spans are pulled from the protocol rather than restated, so the tool contract +// cannot drift from what the executor actually enforces. +const span = (type) => (MAX_ACTION_SPAN_MS[type] ? ` Max ${MAX_ACTION_SPAN_MS[type]}ms.` : ''); + +const cell = z + .object({ + overlay_row: z.number().int().min(0).describe(`Grid row, 0-${DEFAULT_MARK_GRID.rows - 1}.`), + overlay_col: z.number().int().min(0).describe(`Grid column, 0-${DEFAULT_MARK_GRID.cols - 1}.`), + }) + .describe('Grid cell target. Resolves to the centre of that cell.'); + +const point = z.object({ + x: z.number().optional().describe('Viewport pixel X. Preferred for precise targets.'), + y: z.number().optional().describe('Viewport pixel Y.'), + overlay_row: z.number().int().min(0).optional(), + overlay_col: z.number().int().min(0).optional(), +}); + +const start = z.number().min(0).describe('Offset in ms from the start of the sequence.'); + +const keyAction = z.object({ + type: z.literal('key'), + start, + end: z.number().min(0).optional().describe('Release offset in ms. Omit for a ~50ms tap. Hold longer to move further.'), + key: z.string().describe('Key name, e.g. ArrowRight, Space, KeyW, Enter. Aliases: left/right/up/down/jump.'), +}); + +const clickAction = z.object({ + type: z.literal('click'), + start, + end: z.number().min(0).optional().describe(`Hold duration.${span('click')}`), + ...point.shape, + button: z.enum(['left', 'middle', 'right']).optional(), + clickCount: z.number().int().min(1).max(3).optional(), +}); + +const multiClickAction = z.object({ + type: z.literal('multi_click'), + start, + ...point.shape, + cells: z.array(cell).max(4).optional().describe('Up to 4 candidate cells; clicks scatter across them.'), + count: z.number().int().min(1).max(20).optional().describe('Number of clicks, default 10.'), + intervalMs: z.number().min(20).max(500).optional(), + button: z.enum(['left', 'middle', 'right']).optional(), +}); + +const dragAction = z.object({ + type: z.literal('drag'), + start, + end: z.number().min(0).optional().describe(`Drag duration.${span('drag')}`), + from: point.describe('Drag origin.'), + to: point.describe('Drag destination.'), + button: z.enum(['left', 'middle', 'right']).optional(), + mode: z.enum(['mouse', 'html5']).optional().describe('mouse for canvas games; html5 only for native draggable elements.'), + steps: z.number().int().min(1).max(80).optional(), +}); + +const cursorMoveAction = z.object({ + type: z.literal('cursor_move'), + start, + end: z.number().min(0).optional().describe(`Move duration.${span('cursor_move')}`), + ...point.shape, + steps: z.number().int().min(1).max(80).optional(), +}); + +const viewMoveAction = z.object({ + type: z.literal('view_move'), + start, + end: z.number().min(0).optional(), + dx: z.number().optional().describe('Relative pointer delta X. Positive is right.'), + dy: z.number().optional().describe('Relative pointer delta Y. Positive is down.'), + steps: z.number().int().min(1).max(80).optional(), +}).describe('Relative mouse movement for pointer-lock/FPS camera control.'); + +const action = z + .discriminatedUnion('type', [ + keyAction, + clickAction, + multiClickAction, + dragAction, + cursorMoveAction, + viewMoveAction, + ]) + .describe('One timed input. Offsets are ms from sequence start; actions may overlap.'); + +const region = z.object({ + x: z.number().min(0), + y: z.number().min(0), + width: z.number().min(1), + height: z.number().min(1), +}); + +module.exports = { + action, + cell, + clickAction, + cursorMoveAction, + dragAction, + keyAction, + multiClickAction, + point, + region, + span, + start, + viewMoveAction, +}; diff --git a/mcp/src/server.js b/mcp/src/server.js new file mode 100644 index 0000000..5ac9ed7 --- /dev/null +++ b/mcp/src/server.js @@ -0,0 +1,50 @@ +'use strict'; + +const os = require('os'); +const path = require('path'); +const { McpServer } = require('@modelcontextprotocol/sdk/server/mcp.js'); +const { SessionRegistry } = require('./registry'); +const { registerAct, registerObserve, registerPlayTools } = require('./tools-play'); +const { registerCapture, registerSessionTools, registerZoom } = require('./tools-aux'); + +const VERSION = '0.1.0'; + +// An MCP server starts in whatever directory the host happens to use, so the +// workspace is resolved explicitly instead of from cwd. This is the same trap +// runwave's paths.js falls into by capturing cwd at require time. +function resolveWorkspace() { + const configured = process.env.RUNWAVE_MCP_WORKSPACE; + if (configured) return path.resolve(configured); + return path.join(os.tmpdir(), 'runwave-mcp'); +} + +function createServer({ workspace = resolveWorkspace() } = {}) { + const server = new McpServer( + { name: 'runwave', version: VERSION }, + { + instructions: [ + 'Play browser games by looking at frames and sending timed input sequences.', + 'Call launch_game once, then loop act/observe, then end_game.', + 'Frames come back downscaled to save context; use zoom to inspect detail and full_res only when you must.', + 'Prefer one act call that commits to a move (hold a key for several hundred ms) over many single taps.', + 'If a result says the frame did not change, the input did not land: change approach rather than repeating it.', + 'Use journal to recall what you have already tried, and capture to save the final screenshot.', + ].join(' '), + } + ); + + const registry = new SessionRegistry({ workspace }); + registerPlayTools(server, registry); + registerObserve(server, registry); + registerAct(server, registry); + registerZoom(server, registry); + registerCapture(server, registry); + registerSessionTools(server, registry); + return { server, registry, workspace }; +} + +module.exports = { + VERSION, + createServer, + resolveWorkspace, +}; diff --git a/mcp/src/session.js b/mcp/src/session.js new file mode 100644 index 0000000..933cd97 --- /dev/null +++ b/mcp/src/session.js @@ -0,0 +1,109 @@ +'use strict'; + +const path = require('path'); +const { createSession } = require('../../runwave/controller/src/session-factory'); +const { OutputWriter } = require('../../runwave/controller/src/output-writer'); +const { ensureDir, timestamp } = require('../../runwave/controller/src/file-utils'); +const { buildSessionConfig } = require('./config'); + +const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; + +class Session { + constructor({ id, workspace, options }) { + this.id = id; + this.config = buildSessionConfig(options); + this.paths = { + runDir: path.join(workspace, 'sessions', id), + outputRoot: path.join(workspace, 'sessions', id, 'output'), + }; + ensureDir(this.paths.runDir); + this.output = new OutputWriter(this.paths.outputRoot); + this.browser = createSession(this.config, this.paths, null); + this.stepIndex = 0; + this.turn = 0; + this.journal = []; + this.closed = false; + this.createdAt = Date.now(); + this.idleTimeoutMs = Number(options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS); + this.idleTimer = null; + // Serializes work per session. One Playwright page, one shared step + // counter: concurrent steps would interleave keypresses and collide on + // output filenames. Subagents sharing a session_id hit this too. + this.queue = Promise.resolve(); + } + + async start() { + await this.browser.start(); + this.touch(); + return this; + } + + // Every tool call runs through here, so ordering is guaranteed even when + // several agents hold the same session id. + run(fn) { + const result = this.queue.then(() => { + if (this.closed) throw new Error(`session ${this.id} has ended`); + return fn(); + }); + this.queue = result.then(() => undefined, () => undefined); + return result; + } + + // An agent that forgets to call end_game would otherwise leak Chromium for + // the lifetime of the server. + touch() { + if (this.idleTimer) clearTimeout(this.idleTimer); + if (!this.idleTimeoutMs || this.closed) return; + this.idleTimer = setTimeout(() => { + this.close().catch(() => {}); + }, this.idleTimeoutMs); + if (typeof this.idleTimer.unref === 'function') this.idleTimer.unref(); + } + + // Append-only text log. This is how an agent re-orients after its context is + // compacted, without paying to replay screenshots. + note(entry) { + this.journal.push({ turn: this.turn, at: Date.now() - this.createdAt, ...entry }); + return this.journal[this.journal.length - 1]; + } + + nextStepIndex() { + this.stepIndex += 1; + this.turn += 1; + return this.stepIndex; + } + + actionDir(name) { + return this.output.actionDir(name); + } + + async close() { + if (this.closed) return; + this.closed = true; + if (this.idleTimer) clearTimeout(this.idleTimer); + await this.browser.close(); + } + + summary() { + return { + session_id: this.id, + url: this.browser.launchUrl, + viewport: this.config.viewport, + turns: this.turn, + steps: this.stepIndex, + uptime_ms: Date.now() - this.createdAt, + run_dir: this.paths.runDir, + closed: this.closed, + }; + } +} + +function newSessionId() { + return `game-${timestamp()}`; +} + +module.exports = { + DEFAULT_IDLE_TIMEOUT_MS, + Session, + newSessionId, +}; diff --git a/mcp/src/state.js b/mcp/src/state.js new file mode 100644 index 0000000..0fa69fa --- /dev/null +++ b/mcp/src/state.js @@ -0,0 +1,46 @@ +'use strict'; + +// Runwave's raw state carries a full WebGL renderer probe and every canvas on +// the page. Useful for a playtest report, mostly noise for an agent deciding a +// next move, and it is paid for on every single turn. Only the fields that +// change a decision survive. +function compactState(raw) { + const generic = (raw && raw.generic) || raw || {}; + const canvases = Array.isArray(generic.canvases) ? generic.canvases : []; + const state = {}; + if (generic.title) state.title = generic.title; + if (generic.url) state.url = generic.url; + + const active = generic.activeElement; + if (active && active.tagName && active.tagName !== 'BODY') { + state.focus = [active.tagName, active.id ? `#${active.id}` : ''].filter(Boolean).join(''); + } + if (generic.pointerLockElement && generic.pointerLockElement.tagName) { + state.pointer_locked = true; + } + + // The largest canvas is almost always the game surface. Its client rect tells + // an agent which part of the viewport is actually playable. + const surface = canvases + .filter((canvas) => canvas && canvas.clientWidth > 0 && canvas.clientHeight > 0) + .sort((left, right) => right.clientWidth * right.clientHeight - left.clientWidth * left.clientHeight)[0]; + if (surface) { + state.game_area = { + x: Math.round(surface.left ?? surface.x ?? 0), + y: Math.round(surface.top ?? surface.y ?? 0), + width: Math.round(surface.clientWidth), + height: Math.round(surface.clientHeight), + }; + } + if (canvases.length > 1) state.canvas_count = canvases.length; + + // A stateExpression is opt-in and game-specific, so whatever it returns is + // assumed relevant and passed through intact. + if (raw && raw.custom !== undefined) state.custom = raw.custom; + if (raw && raw.customError) state.custom_error = String(raw.customError).slice(0, 300); + return state; +} + +module.exports = { + compactState, +}; diff --git a/mcp/src/tools-aux.js b/mcp/src/tools-aux.js new file mode 100644 index 0000000..be1fcf8 --- /dev/null +++ b/mcp/src/tools-aux.js @@ -0,0 +1,140 @@ +'use strict'; + +const { z } = require('zod'); +const { region } = require('./schema'); +const { captureFrame, frameResult } = require('./frame'); +const { errorResult, textBlock } = require('./result'); + +function registerZoom(server, registry) { + server.registerTool('zoom', { + title: 'Zoom', + description: 'Screenshot a rectangle of the viewport at full resolution. Use this to read small UI or confirm a target before clicking, instead of paying for a full-resolution frame.', + inputSchema: { + session_id: z.string(), + region: region.describe('Viewport rectangle in real pixels.'), + }, + }, async (args) => { + try { + const session = registry.get(args.session_id); + return session.run(async () => { + const name = `zoom-${String(session.turn).padStart(3, '0')}`; + const { file } = await captureFrame(session, { name, grid: false }); + const state = await session.browser.state(session.config.stateExpression); + const result = frameResult({ + file, margin: 0, state, fullRes: true, region: args.region, + label: `zoom ${args.region.width}x${args.region.height} at (${args.region.x},${args.region.y})`, + }); + session.touch(); + return result; + }); + } catch (error) { + return errorResult(error); + } + }); +} + +function registerCapture(server, registry) { + server.registerTool('capture', { + title: 'Capture deliverable', + description: 'Save a clean, full-resolution, un-annotated screenshot to disk and return its path. Use this for the final artifact once the target is reached.', + inputSchema: { + session_id: z.string(), + name: z.string().describe('File label, e.g. "target-reached".'), + preview: z.boolean().optional().describe('Also return a downscaled preview to confirm what was saved.'), + }, + }, async (args) => { + try { + const session = registry.get(args.session_id); + return session.run(async () => { + const { file } = await captureFrame(session, { name: `capture-${args.name}`, grid: false }); + session.note({ event: 'capture', name: args.name, path: file }); + session.touch(); + const summary = textBlock(`saved ${session.config.viewport.width}x${session.config.viewport.height} clean capture\npath: ${file}`); + if (!args.preview) return { content: [summary] }; + const state = await session.browser.state(session.config.stateExpression); + const preview = frameResult({ file, margin: 0, state, label: 'saved' }); + return { content: [summary, ...preview.content] }; + }); + } catch (error) { + return errorResult(error); + } + }); +} + +function registerSessionTools(server, registry) { + server.registerTool('reset_game', { + title: 'Reset game', + description: 'Reload the game at its launch URL. Use this when stuck or to start a fresh attempt.', + inputSchema: { session_id: z.string() }, + }, async (args) => { + try { + const session = registry.get(args.session_id); + return session.run(async () => { + await session.browser.navigate({ url: session.browser.launchUrl }); + session.stepIndex = 0; + session.note({ event: 'reset' }); + const { file } = await captureFrame(session, { name: `reset-${session.turn}`, grid: false }); + session.lastFrame = file; + const state = await session.browser.state(session.config.stateExpression); + const result = frameResult({ file, margin: 0, state, label: 'after reset' }); + session.touch(); + return result; + }); + } catch (error) { + return errorResult(error); + } + }); + + // Cheap way back into context after a compaction: text only, no frames. + server.registerTool('journal', { + title: 'Journal', + description: 'Read the log of what has been tried this session. Use this to re-orient without replaying screenshots.', + inputSchema: { + session_id: z.string(), + limit: z.number().int().positive().max(200).optional().describe('Most recent entries to return. Default 40.'), + }, + }, async (args) => { + try { + const session = registry.get(args.session_id); + const limit = args.limit ?? 40; + const entries = session.journal.slice(-limit); + const lines = entries.map((entry) => { + const seconds = (entry.at / 1000).toFixed(1); + const rest = Object.entries(entry) + .filter(([key]) => !['turn', 'at', 'event'].includes(key)) + .map(([key, value]) => `${key}=${Array.isArray(value) ? value.join('+') : value}`) + .join(' '); + return `[${seconds}s] turn ${entry.turn} ${entry.event}${rest ? ` ${rest}` : ''}`; + }); + const header = `${session.journal.length} entries, showing last ${entries.length}`; + return { content: [textBlock([header, ...lines].join('\n'))] }; + } catch (error) { + return errorResult(error); + } + }); + + server.registerTool('list_sessions', { + title: 'List sessions', + description: 'List running game sessions.', + inputSchema: {}, + }, async () => { + const sessions = registry.list(); + if (!sessions.length) return { content: [textBlock('no sessions running')] }; + return { content: [textBlock(JSON.stringify(sessions, null, 2))] }; + }); + + server.registerTool('end_game', { + title: 'End game', + description: 'Close the browser and stop the game process. Always call this when finished.', + inputSchema: { session_id: z.string() }, + }, async (args) => { + try { + const summary = await registry.end(args.session_id); + return { content: [textBlock(JSON.stringify(summary, null, 2))] }; + } catch (error) { + return errorResult(error); + } + }); +} + +module.exports = { registerCapture, registerSessionTools, registerZoom }; diff --git a/mcp/src/tools-play.js b/mcp/src/tools-play.js new file mode 100644 index 0000000..511ce0c --- /dev/null +++ b/mcp/src/tools-play.js @@ -0,0 +1,178 @@ +'use strict'; + +const { z } = require('zod'); +const { runStep } = require('../../runwave/controller/src/step-runner'); +const { action } = require('./schema'); +const { captureFrame, frameResult } = require('./frame'); +const { errorResult } = require('./result'); +const { changedSince } = require('./diff'); +const { applyGrid } = require('./frame'); + +// Returning several frames from one turn is occasionally worth it to see a +// trajectory, but each one costs context, so the count is capped. +const MAX_FRAMES_PER_TURN = 4; + +const frameOptions = { + full_res: z.boolean().optional().describe('Return the frame at full resolution. Costs ~4x the context of the default.'), + grid: z.boolean().optional().describe('Overlay a labelled row/column grid to help aim. Adds a label margin around the image.'), +}; + +function registerPlayTools(server, registry) { + server.registerTool('launch_game', { + title: 'Launch game', + description: 'Start a headless browser game session and return the first frame. Provide either url, or game_dir plus port for a directory containing start.sh.', + inputSchema: { + url: z.string().optional().describe('URL to open, e.g. http://127.0.0.1:3000/'), + game_dir: z.string().optional().describe('Directory containing start.sh. Launched with the given port.'), + port: z.number().int().positive().optional().describe('Port for game_dir, also used to build the URL.'), + viewport: z.object({ width: z.number().int().positive(), height: z.number().int().positive() }).optional(), + session_id: z.string().optional().describe('Reuse a specific id. Generated when omitted.'), + state_expression: z.string().optional().describe('JS expression evaluated in the page each turn for game-specific state.'), + ...frameOptions, + }, + }, async (args) => { + try { + if (!args.url && !args.game_dir) throw new Error('launch_game requires url or game_dir'); + const session = await registry.create({ + url: args.url, + gameDir: args.game_dir, + port: args.port, + viewport: args.viewport, + sessionId: args.session_id, + stateExpression: args.state_expression, + }); + return session.run(async () => { + const { file, margin } = await captureFrame(session, { name: 'launch', grid: args.grid }); + session.lastFrame = file; + session.note({ event: 'launch', url: session.browser.launchUrl }); + const state = await session.browser.state(session.config.stateExpression); + const result = frameResult({ + file, margin, state, fullRes: args.full_res, label: 'initial', + extra: [`session_id: ${session.id}`, `viewport: ${session.config.viewport.width}x${session.config.viewport.height}`], + }); + session.touch(); + return result; + }); + } catch (error) { + return errorResult(error); + } + }); +} + +// runStep writes clean captures. The grid, when asked for, is applied afterwards +// to the frames actually being returned, so the on-disk originals stay usable. +function actResult({ session, step, args, previousFrame, actionName }) { + const captures = Array.isArray(step.captures) ? step.captures : []; + if (!captures.length) throw new Error('step produced no frames'); + const wanted = captures.slice(-MAX_FRAMES_PER_TURN); + const finalCapture = wanted[wanted.length - 1]; + session.lastFrame = finalCapture.path; + + const changed = changedSince(previousFrame, finalCapture.path); + session.note({ + event: 'act', + action: actionName, + duration_ms: step.duration, + inputs: step.actions.map((item) => item.type === 'key' ? item.key : item.type), + changed, + ...(args.note ? { intent: args.note } : {}), + }); + + const content = []; + for (const capture of wanted.slice(0, -1)) { + const frame = frameResult({ + file: capture.path, margin: 0, state: capture.state, + fullRes: args.full_res, label: `t=${capture.at}ms`, + }); + content.push(...frame.content); + } + const last = frameResult({ + file: finalCapture.path, + margin: args.grid ? applyGrid(session, finalCapture.path) : 0, + state: step.endState, + fullRes: args.full_res, + label: wanted.length > 1 ? `t=${finalCapture.at}ms` : null, + extra: [ + `sequence ran ${step.duration}ms`, + changed === false + ? 'frame is byte-identical to the previous one: the input probably did not reach the game. Check focus, try a different key, or hold it longer.' + : null, + ], + }); + session.touch(); + return { content: [...content, ...last.content] }; +} + +function registerObserve(server, registry) { + server.registerTool('observe', { + title: 'Observe', + description: 'Take a fresh screenshot and read game state without sending any input.', + inputSchema: { + session_id: z.string(), + ...frameOptions, + }, + }, async (args) => { + try { + const session = registry.get(args.session_id); + return session.run(async () => { + const name = `observe-${String(session.turn).padStart(3, '0')}`; + const { file, margin } = await captureFrame(session, { name, grid: args.grid }); + const state = await session.browser.state(session.config.stateExpression); + session.lastFrame = file; + const result = frameResult({ file, margin, state, fullRes: args.full_res }); + session.touch(); + return result; + }); + } catch (error) { + return errorResult(error); + } + }); +} + +function registerAct(server, registry) { + server.registerTool('act', { + title: 'Act', + description: [ + 'Send a timed sequence of inputs, then return the resulting frame.', + 'Offsets are milliseconds from the start of the sequence and actions may overlap, so one call can express "hold right for 900ms and jump at 150ms".', + 'This is the main way to play: prefer one sequence that commits to a move over many single-input calls.', + ].join(' '), + inputSchema: { + session_id: z.string(), + actions: z.array(action).min(1).describe('Inputs to run. Total sequence should stay under 8000ms.'), + captures: z.array(z.number().min(0)).max(MAX_FRAMES_PER_TURN).optional() + .describe('Offsets in ms to screenshot at. Defaults to the end of the sequence. Each extra frame costs context.'), + note: z.string().optional().describe('Short intent for the journal, e.g. "cross bridge east".'), + ...frameOptions, + }, + }, async (args) => { + try { + const session = registry.get(args.session_id); + return session.run(async () => { + const stepIndex = session.nextStepIndex(); + const actionName = `act-${String(stepIndex).padStart(3, '0')}`; + const previousFrame = session.lastFrame; + const step = await runStep({ + input: { + action: 'step', + action_name: actionName, + actions: args.actions, + ...(args.captures ? { captures: args.captures } : {}), + autoCaptures: false, + }, + config: session.config, + browser: session.browser, + outputDir: session.actionDir(actionName), + nextStepIndex: stepIndex, + actionName, + profiler: null, + }); + return actResult({ session, step, args, previousFrame, actionName }); + }); + } catch (error) { + return errorResult(error); + } + }); +} + +module.exports = { MAX_FRAMES_PER_TURN, frameOptions, registerAct, registerObserve, registerPlayTools }; diff --git a/mcp/test/fixtures/game/index.html b/mcp/test/fixtures/game/index.html new file mode 100644 index 0000000..acaf3de --- /dev/null +++ b/mcp/test/fixtures/game/index.html @@ -0,0 +1,62 @@ + + + + + MCP Test Game + + + + + + + diff --git a/mcp/test/integration.test.js b/mcp/test/integration.test.js new file mode 100644 index 0000000..b34bdf8 --- /dev/null +++ b/mcp/test/integration.test.js @@ -0,0 +1,131 @@ +'use strict'; + +// Drives a real headless Chromium against a real game page through the MCP +// tool surface. Skipped automatically when Chromium's system libraries are +// missing, so the suite still runs on a bare machine. + +const assert = require('node:assert/strict'); +const fsp = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { pathToFileURL } = require('node:url'); + +const { chromium } = require('playwright'); +const { SessionRegistry } = require('../src/registry'); +const { captureFrame } = require('../src/frame'); +const { changedSince } = require('../src/diff'); +const { compactState } = require('../src/state'); +const { imageBlock } = require('../src/image'); +const { runStep } = require('../../runwave/controller/src/step-runner'); + +const GAME_URL = pathToFileURL(path.join(__dirname, 'fixtures', 'game', 'index.html')).href; +const VIEWPORT = { width: 640, height: 360 }; + +async function chromiumUsable() { + try { + const browser = await chromium.launch({ headless: true }); + await browser.close(); + return true; + } catch { + return false; + } +} + +test('MCP session plays a real browser game', async (t) => { + if (!(await chromiumUsable())) { + t.skip('chromium cannot launch here; run "npx playwright install-deps chromium"'); + return; + } + const workspace = await fsp.mkdtemp(path.join(os.tmpdir(), 'runwave-mcp-it-')); + const registry = new SessionRegistry({ workspace }); + t.after(async () => { + await registry.closeAll(); + await fsp.rm(workspace, { recursive: true, force: true }); + }); + + const session = await registry.create({ + url: GAME_URL, + viewport: VIEWPORT, + stateExpression: '() => window.gameState', + }); + + await t.test('launch returns a usable downscaled frame', async () => { + const { file } = await captureFrame(session, { name: 'launch', grid: false }); + session.lastFrame = file; + const image = imageBlock(file); + assert.equal(image.sourceWidth, VIEWPORT.width, 'capture must match the viewport exactly'); + assert.equal(image.width, VIEWPORT.width / 2, 'frames are halved by default to save context'); + assert.ok(image.block.data.length > 100, 'image block must carry real base64 payload'); + assert.equal(image.block.mimeType, 'image/png'); + }); + + await t.test('state exposes the game canvas and custom state', async () => { + const state = compactState(await session.browser.state(session.config.stateExpression)); + assert.deepEqual(state.game_area, { x: 0, y: 0, width: VIEWPORT.width, height: VIEWPORT.height }); + assert.equal(state.custom.hits, 0); + assert.equal(state.webgl, undefined, 'renderer probe must be stripped from per-turn state'); + }); + + await t.test('a held key moves the player and the frame changes', async () => { + const before = await session.browser.state(session.config.stateExpression); + const previousFrame = session.lastFrame; + const stepIndex = session.nextStepIndex(); + const actionName = `act-${stepIndex}`; + const step = await runStep({ + input: { + action: 'step', + action_name: actionName, + actions: [{ type: 'key', start: 0, end: 600, key: 'ArrowRight' }], + autoCaptures: false, + }, + config: session.config, + browser: session.browser, + outputDir: session.actionDir(actionName), + nextStepIndex: stepIndex, + actionName, + profiler: null, + }); + + assert.equal(step.captures.length, 1, 'one frame per turn unless more are requested'); + const after = step.endState; + assert.ok( + after.custom.x > before.custom.x + 20, + `holding right must move the player: ${before.custom.x} -> ${after.custom.x}` + ); + assert.equal(changedSince(previousFrame, step.captures[0].path), true); + session.lastFrame = step.captures[0].path; + }); + + await t.test('a grid cell click lands on the intended target', async () => { + // The target sits at x 480-560, y 140-220 in a 640x360 viewport. On a 16x16 + // grid that is columns 12-13 and rows 6-9, so cell (7, 12) must hit it. + const stepIndex = session.nextStepIndex(); + const actionName = `act-${stepIndex}`; + const step = await runStep({ + input: { + action: 'step', + action_name: actionName, + actions: [{ type: 'click', start: 50, overlay_row: 7, overlay_col: 12 }], + autoCaptures: false, + }, + config: session.config, + browser: session.browser, + outputDir: session.actionDir(actionName), + nextStepIndex: stepIndex, + actionName, + profiler: null, + }); + + // Cell centre for (row 7, col 12) on a 16x16 grid over 640x360. + const expected = { + x: Math.round((12 + 0.5) * (VIEWPORT.width / 16)), + y: Math.round((7 + 0.5) * (VIEWPORT.height / 16)), + }; + const click = step.actions.find((item) => item.type === 'click'); + assert.deepEqual({ x: click.x, y: click.y }, expected, 'cell must resolve to its centre'); + assert.ok(click.x >= 480 && click.x <= 560 && click.y >= 140 && click.y <= 220, 'centre must fall inside the target'); + assert.equal(step.endState.custom.hits, 1, 'the click must register on the target'); + assert.equal(step.endState.custom.lit, true); + }); +}); diff --git a/mcp/test/unit.test.js b/mcp/test/unit.test.js new file mode 100644 index 0000000..721d4da --- /dev/null +++ b/mcp/test/unit.test.js @@ -0,0 +1,143 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { PNG } = require('pngjs'); + +const { buildSessionConfig, normalizeViewport } = require('../src/config'); +const { clampScale, crop, resize } = require('../src/image'); +const { compactState } = require('../src/state'); +const { action } = require('../src/schema'); +const { normalizeActions } = require('../../runwave/controller/src/action-normalizer'); + +function solidPng(width, height, color = [10, 20, 30]) { + const png = new PNG({ width, height }); + for (let i = 0; i < width * height; i += 1) { + const idx = i << 2; + png.data[idx] = color[0]; + png.data[idx + 1] = color[1]; + png.data[idx + 2] = color[2]; + png.data[idx + 3] = 255; + } + return png; +} + +test('session config disables recording and grid overlay by default', () => { + const config = buildSessionConfig({ url: 'http://127.0.0.1:1/' }); + assert.equal(config.record, false, 'recording must be off so gstreamer is never required'); + assert.equal(config.headless, true); + assert.equal(config.gridScreenshots, false, 'overlay must be opt-in to keep image and input coordinates aligned'); + assert.equal(config.autoCaptures, false); +}); + +test('session config always carries a numeric viewport so grid cells resolve', () => { + // The daemon passes raw CLI input through, which is why cell actions fail + // there when no viewport was given. Building the config must close that gap. + const config = buildSessionConfig({ url: 'http://127.0.0.1:1/' }); + assert.equal(typeof config.viewport.width, 'number'); + assert.ok(config.viewport.width > 0 && config.viewport.height > 0); + const [click] = normalizeActions( + [{ type: 'click', start: 0, overlay_row: 2, overlay_col: 3 }], + 500, + { strict: true, config, aliases: {}, roundPoints: true } + ); + assert.equal(typeof click.x, 'number'); + assert.equal(typeof click.y, 'number'); +}); + +test('grid cell targets resolve to a stable point so traces replay identically', () => { + const config = buildSessionConfig({ viewport: { width: 1280, height: 720 } }); + const points = new Set(); + for (let i = 0; i < 50; i += 1) { + const [click] = normalizeActions( + [{ type: 'click', start: 0, overlay_row: 6, overlay_col: 7 }], + 500, + { strict: true, config, aliases: {}, roundPoints: true } + ); + points.add(`${click.x},${click.y}`); + } + assert.equal(points.size, 1, `expected one deterministic point, got ${[...points].join(' ')}`); +}); + +test('playtest scatter is preserved when sample mode is not set', () => { + const config = buildSessionConfig({ viewport: { width: 1280, height: 720 } }); + delete config.markGridSampleMode; + const points = new Set(); + for (let i = 0; i < 80; i += 1) { + const [click] = normalizeActions( + [{ type: 'click', start: 0, overlay_row: 6, overlay_col: 7 }], + 500, + { strict: true, config, aliases: {}, roundPoints: true } + ); + points.add(`${click.x},${click.y}`); + } + assert.ok(points.size > 5, 'default runwave behaviour must remain random'); +}); + +test('normalizeViewport falls back on invalid input', () => { + assert.deepEqual(normalizeViewport({ width: 0, height: -4 }), { width: 1280, height: 720 }); + assert.deepEqual(normalizeViewport({ width: 800, height: 600 }), { width: 800, height: 600 }); +}); + +test('resize halves dimensions and clamps scale above one', () => { + const png = solidPng(100, 50); + const half = resize(png, 0.5); + assert.equal(half.width, 50); + assert.equal(half.height, 25); + assert.equal(clampScale(4), 1); + assert.equal(resize(png, 1).width, 100); +}); + +test('resize preserves colour when downscaling a solid image', () => { + const png = solidPng(40, 40, [200, 100, 50]); + const small = resize(png, 0.25); + assert.deepEqual([small.data[0], small.data[1], small.data[2]], [200, 100, 50]); +}); + +test('crop clamps an out-of-bounds region instead of throwing', () => { + const png = solidPng(100, 100); + const { region } = crop(png, { x: 90, y: 90, width: 400, height: 400 }); + assert.deepEqual(region, { x: 90, y: 90, width: 10, height: 10 }); +}); + +test('compactState keeps the largest canvas as the game area and drops noise', () => { + const state = compactState({ + generic: { + title: 'Game', + url: 'http://x/', + activeElement: { tagName: 'BODY' }, + webgl: { renderer: 'SwiftShader', vendor: 'Google', supported: true }, + canvases: [ + { clientWidth: 10, clientHeight: 10, left: 0, top: 0 }, + { clientWidth: 640, clientHeight: 360, left: 20, top: 30 }, + ], + }, + }); + assert.deepEqual(state.game_area, { x: 20, y: 30, width: 640, height: 360 }); + assert.equal(state.canvas_count, 2); + assert.equal(state.webgl, undefined, 'renderer probe is per-turn noise for a player'); + assert.equal(state.focus, undefined, 'a BODY focus carries no signal'); +}); + +test('compactState surfaces custom state and errors from a stateExpression', () => { + assert.equal(compactState({ generic: {}, custom: { score: 7 } }).custom.score, 7); + assert.match(compactState({ generic: {}, customError: 'boom' }).custom_error, /boom/); +}); + +test('act schema accepts every action type the executor implements', () => { + const cases = [ + { type: 'key', start: 0, end: 900, key: 'ArrowRight' }, + { type: 'click', start: 100, x: 10, y: 20 }, + { type: 'click', start: 100, overlay_row: 6, overlay_col: 7 }, + { type: 'multi_click', start: 0, cells: [{ overlay_row: 1, overlay_col: 1 }], count: 5 }, + { type: 'drag', start: 0, end: 500, from: { x: 1, y: 2 }, to: { x: 3, y: 4 }, mode: 'mouse' }, + { type: 'cursor_move', start: 0, x: 5, y: 5, steps: 8 }, + { type: 'view_move', start: 0, end: 400, dx: 120, dy: -20 }, + ]; + for (const item of cases) assert.doesNotThrow(() => action.parse(item), `failed: ${item.type}`); +}); + +test('act schema rejects unknown action types and negative offsets', () => { + assert.throws(() => action.parse({ type: 'scroll', start: 0 })); + assert.throws(() => action.parse({ type: 'key', start: -5, key: 'a' })); +}); diff --git a/package-lock.json b/package-lock.json index a78e6e4..79d0119 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,17 +9,508 @@ "version": "0.1.0", "license": "UNLICENSED", "dependencies": { + "@modelcontextprotocol/sdk": "1.30.0", "playwright": "1.61.1", - "pngjs": "7.0.0" + "pngjs": "7.0.0", + "zod": "3.25.76" }, "bin": { "runwave": "runwave/cli.js", - "runwave-controller": "runwave/controller.js" + "runwave-controller": "runwave/controller.js", + "runwave-mcp": "mcp/bin/runwave-mcp.js" }, "engines": { "node": ">=20" } }, + "node_modules/@hono/node-server": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.0.tgz", + "integrity": "sha512-XovyyCCnBzW+zKu+z/zq8hwNs4KOR5rEMAOxo2f40Q5xoOI37IMm6MIg2COOUtUApo0i6850MTBKH2u4QLGIqg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz", + "integrity": "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9 || ^2.0.5", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.0.tgz", + "integrity": "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "8.6.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.6.1.tgz", + "integrity": "sha512-0D493aP61w0TJ2A0wy27riRsO7FMQ7FK+KUHOKCSfPvYo0R55aiC6emCVgFUeShH0fq0ICPVzNcgoS+BsbXQCA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", + "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -34,6 +525,343 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hono": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.0.tgz", + "integrity": "sha512-jhunvfHWxd7J5EFfSgH4xsYJzSe/lfqbUCxiyyeaQasUsXeEHXtzVid+7EOGByc5JnFa23SSFL3Y2RV/z1T+eQ==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz", + "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.8", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.8.tgz", + "integrity": "sha512-Bsdjwm3Qsd/P0jR+BHDe3LytDfY7WBq2HmCCLIwuVRHMuEC9ae7/R474GIUdF1NgCyZjzVo/A9DOiOBtXq8ZoQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/playwright": { "version": "1.61.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", @@ -72,6 +900,344 @@ "engines": { "node": ">=14.19.0" } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } } } } diff --git a/package.json b/package.json index 1a1ed5e..ec7d693 100644 --- a/package.json +++ b/package.json @@ -7,13 +7,14 @@ "main": "runwave/index.js", "bin": { "runwave": "runwave/cli.js", - "runwave-controller": "runwave/controller.js" + "runwave-controller": "runwave/controller.js", + "runwave-mcp": "mcp/bin/runwave-mcp.js" }, "scripts": { "cli": "node runwave/cli.js", "controller": "node runwave/controller.js", "smoke": "npm run test:smoke", - "test": "node --test runwave/protocol/test/*.test.js runwave/agent/test/*.test.js runwave/controller/test/*.test.js stress-test/test/*.test.js runwave/test/*.test.js", + "test": "node --test runwave/protocol/test/*.test.js runwave/agent/test/*.test.js runwave/controller/test/*.test.js stress-test/test/*.test.js runwave/test/*.test.js mcp/test/*.test.js", "test:agent": "node --test runwave/agent/test/*.test.js", "test:all": "npm test && npm run test:py", "test:controller": "node --test runwave/controller/test/*.test.js", @@ -21,6 +22,7 @@ "test:integration": "npm run test:runwave", "test:runwave": "node --test runwave/test/*.test.js", "test:stress-test": "node --test stress-test/test/*.test.js", + "test:mcp": "node --test mcp/test/*.test.js", "test:smoke": "node --test runwave/test/smoke.test.js", "test:py": "PYTHONWARNINGS=error python3 -m unittest discover -s runwavepy/tests -t runwavepy" }, @@ -33,7 +35,9 @@ }, "license": "UNLICENSED", "dependencies": { + "@modelcontextprotocol/sdk": "1.30.0", "playwright": "1.61.1", - "pngjs": "7.0.0" + "pngjs": "7.0.0", + "zod": "3.25.76" } } diff --git a/runwave/controller/src/action-normalizer.js b/runwave/controller/src/action-normalizer.js index 388499b..a9548f8 100644 --- a/runwave/controller/src/action-normalizer.js +++ b/runwave/controller/src/action-normalizer.js @@ -4,6 +4,7 @@ const { cellsFromObject, clickBurstTimes, gridSafeSampleRatio, + gridSampleMode, markGridFromConfig, randomPointInCells, viewportFromConfig, @@ -167,7 +168,8 @@ function normalizePoint(point, label, options) { viewport, grid, Math.random, - gridSafeSampleRatio(options.config || {}) + gridSafeSampleRatio(options.config || {}), + gridSampleMode(options.config || {}) ); } catch (error) { if (options.strict) throw new Error(`${label} ${error.message}`); diff --git a/runwave/protocol/src/mark-grid.js b/runwave/protocol/src/mark-grid.js index 57640c4..f657aca 100644 --- a/runwave/protocol/src/mark-grid.js +++ b/runwave/protocol/src/mark-grid.js @@ -17,6 +17,12 @@ function viewportFromConfig(config = {}) { return config.viewport || config.videoSize || null; } +// Defaults to the historical scatter so existing playtest behaviour is unchanged. +function gridSampleMode(config = {}) { + const raw = String(config.markGridSampleMode ?? config.gridSampleMode ?? 'random').toLowerCase(); + return raw === 'center' || raw === 'centre' ? 'center' : 'random'; +} + function gridSafeSampleRatio(config = {}) { const raw = Number( config.markGridSafeSampleRatio @@ -91,7 +97,8 @@ function randomPointInCells( viewport, grid = DEFAULT_MARK_GRID, rng = Math.random, - safeSampleRatio = DEFAULT_GRID_SAFE_SAMPLE_RATIO + safeSampleRatio = DEFAULT_GRID_SAFE_SAMPLE_RATIO, + sampleMode = 'random' ) { const normalized = normalizeCellList(cells, grid, 4); if (!normalized.length) { @@ -99,6 +106,16 @@ function randomPointInCells( } const cell = normalized[Math.floor(rng() * normalized.length)]; const bounds = cellBounds(cell, viewport, grid); + // Scattering within a cell varies footage for playtest recordings. An agent + // aiming at a specific target instead needs the same cell to mean the same + // pixel every time, so a saved action trace replays identically. + if (sampleMode === 'center') { + return { + x: Math.max(0, Math.min(Math.round((bounds.left + bounds.right) / 2), Math.round(Number(viewport.width)) - 1)), + y: Math.max(0, Math.min(Math.round((bounds.top + bounds.bottom) / 2), Math.round(Number(viewport.height)) - 1)), + cells: normalized, + }; + } const ratio = Number.isFinite(Number(safeSampleRatio)) && Number(safeSampleRatio) > 0 && Number(safeSampleRatio) <= 1 @@ -131,6 +148,7 @@ function clickBurstTimes(at, duration, count = 10, intervalMs = 100) { module.exports = { DEFAULT_GRID_SAFE_SAMPLE_RATIO, DEFAULT_MARK_GRID, + gridSampleMode, gridSafeSampleRatio, markGridFromConfig, viewportFromConfig, From 814d83263feacd236e78efb49a15aedb6bde9069 Mon Sep 17 00:00:00 2001 From: Vaibhav

Date: Mon, 17 Aug 2026 22:19:29 +0000 Subject: [PATCH 2/2] Add MCP pause control and clean playthrough replay --- mcp/README.md | 96 ++++++- mcp/src/config.js | 7 + mcp/src/replay.js | 124 ++++++++ mcp/src/result.js | 8 + mcp/src/schema.js | 5 +- mcp/src/server.js | 10 +- mcp/src/session.js | 173 +++++++++++ mcp/src/state.js | 4 +- mcp/src/tools-aux.js | 184 ++++++++++-- mcp/src/tools-play.js | 116 +++++--- mcp/test/integration.test.js | 211 ++++++++++++++ mcp/test/unit.test.js | 128 +++++++++ runwave/controller/README.md | 12 +- runwave/controller/src/action-normalizer.js | 5 +- runwave/controller/src/browser-pause.js | 270 ++++++++++++++++++ runwave/controller/src/browser-session.js | 235 +++++++++++---- runwave/controller/src/protocol.js | 23 +- runwave/controller/src/step-executor.js | 3 +- runwave/controller/src/step-runner.js | 3 +- .../controller/test/browser-session.test.js | 134 +++++++++ .../controller/test/session-cleanup.test.js | 29 ++ 21 files changed, 1635 insertions(+), 145 deletions(-) create mode 100644 mcp/src/replay.js create mode 100644 runwave/controller/src/browser-pause.js diff --git a/mcp/README.md b/mcp/README.md index 4d0e75f..0d7f5fc 100644 --- a/mcp/README.md +++ b/mcp/README.md @@ -6,9 +6,10 @@ frame. This is the interactive counterpart to the `runwave` CLI. The CLI runs a VLM in a loop by itself and produces a recorded video for playtesting. Here the connected -agent *is* the player, so the OpenRouter agent loop is not used and there is no -recording — which is also why this needs none of runwave's gstreamer, -PulseAudio, or Xvfb setup. +agent *is* the player, so the OpenRouter agent loop is not used. Interactive +play itself is not recorded: successful input sequences are persisted and can +be replayed afterwards in a separate Playwright-native recorder. This needs +none of runwave's gstreamer, PulseAudio, or Xvfb setup. ## Requirements @@ -28,7 +29,8 @@ No X server, no audio, no display. Chromium runs headless. node mcp/bin/runwave-mcp.js ``` -Artifacts (screenshots, per-step JSON) are written under +Artifacts (screenshots, per-step JSON, `playthrough.json`, and rendered videos) +are written under `RUNWAVE_MCP_WORKSPACE`, defaulting to a directory in the system temp dir. Set it explicitly if you want to keep them: @@ -54,15 +56,56 @@ claude mcp add runwave -- docker run --rm -i runwave-mcp | Tool | Purpose | | --- | --- | | `launch_game` | Start a session from a `url`, or a `game_dir` containing `start.sh` plus a `port`. Returns the first frame. | -| `observe` | Fresh frame and state, no input sent. | -| `act` | Send a timed input sequence, return the resulting frame. | +| `observe` | Fresh frame and state, no input sent. The game is paused while the agent reasons. | +| `act` | Resume, send a timed input sequence, pause, and return the resulting frame. | | `zoom` | Full-resolution crop of a region, to read small UI without a full frame. | | `capture` | Save a clean, full-resolution, un-annotated screenshot. The deliverable. | +| `focus_game` | Acquire pointer lock on the largest game canvas without rotating an FPS camera. | | `reset_game` | Reload at the launch URL. | +| `render_playthrough` | Replay the current successful input timeline in a fresh browser and record a game-only WebM without agent reasoning gaps. | | `journal` | Text log of what has been tried this session. | | `list_sessions` | Running sessions. | +| `pause_game` | Immediately pause the browser game at the harness level. | +| `resume_game` | Release a manual pause. | | `end_game` | Close the browser and stop the game process. | +## Playthrough recording + +Every successful `act` is appended to an atomically replaced +`playthrough.json`. It stores the concrete pixel coordinates, resolved keys, +step durations, and optional intent notes needed to reproduce the current +attempt. `reset_game` starts a fresh attempt and clears its steps. + +Call `render_playthrough` once the attempt is worth keeping. It leaves the live +game paused, opens the same launch URL in a fresh isolated browser, executes the +timeline without screenshots or model turns, and records that browser viewport +with Playwright's native screencast. The result includes paths to the WebM and a +replay manifest under the session's `replays/` directory. + +This is intentionally an input replay, not a game-engine clock patch. It stays +agnostic across canvas, WebGL, and DOM games. Games with randomness, networked +state, or nondeterministic physics can diverge from the original run; the +timeline preserves the agent's inputs and timing, not private engine state. + +## Pause behavior + +The MCP automatically pauses the page after `launch_game`, `observe`, `act`, +`zoom`, `capture`, and `reset_game` return their frame. This keeps a live game +from progressing while the agent is inspecting the image and deciding its next +move. `act` and `reset_game` temporarily resume the page for their operation, +then pause it again before returning. + +The pause is implemented in the browser harness rather than by sending the +game's own pause key. It holds animation frames and timers, virtualizes +`performance.now()`/`Date.now()` so a long reasoning turn does not create a +large simulation delta, and blocks new gameplay input while paused. Screenshots +and state reads remain available. + +`pause_game` is an immediate, higher-authority manual override. It does not wait +behind an in-progress `act` call. A manually paused session rejects `act` and +`reset_game` until `resume_game` is called, so an emergency pause cannot be +silently undone by a queued gameplay action. + ## How `act` works An action sequence is timed, not a single keypress. Offsets are milliseconds @@ -75,18 +118,53 @@ from the start of the sequence and actions may overlap, so one call can express "actions": [ { "type": "key", "start": 0, "end": 900, "key": "ArrowRight" }, { "type": "key", "start": 150, "end": 230, "key": "Space" } - ] + ], + "duration_ms": 900 } ``` -This matters because an agent turn costs seconds. Committing to a move beats -sending one tap per turn. +This matters because an agent turn costs seconds, but the useful batch size +depends on uncertainty: + +- Use a short 300-1200ms probe when calibrating direction or camera sensitivity, + approaching a collider, or searching for an interaction range. +- Commit to a longer sequence once the heading and open route are visually + verified. +- On an uncertain sequence longer than about 1500ms, request 2-3 spaced + `captures`. If the same landmark or surface fills consecutive frames, stop + pushing forward and recover. Omit trajectory frames on known traversal to + keep context and capture overhead low. + +This gives the agent feedback where mistakes are expensive without turning all +gameplay into one tap per turn. + +`duration_ms` is normally inferred from the latest action end. Set it explicitly +when an action starts a load or animation that should run before the final pause; +for example, click a play button at 0ms and use `duration_ms: 3000` to receive a +settled game frame rather than the first loading frame. + +To advance a loading screen or animation without sending input, use an empty +sequence with a positive duration: + +```json +{ + "session_id": "game-...", + "actions": [], + "duration_ms": 2000 +} +``` Action types: `key`, `click`, `multi_click`, `drag`, `cursor_move`, `view_move`. Pointer actions take either `x`/`y` in viewport pixels or an `overlay_row`/`overlay_col` grid cell. The full schema is enforced on the tool input, so a malformed sequence is rejected before any input is sent. +In a pointer-locked game, a `click` ignores its coordinates and presses the +current mouse button. A click may be held for up to two seconds, which is useful +for continuous fire or aim and avoids a slow burst of separate browser calls. +When state reports `pointer_locked: false`, call `focus_game` before using +`view_move`; calibrate with a small delta because sensitivity is game-defined. + ## Notes on the design **Frames are downscaled by default.** A 1280x720 PNG is roughly 1200 tokens; at diff --git a/mcp/src/config.js b/mcp/src/config.js index 872aa3b..67a4625 100644 --- a/mcp/src/config.js +++ b/mcp/src/config.js @@ -40,6 +40,13 @@ function buildSessionConfig(options = {}) { // The agent asks for frames explicitly; interval captures would spend // context on frames nobody requested. autoCaptures: false, + // Install the renderer-level pause gate. This remains opt-in so the normal + // runwave playtest path is not affected by MCP-specific browser patches. + pauseController: true, + // Pointer-locked games need continuous mouse holds for actions such as + // firing and aiming. Keep the normal Runwave/UI click guard at 100ms; only + // MCP gameplay sessions opt into a bounded longer hold. + maxActionSpanMs: { click: 2000 }, markGridSampleMode: CELL_SAMPLE_MODE, ...(options.markGridRows ? { markGridRows: Number(options.markGridRows) } : {}), ...(options.markGridCols ? { markGridCols: Number(options.markGridCols) } : {}), diff --git a/mcp/src/replay.js b/mcp/src/replay.js new file mode 100644 index 0000000..7a92bd5 --- /dev/null +++ b/mcp/src/replay.js @@ -0,0 +1,124 @@ +'use strict'; + +const path = require('path'); +const { createSession } = require('../../runwave/controller/src/session-factory'); +const { ensureDir, sleep, timestamp, writeJson } = require('../../runwave/controller/src/file-utils'); +const { executeTimeline } = require('../../runwave/controller/src/step-executor'); +const { normalizeStep } = require('../../runwave/controller/src/step-normalizer'); +const { buildStepTimeline } = require('../../runwave/controller/src/step-timeline'); + +function replayConfigFor(session) { + return { + kind: 'web', + url: session.browser.launchUrl, + viewport: { ...session.config.viewport }, + videoSize: { ...session.config.viewport }, + deviceScaleFactor: Number(session.config.deviceScaleFactor ?? 1), + record: true, + recordAudio: false, + recordingBackend: 'playwright', + playwrightVideoFileName: 'playthrough', + headless: true, + gridScreenshots: false, + fullPageScreenshots: false, + autoCaptures: false, + // The controller also supplies trusted pointer-lock movement filtering. + // It remains unpaused unless the replay explicitly calls pause/resume. + pauseController: true, + // Replay must accept the same bounded input spans that were valid in the + // live MCP session. Otherwise a successfully persisted held button can + // fail only when the clean recording is rendered. + maxActionSpanMs: { ...session.config.maxActionSpanMs }, + markGridSampleMode: session.config.markGridSampleMode, + keyAliases: session.config.keyAliases, + waitAfterLoad: Number(session.config.waitAfterLoad ?? 700), + }; +} + +function replayStep(step, config, index) { + return normalizeStep({ + action: 'step', + action_name: `replay-${String(index).padStart(3, '0')}`, + actions: step.actions, + duration: step.duration_ms, + captures: [], + autoCaptures: false, + }, config, index); +} + +async function executeReplayStep(browser, step, config, index) { + const focusAction = step.actions.find((action) => action.type === 'focus_game'); + if (focusAction) { + if (step.actions.length !== 1) throw new Error('focus_game replay steps cannot contain other actions'); + const startedAt = Date.now(); + if (focusAction.start > 0) await sleep(focusAction.start); + const focused = await browser.focusGame(focusAction); + if (!focused.pointerLocked) throw new Error('replay could not reacquire pointer lock'); + const remaining = step.duration_ms - (Date.now() - startedAt); + if (remaining > 0) await sleep(remaining); + return; + } + const normalized = replayStep(step, config, index); + const events = buildStepTimeline(normalized).filter((event) => event.type !== 'capture'); + await executeTimeline({ + browser, + events, + duration: normalized.duration, + outputDir: null, + prefix: `replay-${String(index).padStart(3, '0')}`, + stateExpression: null, + beforeEndCapture: null, + profiler: null, + }); +} + +async function renderPlaythrough(session, { tailMs = 1000 } = {}) { + const source = JSON.parse(JSON.stringify(session.playthrough)); + if (!source.steps.length) throw new Error('playthrough has no successful act steps to render'); + + const replayId = `replay-${timestamp()}`; + const runDir = ensureDir(path.join(session.paths.runDir, 'replays', replayId)); + const sourcePlaythrough = writeJson(path.join(runDir, 'playthrough.json'), source); + const config = replayConfigFor(session); + const browser = createSession(config, { runDir }, null); + const startedAt = Date.now(); + let closeResult; + + try { + await browser.start(); + for (let index = 0; index < source.steps.length; index += 1) { + await executeReplayStep(browser, source.steps[index], config, index + 1); + } + if (tailMs > 0) await sleep(tailMs); + closeResult = await browser.close(); + } catch (error) { + await browser.close().catch(() => {}); + throw error; + } + + if (!closeResult.video) throw new Error('Playwright replay recorder did not produce a video'); + const result = { + version: 1, + replay_id: replayId, + source_playthrough: sourcePlaythrough, + session_playthrough: session.paths.playthrough, + source_attempt: source.attempt, + recording_backend: 'playwright', + step_count: source.steps.length, + playthrough_duration_ms: source.total_duration_ms, + tail_ms: tailMs, + render_elapsed_ms: Date.now() - startedAt, + video: closeResult.video, + run_dir: runDir, + }; + result.manifest = writeJson(path.join(runDir, 'replay.json'), result); + session.replays.push(result); + return result; +} + +module.exports = { + executeReplayStep, + renderPlaythrough, + replayConfigFor, + replayStep, +}; diff --git a/mcp/src/result.js b/mcp/src/result.js index 83faefd..0950474 100644 --- a/mcp/src/result.js +++ b/mcp/src/result.js @@ -41,10 +41,18 @@ function gridNote(image, margin) { return `grid overlay is on. The image includes a ${margin}px label margin on every side, so image pixel (px, py) is viewport (px - ${margin}, py - ${margin}). Prefer overlay_row/overlay_col targets while the grid is on.`; } +function pauseNote(mode) { + if (mode === 'manual') { + return 'game_status: paused by manual override; call resume_game before act or reset_game.'; + } + return 'game_status: paused at this frame; act or reset_game resumes only for its controlled duration.'; +} + module.exports = { errorResult, frameBlocks, gridNote, + pauseNote, stateText, textBlock, }; diff --git a/mcp/src/schema.js b/mcp/src/schema.js index e51803d..26d30ce 100644 --- a/mcp/src/schema.js +++ b/mcp/src/schema.js @@ -7,6 +7,7 @@ const { DEFAULT_MARK_GRID } = require('../../runwave/protocol/src/mark-grid'); // Spans are pulled from the protocol rather than restated, so the tool contract // cannot drift from what the executor actually enforces. const span = (type) => (MAX_ACTION_SPAN_MS[type] ? ` Max ${MAX_ACTION_SPAN_MS[type]}ms.` : ''); +const MCP_CLICK_MAX_MS = 2000; const cell = z .object({ @@ -34,7 +35,7 @@ const keyAction = z.object({ const clickAction = z.object({ type: z.literal('click'), start, - end: z.number().min(0).optional().describe(`Hold duration.${span('click')}`), + end: z.number().min(0).optional().describe(`Release offset. Omit for a short click; use a longer span for a held game button such as sustained fire. Max ${MCP_CLICK_MAX_MS}ms.`), ...point.shape, button: z.enum(['left', 'middle', 'right']).optional(), clickCount: z.number().int().min(1).max(3).optional(), @@ -76,7 +77,7 @@ const viewMoveAction = z.object({ dx: z.number().optional().describe('Relative pointer delta X. Positive is right.'), dy: z.number().optional().describe('Relative pointer delta Y. Positive is down.'), steps: z.number().int().min(1).max(80).optional(), -}).describe('Relative mouse movement for pointer-lock/FPS camera control.'); +}).describe('Relative mouse movement for pointer-lock/FPS camera control. Start with a small calibration move because game sensitivity varies.'); const action = z .discriminatedUnion('type', [ diff --git a/mcp/src/server.js b/mcp/src/server.js index 5ac9ed7..d1a13ab 100644 --- a/mcp/src/server.js +++ b/mcp/src/server.js @@ -24,10 +24,16 @@ function createServer({ workspace = resolveWorkspace() } = {}) { { instructions: [ 'Play browser games by looking at frames and sending timed input sequences.', - 'Call launch_game once, then loop act/observe, then end_game.', + 'Call launch_game once, then loop act/observe, then end_game. Returned frames automatically leave the game paused while you reason; act and reset_game resume for their operation and pause again before returning.', + 'Use pause_game for a higher-authority manual pause and resume_game to release it.', 'Frames come back downscaled to save context; use zoom to inspect detail and full_res only when you must.', - 'Prefer one act call that commits to a move (hold a key for several hundred ms) over many single taps.', + 'When direction, distance, or collision is uncertain, use a short 300-1200ms act as a probe and inspect the result. Commit to longer batched movement only after the heading and open route are verified.', + 'For uncertain actions longer than about 1500ms, request 2-3 captures across the sequence; if landmarks stop changing or the same surface fills them, recover instead of repeating forward movement.', + 'If state reports pointer_locked=false in a mouse-look game, call focus_game before view_move. Begin with a small view_move because sensitivity varies by game.', + 'A click span may last up to two seconds, so use one held click for sustained fire or aiming instead of many rapid click actions.', + 'When a load or animation needs controlled time, set act duration_ms; actions may be an empty array for a wait with no input.', 'If a result says the frame did not change, the input did not land: change approach rather than repeating it.', + 'When the run is worth keeping, call render_playthrough before end_game to replay the successful input timeline in a separate recorder without reasoning gaps.', 'Use journal to recall what you have already tried, and capture to save the final screenshot.', ].join(' '), } diff --git a/mcp/src/session.js b/mcp/src/session.js index 933cd97..3ff71d4 100644 --- a/mcp/src/session.js +++ b/mcp/src/session.js @@ -1,5 +1,6 @@ 'use strict'; +const fs = require('fs'); const path = require('path'); const { createSession } = require('../../runwave/controller/src/session-factory'); const { OutputWriter } = require('../../runwave/controller/src/output-writer'); @@ -15,6 +16,7 @@ class Session { this.paths = { runDir: path.join(workspace, 'sessions', id), outputRoot: path.join(workspace, 'sessions', id, 'output'), + playthrough: path.join(workspace, 'sessions', id, 'playthrough.json'), }; ensureDir(this.paths.runDir); this.output = new OutputWriter(this.paths.outputRoot); @@ -22,7 +24,20 @@ class Session { this.stepIndex = 0; this.turn = 0; this.journal = []; + this.playthrough = { + version: 1, + session_id: id, + attempt: 1, + created_at: new Date().toISOString(), + launch_url: null, + viewport: { ...this.config.viewport }, + total_duration_ms: 0, + steps: [], + }; + this.replays = []; this.closed = false; + this.pauseMode = null; + this.manualPauseRevision = 0; this.createdAt = Date.now(); this.idleTimeoutMs = Number(options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS); this.idleTimer = null; @@ -34,6 +49,8 @@ class Session { async start() { await this.browser.start(); + this.playthrough.launch_url = this.browser.launchUrl; + this.persistPlaythrough(); this.touch(); return this; } @@ -77,6 +94,40 @@ class Session { return this.output.actionDir(name); } + // The live session writes screenshots and per-step result files. This + // compact, atomically replaced file is the reproducible input program used + // to render a clean playthrough in a separate browser later. + persistPlaythrough() { + const temporaryPath = `${this.paths.playthrough}.${process.pid}.${Date.now()}.tmp`; + fs.writeFileSync(temporaryPath, JSON.stringify(this.playthrough, null, 2)); + fs.renameSync(temporaryPath, this.paths.playthrough); + return this.paths.playthrough; + } + + appendPlaythroughStep({ duration, actions, note }) { + const entry = { + index: this.playthrough.steps.length + 1, + duration_ms: Number(duration), + actions: actions.map(canonicalReplayAction), + ...(note ? { note: String(note) } : {}), + }; + this.playthrough.steps.push(entry); + this.playthrough.total_duration_ms += entry.duration_ms; + this.persistPlaythrough(); + return entry; + } + + resetPlaythrough() { + this.playthrough = { + ...this.playthrough, + attempt: this.playthrough.attempt + 1, + created_at: new Date().toISOString(), + total_duration_ms: 0, + steps: [], + }; + return this.persistPlaythrough(); + } + async close() { if (this.closed) return; this.closed = true; @@ -84,6 +135,68 @@ class Session { await this.browser.close(); } + // A manual pause is an override: normal gameplay calls must not silently + // resume it. Automatic pauses are released by the next act/reset call. + async pause({ mode = 'automatic', reason } = {}) { + if (this.closed) throw new Error(`session ${this.id} has ended`); + if (this.pauseMode === 'manual' && mode !== 'manual') return this.pauseMode; + if (this.pauseMode === mode) return this.pauseMode; + if (mode === 'manual') { + // Publish the override before awaiting the renderer so a concurrently + // starting act/reset cannot resume past it. + const previousMode = this.pauseMode; + this.manualPauseRevision += 1; + this.pauseMode = 'manual'; + try { + await this.browser.pause(); + } catch (error) { + if (this.pauseMode === 'manual') this.pauseMode = previousMode; + throw error; + } + } else { + await this.browser.pause(); + if (this.pauseMode === 'manual') return this.pauseMode; + this.pauseMode = mode; + } + if (mode === 'manual') this.note({ event: 'pause', mode, ...(reason ? { reason } : {}) }); + this.touch(); + return this.pauseMode; + } + + async resume({ force = false, reason } = {}) { + if (this.closed) throw new Error(`session ${this.id} has ended`); + if (this.pauseMode === 'manual' && !force) { + throw new Error('game is manually paused; call resume_game before sending gameplay input'); + } + if (this.pauseMode === null) return false; + const previousMode = this.pauseMode; + this.pauseMode = null; + const manualPauseRevision = this.manualPauseRevision; + await this.browser.resume(); + if (manualPauseRevision !== this.manualPauseRevision || this.pauseMode === 'manual') { + await this.browser.pause(); + this.pauseMode = 'manual'; + throw new Error('game was manually paused while resume was in progress'); + } + if (previousMode === 'manual' || force) { + this.note({ event: 'resume', ...(reason ? { reason } : {}) }); + } + this.touch(); + return true; + } + + async prepareForGameplay(reason = 'act') { + if (this.pauseMode === 'manual') { + throw new Error('game is manually paused; call resume_game before sending gameplay input'); + } + if (this.pauseMode === 'automatic') await this.resume({ reason }); + } + + async pauseForAgent(reason = 'awaiting_agent') { + if (this.pauseMode === 'manual') return this.pauseMode; + return this.pause({ mode: 'automatic', reason }); + } + summary() { return { session_id: this.id, @@ -91,13 +204,72 @@ class Session { viewport: this.config.viewport, turns: this.turn, steps: this.stepIndex, + paused: this.pauseMode !== null, + pause_mode: this.pauseMode, uptime_ms: Date.now() - this.createdAt, run_dir: this.paths.runDir, + playthrough_path: this.paths.playthrough, + playthrough_steps: this.playthrough.steps.length, + replay_count: this.replays.length, closed: this.closed, }; } } +function canonicalPoint(point) { + return { x: Number(point.x), y: Number(point.y) }; +} + +function canonicalReplayAction(action) { + const common = { + type: action.type, + start: Number(action.start), + end: Number(action.end), + }; + if (action.type === 'key') { + return { ...common, key: action.resolvedKey || action.key }; + } + if (action.type === 'click') { + return { + ...common, + x: Number(action.x), + y: Number(action.y), + button: action.button, + clickCount: Number(action.clickCount), + }; + } + if (action.type === 'drag') { + return { + ...common, + from: canonicalPoint(action.from), + to: canonicalPoint(action.to), + button: action.button, + mode: action.mode, + steps: Number(action.steps), + }; + } + if (action.type === 'cursor_move') { + return { ...common, to: canonicalPoint(action.to), steps: Number(action.steps) }; + } + if (action.type === 'view_move') { + return { + ...common, + dx: Number(action.dx), + dy: Number(action.dy), + steps: Number(action.steps), + }; + } + if (action.type === 'focus_game') { + return { + ...common, + x: Number(action.x), + y: Number(action.y), + button: action.button, + }; + } + throw new Error(`cannot persist unsupported replay action: ${action.type}`); +} + function newSessionId() { return `game-${timestamp()}`; } @@ -105,5 +277,6 @@ function newSessionId() { module.exports = { DEFAULT_IDLE_TIMEOUT_MS, Session, + canonicalReplayAction, newSessionId, }; diff --git a/mcp/src/state.js b/mcp/src/state.js index 0fa69fa..b68e153 100644 --- a/mcp/src/state.js +++ b/mcp/src/state.js @@ -15,9 +15,6 @@ function compactState(raw) { if (active && active.tagName && active.tagName !== 'BODY') { state.focus = [active.tagName, active.id ? `#${active.id}` : ''].filter(Boolean).join(''); } - if (generic.pointerLockElement && generic.pointerLockElement.tagName) { - state.pointer_locked = true; - } // The largest canvas is almost always the game surface. Its client rect tells // an agent which part of the viewport is actually playable. @@ -25,6 +22,7 @@ function compactState(raw) { .filter((canvas) => canvas && canvas.clientWidth > 0 && canvas.clientHeight > 0) .sort((left, right) => right.clientWidth * right.clientHeight - left.clientWidth * left.clientHeight)[0]; if (surface) { + state.pointer_locked = Boolean(generic.pointerLockElement && generic.pointerLockElement.tagName); state.game_area = { x: Math.round(surface.left ?? surface.x ?? 0), y: Math.round(surface.top ?? surface.y ?? 0), diff --git a/mcp/src/tools-aux.js b/mcp/src/tools-aux.js index be1fcf8..49dcc07 100644 --- a/mcp/src/tools-aux.js +++ b/mcp/src/tools-aux.js @@ -3,7 +3,8 @@ const { z } = require('zod'); const { region } = require('./schema'); const { captureFrame, frameResult } = require('./frame'); -const { errorResult, textBlock } = require('./result'); +const { renderPlaythrough } = require('./replay'); +const { errorResult, pauseNote, textBlock } = require('./result'); function registerZoom(server, registry) { server.registerTool('zoom', { @@ -17,15 +18,19 @@ function registerZoom(server, registry) { try { const session = registry.get(args.session_id); return session.run(async () => { - const name = `zoom-${String(session.turn).padStart(3, '0')}`; - const { file } = await captureFrame(session, { name, grid: false }); - const state = await session.browser.state(session.config.stateExpression); - const result = frameResult({ - file, margin: 0, state, fullRes: true, region: args.region, - label: `zoom ${args.region.width}x${args.region.height} at (${args.region.x},${args.region.y})`, - }); - session.touch(); - return result; + try { + await session.pauseForAgent('awaiting_agent'); + const name = `zoom-${String(session.turn).padStart(3, '0')}`; + const { file } = await captureFrame(session, { name, grid: false }); + const state = await session.browser.state(session.config.stateExpression); + return frameResult({ + file, margin: 0, state, fullRes: true, region: args.region, + label: `zoom ${args.region.width}x${args.region.height} at (${args.region.x},${args.region.y})`, + extra: [pauseNote(session.pauseMode)], + }); + } finally { + await session.pauseForAgent('awaiting_agent'); + } }); } catch (error) { return errorResult(error); @@ -46,14 +51,18 @@ function registerCapture(server, registry) { try { const session = registry.get(args.session_id); return session.run(async () => { - const { file } = await captureFrame(session, { name: `capture-${args.name}`, grid: false }); - session.note({ event: 'capture', name: args.name, path: file }); - session.touch(); - const summary = textBlock(`saved ${session.config.viewport.width}x${session.config.viewport.height} clean capture\npath: ${file}`); - if (!args.preview) return { content: [summary] }; - const state = await session.browser.state(session.config.stateExpression); - const preview = frameResult({ file, margin: 0, state, label: 'saved' }); - return { content: [summary, ...preview.content] }; + try { + await session.pauseForAgent('awaiting_agent'); + const { file } = await captureFrame(session, { name: `capture-${args.name}`, grid: false }); + session.note({ event: 'capture', name: args.name, path: file }); + const summary = textBlock(`saved ${session.config.viewport.width}x${session.config.viewport.height} clean capture\npath: ${file}\n${pauseNote(session.pauseMode)}`); + if (!args.preview) return { content: [summary] }; + const state = await session.browser.state(session.config.stateExpression); + const preview = frameResult({ file, margin: 0, state, label: 'saved' }); + return { content: [summary, ...preview.content] }; + } finally { + await session.pauseForAgent('awaiting_agent'); + } }); } catch (error) { return errorResult(error); @@ -62,6 +71,58 @@ function registerCapture(server, registry) { } function registerSessionTools(server, registry) { + server.registerTool('focus_game', { + title: 'Focus game controls', + description: 'Acquire pointer lock on the game canvas without moving the FPS camera. Call this when state reports pointer_locked=false before using view_move. Middle click avoids firing in most games; retry with left only if the game requires it.', + inputSchema: { + session_id: z.string(), + button: z.enum(['left', 'middle', 'right']).optional() + .describe('Button used to acquire focus. Default middle.'), + }, + }, async (args) => { + try { + const session = registry.get(args.session_id); + return session.run(async () => { + let focus; + try { + await session.prepareForGameplay('focus_game'); + session.nextStepIndex(); + focus = await session.browser.focusGame({ button: args.button || 'middle' }); + if (!focus.pointerLocked) { + throw new Error(`game canvas did not acquire pointer lock with ${focus.button} click`); + } + } finally { + await session.pauseForAgent('awaiting_agent'); + } + session.appendPlaythroughStep({ + duration: 75, + actions: [{ + type: 'focus_game', + start: 0, + end: 75, + x: focus.x, + y: focus.y, + button: focus.button, + }], + note: 'acquire pointer lock without camera movement', + }); + session.note({ event: 'focus_game', button: focus.button, pointer_locked: true }); + const { file } = await captureFrame(session, { + name: `focus-${String(session.stepIndex).padStart(3, '0')}`, + grid: false, + }); + session.lastFrame = file; + const state = await session.browser.state(session.config.stateExpression); + return frameResult({ + file, margin: 0, state, label: 'game focused', + extra: [pauseNote(session.pauseMode)], + }); + }); + } catch (error) { + return errorResult(error); + } + }); + server.registerTool('reset_game', { title: 'Reset game', description: 'Reload the game at its launch URL. Use this when stuck or to start a fresh attempt.', @@ -70,15 +131,23 @@ function registerSessionTools(server, registry) { try { const session = registry.get(args.session_id); return session.run(async () => { - await session.browser.navigate({ url: session.browser.launchUrl }); - session.stepIndex = 0; - session.note({ event: 'reset' }); - const { file } = await captureFrame(session, { name: `reset-${session.turn}`, grid: false }); - session.lastFrame = file; - const state = await session.browser.state(session.config.stateExpression); - const result = frameResult({ file, margin: 0, state, label: 'after reset' }); - session.touch(); - return result; + try { + await session.prepareForGameplay('reset_game'); + await session.browser.navigate({ url: session.browser.launchUrl }); + session.stepIndex = 0; + session.resetPlaythrough(); + session.note({ event: 'reset' }); + await session.pauseForAgent('awaiting_agent'); + const { file } = await captureFrame(session, { name: `reset-${session.turn}`, grid: false }); + session.lastFrame = file; + const state = await session.browser.state(session.config.stateExpression); + return frameResult({ + file, margin: 0, state, label: 'after reset', + extra: [pauseNote(session.pauseMode)], + }); + } finally { + await session.pauseForAgent('awaiting_agent'); + } }); } catch (error) { return errorResult(error); @@ -135,6 +204,67 @@ function registerSessionTools(server, registry) { return errorResult(error); } }); + + server.registerTool('render_playthrough', { + title: 'Render clean playthrough', + description: 'Replay the successful act timeline in a fresh isolated browser and record a smooth game-only WebM. The live agent session remains paused, so model reasoning and screenshot latency do not appear in the video.', + inputSchema: { + session_id: z.string(), + tail_ms: z.number().int().min(0).max(5000).optional() + .describe('Extra live time after the final action. Default 1000ms.'), + }, + }, async (args) => { + try { + const session = registry.get(args.session_id); + return session.run(async () => { + try { + await session.pauseForAgent('render_playthrough'); + const replay = await renderPlaythrough(session, { tailMs: args.tail_ms ?? 1000 }); + session.note({ + event: 'render_playthrough', + steps: replay.step_count, + video: replay.video, + }); + return { content: [textBlock(JSON.stringify(replay, null, 2))] }; + } finally { + await session.pauseForAgent('awaiting_agent'); + } + }); + } catch (error) { + return errorResult(error); + } + }); + + server.registerTool('pause_game', { + title: 'Pause game immediately', + description: 'Immediately pause the browser game at the harness level. This is a higher-authority override and does not wait behind an in-progress gameplay call. Call resume_game before act if you paused manually.', + inputSchema: { + session_id: z.string(), + reason: z.string().optional().describe('Why the pause was requested.'), + }, + }, async (args) => { + try { + const session = registry.get(args.session_id); + await session.pause({ mode: 'manual', reason: args.reason || 'manual_pause' }); + return { content: [textBlock(JSON.stringify(session.summary(), null, 2))] }; + } catch (error) { + return errorResult(error); + } + }); + + server.registerTool('resume_game', { + title: 'Resume game', + description: 'Release a manual pause so the next act or reset_game call can continue gameplay.', + inputSchema: { session_id: z.string() }, + }, async (args) => { + try { + const session = registry.get(args.session_id); + await session.resume({ force: true, reason: 'manual_resume' }); + return { content: [textBlock(JSON.stringify(session.summary(), null, 2))] }; + } catch (error) { + return errorResult(error); + } + }); } module.exports = { registerCapture, registerSessionTools, registerZoom }; diff --git a/mcp/src/tools-play.js b/mcp/src/tools-play.js index 511ce0c..3f7295a 100644 --- a/mcp/src/tools-play.js +++ b/mcp/src/tools-play.js @@ -1,10 +1,11 @@ 'use strict'; const { z } = require('zod'); +const { inferDurationFromRawActions } = require('../../runwave/controller/src/action-normalizer'); const { runStep } = require('../../runwave/controller/src/step-runner'); const { action } = require('./schema'); const { captureFrame, frameResult } = require('./frame'); -const { errorResult } = require('./result'); +const { errorResult, pauseNote } = require('./result'); const { changedSince } = require('./diff'); const { applyGrid } = require('./frame'); @@ -42,16 +43,23 @@ function registerPlayTools(server, registry) { stateExpression: args.state_expression, }); return session.run(async () => { - const { file, margin } = await captureFrame(session, { name: 'launch', grid: args.grid }); - session.lastFrame = file; - session.note({ event: 'launch', url: session.browser.launchUrl }); - const state = await session.browser.state(session.config.stateExpression); - const result = frameResult({ - file, margin, state, fullRes: args.full_res, label: 'initial', - extra: [`session_id: ${session.id}`, `viewport: ${session.config.viewport.width}x${session.config.viewport.height}`], - }); - session.touch(); - return result; + try { + await session.pauseForAgent('awaiting_agent'); + const { file, margin } = await captureFrame(session, { name: 'launch', grid: args.grid }); + session.lastFrame = file; + session.note({ event: 'launch', url: session.browser.launchUrl }); + const state = await session.browser.state(session.config.stateExpression); + return frameResult({ + file, margin, state, fullRes: args.full_res, label: 'initial', + extra: [ + `session_id: ${session.id}`, + `viewport: ${session.config.viewport.width}x${session.config.viewport.height}`, + pauseNote(session.pauseMode), + ], + }); + } finally { + await session.pauseForAgent('awaiting_agent'); + } }); } catch (error) { return errorResult(error); @@ -97,6 +105,7 @@ function actResult({ session, step, args, previousFrame, actionName }) { changed === false ? 'frame is byte-identical to the previous one: the input probably did not reach the game. Check focus, try a different key, or hold it longer.' : null, + pauseNote(session.pauseMode), ], }); session.touch(); @@ -115,13 +124,19 @@ function registerObserve(server, registry) { try { const session = registry.get(args.session_id); return session.run(async () => { - const name = `observe-${String(session.turn).padStart(3, '0')}`; - const { file, margin } = await captureFrame(session, { name, grid: args.grid }); - const state = await session.browser.state(session.config.stateExpression); - session.lastFrame = file; - const result = frameResult({ file, margin, state, fullRes: args.full_res }); - session.touch(); - return result; + try { + await session.pauseForAgent('awaiting_agent'); + const name = `observe-${String(session.turn).padStart(3, '0')}`; + const { file, margin } = await captureFrame(session, { name, grid: args.grid }); + const state = await session.browser.state(session.config.stateExpression); + session.lastFrame = file; + return frameResult({ + file, margin, state, fullRes: args.full_res, + extra: [pauseNote(session.pauseMode)], + }); + } finally { + await session.pauseForAgent('awaiting_agent'); + } }); } catch (error) { return errorResult(error); @@ -135,37 +150,62 @@ function registerAct(server, registry) { description: [ 'Send a timed sequence of inputs, then return the resulting frame.', 'Offsets are milliseconds from the start of the sequence and actions may overlap, so one call can express "hold right for 900ms and jump at 150ms".', - 'This is the main way to play: prefer one sequence that commits to a move over many single-input calls.', + 'This is the main way to play. When direction, distance, or collision is uncertain, use a short 300-1200ms probe and inspect the result. Batch longer sequences only after the route or target is verified.', + 'For uncertain actions longer than about 1500ms, request 2-3 trajectory captures so repeated landmarks or a filled screen reveal a collision before the whole sequence is wasted.', ].join(' '), inputSchema: { session_id: z.string(), - actions: z.array(action).min(1).describe('Inputs to run. Total sequence should stay under 8000ms.'), + actions: z.array(action).describe('Inputs to run. May be empty only when duration_ms is positive, to advance a load or animation without input.'), + duration_ms: z.number().min(0).max(8000).optional() + .describe('Total live-game time for the sequence. Use this to let a click-triggered load or animation settle before the final pause.'), captures: z.array(z.number().min(0)).max(MAX_FRAMES_PER_TURN).optional() - .describe('Offsets in ms to screenshot at. Defaults to the end of the sequence. Each extra frame costs context.'), + .describe('Offsets in ms to screenshot at. Defaults to the end. For uncertain navigation longer than ~1500ms, use 2-3 spaced offsets to detect collisions or missed turns; omit them on verified traversal to save context.'), note: z.string().optional().describe('Short intent for the journal, e.g. "cross bridge east".'), ...frameOptions, }, }, async (args) => { try { + if (!args.actions.length && !(args.duration_ms > 0)) { + throw new Error('act requires at least one input or a positive duration_ms'); + } const session = registry.get(args.session_id); return session.run(async () => { - const stepIndex = session.nextStepIndex(); - const actionName = `act-${String(stepIndex).padStart(3, '0')}`; - const previousFrame = session.lastFrame; - const step = await runStep({ - input: { - action: 'step', - action_name: actionName, - actions: args.actions, - ...(args.captures ? { captures: args.captures } : {}), - autoCaptures: false, - }, - config: session.config, - browser: session.browser, - outputDir: session.actionDir(actionName), - nextStepIndex: stepIndex, - actionName, - profiler: null, + let step; + let previousFrame; + let actionName; + try { + await session.prepareForGameplay('act'); + const stepIndex = session.nextStepIndex(); + actionName = `act-${String(stepIndex).padStart(3, '0')}`; + previousFrame = session.lastFrame; + const duration = args.duration_ms ?? inferDurationFromRawActions(args.actions); + const captures = args.captures + ? [...args.captures, duration] + : undefined; + step = await runStep({ + input: { + action: 'step', + action_name: actionName, + actions: args.actions, + ...(args.duration_ms !== undefined ? { duration: args.duration_ms } : {}), + ...(captures ? { captures } : {}), + autoCaptures: false, + }, + config: session.config, + browser: session.browser, + outputDir: session.actionDir(actionName), + nextStepIndex: session.stepIndex, + actionName, + beforeEndCapture: () => session.pauseForAgent('awaiting_agent'), + profiler: null, + }); + } finally { + await session.pauseForAgent('awaiting_agent'); + } + session.appendPlaythroughStep({ + duration: step.duration, + actions: step.actions, + note: args.note, }); return actResult({ session, step, args, previousFrame, actionName }); }); diff --git a/mcp/test/integration.test.js b/mcp/test/integration.test.js index b34bdf8..ebb224b 100644 --- a/mcp/test/integration.test.js +++ b/mcp/test/integration.test.js @@ -11,8 +11,11 @@ const path = require('node:path'); const test = require('node:test'); const { pathToFileURL } = require('node:url'); +const { Client } = require('@modelcontextprotocol/sdk/client/index.js'); +const { InMemoryTransport } = require('@modelcontextprotocol/sdk/inMemory.js'); const { chromium } = require('playwright'); const { SessionRegistry } = require('../src/registry'); +const { createServer } = require('../src/server'); const { captureFrame } = require('../src/frame'); const { changedSince } = require('../src/diff'); const { compactState } = require('../src/state'); @@ -84,6 +87,7 @@ test('MCP session plays a real browser game', async (t) => { outputDir: session.actionDir(actionName), nextStepIndex: stepIndex, actionName, + beforeEndCapture: () => session.pauseForAgent('integration test'), profiler: null, }); @@ -94,6 +98,7 @@ test('MCP session plays a real browser game', async (t) => { `holding right must move the player: ${before.custom.x} -> ${after.custom.x}` ); assert.equal(changedSince(previousFrame, step.captures[0].path), true); + assert.equal(await session.browser.isPaused(), true, 'the returned frame must be the pause boundary'); session.lastFrame = step.captures[0].path; }); @@ -102,6 +107,7 @@ test('MCP session plays a real browser game', async (t) => { // grid that is columns 12-13 and rows 6-9, so cell (7, 12) must hit it. const stepIndex = session.nextStepIndex(); const actionName = `act-${stepIndex}`; + await session.prepareForGameplay('integration test'); const step = await runStep({ input: { action: 'step', @@ -114,6 +120,7 @@ test('MCP session plays a real browser game', async (t) => { outputDir: session.actionDir(actionName), nextStepIndex: stepIndex, actionName, + beforeEndCapture: () => session.pauseForAgent('integration test'), profiler: null, }); @@ -129,3 +136,207 @@ test('MCP session plays a real browser game', async (t) => { assert.equal(step.endState.custom.lit, true); }); }); + +test('harness pause freezes page time and blocks gameplay input', async (t) => { + if (!(await chromiumUsable())) { + t.skip('chromium cannot launch here; run "npx playwright install-deps chromium"'); + return; + } + const workspace = await fsp.mkdtemp(path.join(os.tmpdir(), 'runwave-mcp-pause-')); + const registry = new SessionRegistry({ workspace }); + const html = ``; + const session = await registry.create({ + url: `data:text/html,${encodeURIComponent(html)}`, + viewport: { width: 320, height: 200 }, + waitAfterLoad: 0, + }); + t.after(async () => { + await registry.closeAll(); + await fsp.rm(workspace, { recursive: true, force: true }); + }); + + await session.browser.page.waitForTimeout(120); + await session.pause({ mode: 'manual', reason: 'integration test' }); + const pausedAt = await session.browser.page.evaluate(() => ({ ticks, frames, now: performance.now() })); + await session.browser.page.waitForTimeout(250); + const frozen = await session.browser.page.evaluate(() => ({ + ticks, + frames, + now: performance.now(), + paused: window.__runwavePauseController.isPaused(), + })); + assert.equal(frozen.paused, true); + assert.equal(frozen.ticks, pausedAt.ticks, 'game timers must not advance while paused'); + assert.equal(frozen.frames, pausedAt.frames, 'animation frames must not advance while paused'); + assert.ok(frozen.now - pausedAt.now < 10, 'logical performance time must remain frozen'); + + await session.browser.keyDown('a'); + await session.browser.keyUp('a'); + assert.equal(await session.browser.page.evaluate(() => keys), 0, 'paused input must not reach the game'); + + await session.resume({ force: true, reason: 'integration test' }); + await session.browser.keyDown('a'); + await session.browser.keyUp('a'); + await session.browser.page.waitForTimeout(120); + const after = await session.browser.page.evaluate(() => ({ ticks, frames, keys, paused: window.__runwavePauseController.isPaused() })); + assert.equal(after.paused, false); + assert.ok(after.ticks > frozen.ticks, 'game timers must resume'); + assert.ok(after.frames > frozen.frames, 'animation frames must resume'); + assert.equal(after.keys, 1, 'resumed input must reach the game'); +}); + +test('pointer-locked view movement reaches games as one trusted delta', async (t) => { + if (!(await chromiumUsable())) { + t.skip('chromium cannot launch here; run "npx playwright install-deps chromium"'); + return; + } + const workspace = await fsp.mkdtemp(path.join(os.tmpdir(), 'runwave-mcp-pointer-lock-')); + const registry = new SessionRegistry({ workspace }); + const html = ``; + const session = await registry.create({ + url: `data:text/html,${encodeURIComponent(html)}`, + viewport: VIEWPORT, + waitAfterLoad: 0, + }); + t.after(async () => { + await registry.closeAll(); + await fsp.rm(workspace, { recursive: true, force: true }); + }); + + await session.browser.page.mouse.move(200, 180); + session.browser.mousePosition = { x: 200, y: 180 }; + await session.browser.page.evaluate(() => { window.moves = []; }); + const focus = await session.browser.focusGame({ button: 'middle' }); + assert.equal(focus.pointerLocked, true); + assert.deepEqual( + await session.browser.page.evaluate(() => window.moves), + [], + 'acquiring pointer lock must not rotate the game camera' + ); + + await session.browser.moveView({ dx: 24, dy: -9, steps: 1 }); + await session.browser.page.waitForTimeout(50); + assert.deepEqual(await session.browser.page.evaluate(() => window.moves), [ + { x: 24, y: -9, trusted: true }, + ]); +}); + +test('MCP tool loop pauses every returned frame and resumes only for act', async (t) => { + if (!(await chromiumUsable())) { + t.skip('chromium cannot launch here; run "npx playwright install-deps chromium"'); + return; + } + const workspace = await fsp.mkdtemp(path.join(os.tmpdir(), 'runwave-mcp-tools-pause-')); + const { server, registry } = createServer({ workspace }); + const client = new Client({ name: 'runwave-pause-integration', version: '1.0.0' }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + t.after(async () => { + await registry.closeAll(); + await client.close().catch(() => {}); + await server.close().catch(() => {}); + await fsp.rm(workspace, { recursive: true, force: true }); + }); + + const textOf = (result) => result.content + .filter((item) => item.type === 'text') + .map((item) => item.text) + .join('\n'); + const stateOf = (result) => { + const match = textOf(result).match(/state: (\{.*\})/); + assert.ok(match, `missing state in result: ${textOf(result)}`); + return JSON.parse(match[1]); + }; + const html = '

0
'; + const launch = await client.callTool({ + name: 'launch_game', + arguments: { + url: `data:text/html,${encodeURIComponent(html)}`, + viewport: { width: 320, height: 200 }, + state_expression: '() => ({ ticks: window.ticks })', + full_res: true, + }, + }); + const sessionId = textOf(launch).match(/session_id: (\S+)/)[1]; + const session = registry.get(sessionId); + const initialTicks = stateOf(launch).custom.ticks; + assert.equal(session.pauseMode, 'automatic'); + assert.equal(await session.browser.isPaused(), true); + + await new Promise((resolve) => setTimeout(resolve, 150)); + const still = await client.callTool({ name: 'observe', arguments: { session_id: sessionId, full_res: true } }); + assert.equal(stateOf(still).custom.ticks, initialTicks, 'the game must not advance during model reasoning'); + + const invalidWait = await client.callTool({ + name: 'act', + arguments: { session_id: sessionId, actions: [] }, + }); + assert.equal(invalidWait.isError, true); + assert.match(textOf(invalidWait), /positive duration_ms/); + + const acted = await client.callTool({ + name: 'act', + arguments: { + session_id: sessionId, + actions: [], + duration_ms: 160, + full_res: true, + }, + }); + const actedTicks = stateOf(acted).custom.ticks; + assert.ok(actedTicks > initialTicks, 'act must resume the game for its controlled duration'); + assert.equal(session.pauseMode, 'automatic'); + assert.equal(await session.browser.isPaused(), true); + + await new Promise((resolve) => setTimeout(resolve, 150)); + const after = await client.callTool({ name: 'observe', arguments: { session_id: sessionId, full_res: true } }); + assert.equal(stateOf(after).custom.ticks, actedTicks, 'the final act frame must be the pause boundary'); + + const persisted = JSON.parse(await fsp.readFile(session.paths.playthrough, 'utf8')); + assert.equal(persisted.steps.length, 1); + assert.equal(persisted.steps[0].duration_ms, 160); + assert.deepEqual(persisted.steps[0].actions, []); + + const beforeRenderTicks = await session.browser.page.evaluate(() => ticks); + const rendered = await client.callTool({ + name: 'render_playthrough', + arguments: { session_id: sessionId, tail_ms: 100 }, + }); + assert.notEqual(rendered.isError, true, textOf(rendered)); + const replay = JSON.parse(textOf(rendered)); + assert.equal(replay.recording_backend, 'playwright'); + assert.equal(replay.step_count, 1); + assert.ok((await fsp.stat(replay.video)).size > 0, 'replay must produce a non-empty WebM'); + assert.ok((await fsp.stat(replay.source_playthrough)).size > 0, 'replay must retain its exact timeline snapshot'); + assert.ok((await fsp.stat(replay.manifest)).size > 0, 'replay must produce a manifest'); + assert.equal(await session.browser.page.evaluate(() => ticks), beforeRenderTicks, 'rendering must not advance the live game'); + assert.equal(await session.browser.isPaused(), true, 'live game remains paused while the separate replay renders'); + + await client.callTool({ name: 'pause_game', arguments: { session_id: sessionId, reason: 'override test' } }); + const blocked = await client.callTool({ + name: 'act', + arguments: { session_id: sessionId, actions: [{ type: 'key', start: 0, end: 50, key: 'a' }] }, + }); + assert.equal(blocked.isError, true); + assert.match(textOf(blocked), /resume_game/); + await client.callTool({ name: 'resume_game', arguments: { session_id: sessionId } }); + await client.callTool({ name: 'end_game', arguments: { session_id: sessionId } }); +}); diff --git a/mcp/test/unit.test.js b/mcp/test/unit.test.js index 721d4da..3a90665 100644 --- a/mcp/test/unit.test.js +++ b/mcp/test/unit.test.js @@ -1,6 +1,9 @@ 'use strict'; const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); const test = require('node:test'); const { PNG } = require('pngjs'); @@ -8,6 +11,8 @@ const { buildSessionConfig, normalizeViewport } = require('../src/config'); const { clampScale, crop, resize } = require('../src/image'); const { compactState } = require('../src/state'); const { action } = require('../src/schema'); +const { Session, canonicalReplayAction } = require('../src/session'); +const { replayConfigFor, replayStep } = require('../src/replay'); const { normalizeActions } = require('../../runwave/controller/src/action-normalizer'); function solidPng(width, height, color = [10, 20, 30]) { @@ -28,6 +33,21 @@ test('session config disables recording and grid overlay by default', () => { assert.equal(config.headless, true); assert.equal(config.gridScreenshots, false, 'overlay must be opt-in to keep image and input coordinates aligned'); assert.equal(config.autoCaptures, false); + assert.equal(config.pauseController, true); + assert.equal(config.maxActionSpanMs.click, 2000); +}); + +test('MCP sessions allow a bounded held mouse button without changing normal Runwave clicks', () => { + const heldClick = [{ type: 'click', start: 0, end: 1500, x: 10, y: 20 }]; + assert.doesNotThrow(() => normalizeActions( + heldClick, + 1600, + { strict: true, config: buildSessionConfig(), aliases: {} } + )); + assert.throws( + () => normalizeActions(heldClick, 1600, { strict: true, config: {}, aliases: {} }), + /click action duration exceeds 100ms/ + ); }); test('session config always carries a numeric viewport so grid cells resolve', () => { @@ -114,6 +134,7 @@ test('compactState keeps the largest canvas as the game area and drops noise', ( }, }); assert.deepEqual(state.game_area, { x: 20, y: 30, width: 640, height: 360 }); + assert.equal(state.pointer_locked, false); assert.equal(state.canvas_count, 2); assert.equal(state.webgl, undefined, 'renderer probe is per-turn noise for a player'); assert.equal(state.focus, undefined, 'a BODY focus carries no signal'); @@ -141,3 +162,110 @@ test('act schema rejects unknown action types and negative offsets', () => { assert.throws(() => action.parse({ type: 'scroll', start: 0 })); assert.throws(() => action.parse({ type: 'key', start: -5, key: 'a' })); }); + +test('playthrough persistence stores concrete replay-safe inputs atomically', async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'runwave-playthrough-unit-')); + const session = new Session({ + id: 'playthrough-test', + workspace, + options: { url: 'about:blank', idleTimeoutMs: 0 }, + }); + session.browser = { close: async () => {} }; + try { + session.appendPlaythroughStep({ + duration: 600, + note: 'move and shoot', + actions: [ + { type: 'key', start: 0, end: 600, key: 'right', resolvedKey: 'ArrowRight' }, + { type: 'click', start: 200, end: 250, x: 321, y: 123, button: 'left', clickCount: 1, cells: [{ row: 1, col: 2 }] }, + ], + }); + + const saved = JSON.parse(fs.readFileSync(session.paths.playthrough, 'utf8')); + assert.equal(saved.total_duration_ms, 600); + assert.equal(saved.steps[0].note, 'move and shoot'); + assert.deepEqual(saved.steps[0].actions, [ + { type: 'key', start: 0, end: 600, key: 'ArrowRight' }, + { type: 'click', start: 200, end: 250, x: 321, y: 123, button: 'left', clickCount: 1 }, + ]); + assert.equal(fs.readdirSync(session.paths.runDir).some((name) => name.endsWith('.tmp')), false); + + session.resetPlaythrough(); + assert.equal(session.playthrough.attempt, 2); + assert.equal(session.playthrough.steps.length, 0); + assert.equal(session.playthrough.total_duration_ms, 0); + } finally { + await session.close(); + fs.rmSync(workspace, { recursive: true, force: true }); + } +}); + +test('replay normalization retains step timing without screenshot events', () => { + const session = { + browser: { launchUrl: 'https://example.test/game' }, + config: buildSessionConfig({ url: 'https://example.test/game', viewport: { width: 800, height: 450 } }), + }; + const config = replayConfigFor(session); + const step = replayStep({ + duration_ms: 500, + actions: [canonicalReplayAction({ + type: 'cursor_move', start: 50, end: 300, to: { x: 400, y: 200, cells: [{ row: 1, col: 1 }] }, steps: 8, + })], + }, config, 1); + + assert.equal(config.recordingBackend, 'playwright'); + assert.equal(config.pauseController, true); + assert.equal(config.maxActionSpanMs.click, 2000); + assert.equal(step.duration, 500); + assert.deepEqual(step.cursorMoves[0].to, { x: 400, y: 200 }); +}); + +test('replay accepts the MCP session held-click limit', () => { + const session = { + browser: { launchUrl: 'https://example.test/game' }, + config: buildSessionConfig({ url: 'https://example.test/game' }), + }; + const config = replayConfigFor(session); + assert.doesNotThrow(() => replayStep({ + duration_ms: 1900, + actions: [{ type: 'click', start: 200, end: 1700, x: 640, y: 360 }], + }, config, 1)); +}); + +test('manual pause bypasses queued gameplay and requires an explicit resume', async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'runwave-pause-unit-')); + const session = new Session({ + id: 'pause-test', + workspace, + options: { url: 'about:blank', idleTimeoutMs: 0 }, + }); + const calls = []; + session.browser = { + launchUrl: 'about:blank', + pause: async () => { calls.push('pause'); }, + resume: async () => { calls.push('resume'); }, + close: async () => {}, + }; + let release; + try { + const running = session.run(async () => { + calls.push('act-start'); + await new Promise((resolve) => { release = resolve; }); + calls.push('act-end'); + }); + await new Promise((resolve) => setImmediate(resolve)); + + await session.pause({ mode: 'manual', reason: 'interrupt test' }); + assert.deepEqual(calls, ['act-start', 'pause'], 'pause must not wait for the active session queue'); + await assert.rejects(session.prepareForGameplay(), /manually paused.*resume_game/); + + await session.resume({ force: true }); + assert.deepEqual(calls, ['act-start', 'pause', 'resume']); + release(); + await running; + } finally { + if (release) release(); + await session.close(); + fs.rmSync(workspace, { recursive: true, force: true }); + } +}); diff --git a/runwave/controller/README.md b/runwave/controller/README.md index 5277922..002b28b 100644 --- a/runwave/controller/README.md +++ b/runwave/controller/README.md @@ -235,10 +235,14 @@ List known sessions: runwave-controller '{"action":"sessions"}' ``` -When `record: true` is set, `stop` returns `video` and `audioVideo` pointing at -the same recorded audio/video WebM. All recording goes through gstreamer - see -the top-level [Requirements](../../README.md#requirements) section for the -mandatory environment (Linux, gstreamer, X server/Xvfb, PulseAudio). +When `record: true` is set, `stop` returns the recorded WebM in `video`. +Recording uses gstreamer by default; see the top-level +[Requirements](../../README.md#requirements) section for the mandatory Linux, +X server/Xvfb, and PulseAudio environment. Web sessions can instead set +`"recordingBackend":"playwright"` to capture the browser viewport directly. +That backend can run headlessly and does not require X11, but is video-only. +`recordAudio: true` continues to require the gstreamer backend and returns the +same audio/video WebM in both `video` and `audioVideo`. ## State diff --git a/runwave/controller/src/action-normalizer.js b/runwave/controller/src/action-normalizer.js index a9548f8..b77e5e7 100644 --- a/runwave/controller/src/action-normalizer.js +++ b/runwave/controller/src/action-normalizer.js @@ -100,7 +100,10 @@ function normalizeTiming(action, type, duration, options) { return invalidTiming(`invalid ${type} interval`, action, options); } - const maxSpan = MAX_ACTION_SPAN_MS[type]; + const configuredMaxSpan = Number(options.config?.maxActionSpanMs?.[type]); + const maxSpan = Number.isFinite(configuredMaxSpan) + ? configuredMaxSpan + : MAX_ACTION_SPAN_MS[type]; if (Number.isFinite(maxSpan) && end - start > maxSpan) { return invalidTiming(`${type} action duration exceeds ${maxSpan}ms`, action, options); } diff --git a/runwave/controller/src/browser-pause.js b/runwave/controller/src/browser-pause.js new file mode 100644 index 0000000..8beeb6d --- /dev/null +++ b/runwave/controller/src/browser-pause.js @@ -0,0 +1,270 @@ +'use strict'; + +// This script is installed before the game page's own scripts. It pauses the +// browser game rather than sending a game-specific key such as Escape, which +// makes the control independent of the game UI. The clock is virtualized so a +// long model turn does not become a large physics delta when the page resumes. +const BROWSER_PAUSE_INIT_SCRIPT = String.raw` +(() => { + const name = '__runwavePauseController'; + if (window[name]) return; + + const native = { + dateNow: Date.now.bind(Date), + performanceNow: window.performance.now.bind(window.performance), + requestAnimationFrame: window.requestAnimationFrame.bind(window), + cancelAnimationFrame: window.cancelAnimationFrame.bind(window), + setTimeout: window.setTimeout.bind(window), + clearTimeout: window.clearTimeout.bind(window), + setInterval: window.setInterval.bind(window), + clearInterval: window.clearInterval.bind(window), + }; + const state = { + paused: false, + performanceOffset: 0, + dateOffset: 0, + pausePerformanceNow: 0, + pauseDateNow: 0, + nextId: 1, + frames: new Map(), + timers: new Map(), + intervals: new Map(), + pointerMove: null, + suppressPointerMovesUntil: 0, + }; + + const logicalPerformanceNow = () => state.paused + ? state.pausePerformanceNow + : native.performanceNow() - state.performanceOffset; + const logicalDateNow = () => state.paused + ? state.pauseDateNow + : native.dateNow() - state.dateOffset; + + function invoke(record) { + if (typeof record.callback === 'function') { + return record.callback.apply(window, record.args); + } + return (0, eval)(String(record.callback)); + } + + function scheduleFrame(record) { + if (state.paused || record.nativeId !== null) return; + record.nativeId = native.requestAnimationFrame(() => { + record.nativeId = null; + if (state.paused || !state.frames.has(record.id)) return; + state.frames.delete(record.id); + record.callback(logicalPerformanceNow()); + }); + } + + function scheduleTimer(record) { + if (state.paused || record.nativeId !== null) return; + const delay = Math.max(0, record.due - logicalPerformanceNow()); + record.nativeId = native.setTimeout(() => { + record.nativeId = null; + if (state.paused || !state.timers.has(record.id)) return; + state.timers.delete(record.id); + invoke(record); + }, delay); + } + + function scheduleInterval(record) { + if (state.paused || record.nativeId !== null) return; + const delay = Math.max(0, record.due - logicalPerformanceNow()); + record.nativeId = native.setTimeout(() => { + record.nativeId = null; + if (state.paused || !state.intervals.has(record.id)) return; + try { + invoke(record); + } finally { + if (state.intervals.has(record.id)) { + record.due = logicalPerformanceNow() + record.delay; + scheduleInterval(record); + } + } + }, delay); + } + + function cancelRecord(records, id, clear) { + const record = records.get(id); + if (!record) return false; + if (record.nativeId !== null) clear(record.nativeId); + records.delete(id); + return true; + } + + window.requestAnimationFrame = (callback) => { + const id = state.nextId++; + const record = { id, callback, nativeId: null }; + state.frames.set(id, record); + scheduleFrame(record); + return id; + }; + window.cancelAnimationFrame = (id) => { + cancelRecord(state.frames, id, native.cancelAnimationFrame); + }; + + window.setTimeout = (callback, delay, ...args) => { + const id = state.nextId++; + const record = { + id, + callback, + args, + due: logicalPerformanceNow() + Math.max(0, Number(delay) || 0), + nativeId: null, + }; + state.timers.set(id, record); + scheduleTimer(record); + return id; + }; + window.clearTimeout = (id) => { + cancelRecord(state.timers, id, native.clearTimeout); + cancelRecord(state.intervals, id, native.clearInterval); + }; + + window.setInterval = (callback, delay, ...args) => { + const id = state.nextId++; + const interval = Math.max(1, Number(delay) || 0); + const record = { + id, + callback, + args, + delay: interval, + due: logicalPerformanceNow() + interval, + nativeId: null, + }; + state.intervals.set(id, record); + scheduleInterval(record); + return id; + }; + window.clearInterval = (id) => { + cancelRecord(state.intervals, id, native.clearInterval); + cancelRecord(state.timers, id, native.clearTimeout); + }; + + try { + Object.defineProperty(window.performance, 'now', { + configurable: true, + value: logicalPerformanceNow, + }); + } catch {} + try { + Date.now = logicalDateNow; + } catch {} + + const blockWhilePaused = (event) => { + if (!state.paused) return; + // Let release events through so a pause cannot leave a key or button held. + if (event.type === 'keyup' || event.type === 'mouseup' || event.type === 'pointerup') return; + event.preventDefault(); + event.stopImmediatePropagation(); + }; + for (const type of [ + 'keydown', 'keypress', 'mousedown', 'mousemove', 'click', 'contextmenu', + 'pointerdown', 'pointermove', 'wheel', 'touchstart', 'touchmove', + ]) { + window.addEventListener(type, blockWhilePaused, true); + } + + // Chromium emits a trusted relative movement followed immediately by its + // inverse when CDP moves a pointer-locked mouse. Games see both and the + // camera movement cancels out. The controller arms one expected movement; + // this early capture listener lets the trusted forward event reach the game + // and suppresses only Chromium's matching recenter event. + window.addEventListener('mousemove', (event) => { + if (event.isTrusted && native.performanceNow() < state.suppressPointerMovesUntil) { + event.preventDefault(); + event.stopImmediatePropagation(); + return; + } + const pending = state.pointerMove; + if (!pending || !event.isTrusted) return; + if (native.performanceNow() > pending.deadline) { + state.pointerMove = null; + return; + } + const x = Number(event.movementX || 0); + const y = Number(event.movementY || 0); + if (!pending.forwardSeen && x === pending.dx && y === pending.dy) { + pending.forwardSeen = true; + return; + } + if (pending.forwardSeen && x === -pending.dx && y === -pending.dy) { + state.pointerMove = null; + event.preventDefault(); + event.stopImmediatePropagation(); + } + }, true); + + function pause() { + if (state.paused) return false; + state.pausePerformanceNow = logicalPerformanceNow(); + state.pauseDateNow = logicalDateNow(); + state.paused = true; + for (const record of state.frames.values()) { + if (record.nativeId !== null) native.cancelAnimationFrame(record.nativeId); + record.nativeId = null; + } + for (const record of state.timers.values()) { + if (record.nativeId !== null) native.clearTimeout(record.nativeId); + record.nativeId = null; + } + for (const record of state.intervals.values()) { + if (record.nativeId !== null) native.clearTimeout(record.nativeId); + record.nativeId = null; + } + return true; + } + + function resume() { + if (!state.paused) return false; + state.performanceOffset = native.performanceNow() - state.pausePerformanceNow; + state.dateOffset = native.dateNow() - state.pauseDateNow; + state.paused = false; + for (const record of state.frames.values()) scheduleFrame(record); + for (const record of state.timers.values()) scheduleTimer(record); + for (const record of state.intervals.values()) { + // Do not replay every interval that elapsed during the model's turn. + record.due = logicalPerformanceNow() + record.delay; + scheduleInterval(record); + } + return true; + } + + window[name] = { + pause, + resume, + isPaused: () => state.paused, + preparePointerMove: (dx, dy) => { + if (!document.pointerLockElement) return false; + state.suppressPointerMovesUntil = 0; + state.pointerMove = { + dx: Number(dx), + dy: Number(dy), + forwardSeen: false, + deadline: native.performanceNow() + 15000, + }; + return true; + }, + preparePointerLock: () => { + state.pointerMove = null; + // Chromium may defer the lock-acquisition recenter event until the page + // receives another rendered frame. On a paused software-rendered game, + // that can be minutes later, after the agent has inspected the returned + // screenshot. Keep acquisition noise blocked until the controller arms + // the first intentional view movement, which clears this sentinel. + state.suppressPointerMovesUntil = Number.POSITIVE_INFINITY; + return true; + }, + }; +})(); +`; + +function browserPauseInitScript() { + return BROWSER_PAUSE_INIT_SCRIPT; +} + +module.exports = { + BROWSER_PAUSE_INIT_SCRIPT, + browserPauseInitScript, +}; diff --git a/runwave/controller/src/browser-session.js b/runwave/controller/src/browser-session.js index cc94c8d..e816a0a 100644 --- a/runwave/controller/src/browser-session.js +++ b/runwave/controller/src/browser-session.js @@ -6,7 +6,8 @@ const { chromium } = require('playwright'); const { AudioVideoRecorder } = require('./audio-recorder'); const { ensureDir, safeName, sleep, timestamp } = require('./file-utils'); const { drawGridOnScreenshot } = require('./grid-overlay'); -const { parseArgList, targetUrl } = require('./protocol'); +const { browserPauseInitScript } = require('./browser-pause'); +const { parseArgList, recordingBackend, targetUrl } = require('./protocol'); const { readPageState } = require('./state-reader'); const DEFAULT_CHROMIUM_ARGS = [ @@ -42,9 +43,13 @@ function isRecording(config = {}) { return Boolean(config.record || config.recordAudio); } +function usesGstreamerRecording(config = {}) { + return recordingBackend(config) === 'gstreamer'; +} + function chromiumLaunchArgs(config = {}, env = process.env) { const args = chromiumArgs(config, env); - if (!isRecording(config)) return args; + if (!usesGstreamerRecording(config)) return args; const size = videoSize(config); return [ ...args, @@ -57,7 +62,7 @@ function chromiumLaunchArgs(config = {}, env = process.env) { } function launchHeadless(config = {}) { - return isRecording(config) ? false : config.headless !== false; + return usesGstreamerRecording(config) ? false : config.headless !== false; } function webLaunchConfig(config = {}) { @@ -201,7 +206,10 @@ class BrowserSession { this.videoDir = null; this.audioDir = undefined; this.audioRecorder = null; + this.playwrightRecorderActive = false; + this.playwrightVideoPath = null; this.mousePosition = { x: 0, y: 0 }; + this.paused = false; } timeSync(event, fields, fn) { @@ -218,8 +226,12 @@ class BrowserSession { async start() { this.timeSync('browser.start.ensure_run_dir', { dir: this.paths.runDir }, () => ensureDir(this.paths.runDir)); - await this.time('browser.start.game_process', () => this.startGameProcess()); const record = isRecording(this.config); + const backend = recordingBackend(this.config); + if (backend === 'playwright' && this.config.recordAudio) { + throw new Error(`the ${backend} recording backend is video-only; use gstreamer when recordAudio is enabled`); + } + await this.time('browser.start.game_process', () => this.startGameProcess()); if (record) { this.videoDir = this.timeSync('browser.start.ensure_video_dir', () => ensureDir(path.join(this.paths.runDir, 'video'))); } @@ -245,6 +257,11 @@ class BrowserSession { deviceScaleFactor: Number(this.config.deviceScaleFactor ?? 1), }) ); + if (this.config.pauseController) { + await this.time('browser.start.install_pause_controller', () => + this.context.addInitScript({ content: browserPauseInitScript() }) + ); + } if (record) { await this.time('browser.start.capture_viewport_stabilizer', () => this.context.addInitScript(browserViewportStabilizerScript) @@ -257,7 +274,7 @@ class BrowserSession { await this.time('browser.start.initial_navigate', { url: this.launchUrl }, () => this.navigate({ url: this.launchUrl, waitAfterLoad: this.config.waitAfterLoad }) ); - if (record) { + if (backend === 'gstreamer') { const videoSource = this.config.videoSource || await this.time('browser.start.page_viewport_video_source', () => pageViewportVideoSource(this.page) ); @@ -267,7 +284,30 @@ class BrowserSession { this.profiler ? this.profiler.child('audio-video-recorder') : null ); await this.time('browser.start.audio_video_recorder_start', () => this.audioRecorder.start()); + } else if (backend === 'playwright') { + await this.time('browser.start.playwright_recorder_start', () => this.startPlaywrightRecording()); + } + } + + async startPlaywrightRecording() { + if (!this.page || !this.page.screencast || typeof this.page.screencast.start !== 'function') { + throw new Error('the installed Playwright version does not support page.screencast'); } + const fileName = safeName(this.config.playwrightVideoFileName || '000-runwave-playwright'); + this.playwrightVideoPath = path.join(this.videoDir, `${fileName}.webm`); + await this.page.screencast.start({ + path: this.playwrightVideoPath, + size: videoSize(this.config), + }); + this.playwrightRecorderActive = true; + return this.playwrightVideoPath; + } + + async stopPlaywrightRecording() { + if (!this.playwrightRecorderActive) return null; + this.playwrightRecorderActive = false; + await this.page.screencast.stop(); + return fs.existsSync(this.playwrightVideoPath) ? this.playwrightVideoPath : null; } async startGameProcess() { @@ -335,14 +375,27 @@ class BrowserSession { async click(click) { const holdMs = Math.max(0, Number(click.end ?? click.start) - Number(click.start ?? 0)); const clickCount = Math.max(1, Math.round(Number(click.clickCount || 1))); + const pointerLocked = await this.time('browser.mouse.pointer_lock_state', () => + this.page.evaluate(() => Boolean(document.pointerLockElement)) + ); await this.time('browser.mouse.click', { x: click.x, y: click.y, button: click.button, clickCount, holdMs, + pointerLocked, }, async () => { - await this.page.mouse.move(click.x, click.y); + // In pointer-lock games the click coordinates are irrelevant. Moving to + // them first makes Chromium emit a relative delta and its inverse while + // recentering, which can violently disturb an FPS camera before firing. + if (!pointerLocked) { + await this.page.mouse.move(click.x, click.y); + await this.page.evaluate(() => { + const controller = window.__runwavePauseController; + if (controller && controller.preparePointerLock) controller.preparePointerLock(); + }); + } for (let index = 0; index < clickCount; index += 1) { const eventClickCount = index + 1; await this.page.mouse.down({ button: click.button, clickCount: eventClickCount }); @@ -350,7 +403,49 @@ class BrowserSession { await this.page.mouse.up({ button: click.button, clickCount: eventClickCount }); } }); - this.mousePosition = { x: click.x, y: click.y }; + if (!pointerLocked) this.mousePosition = { x: click.x, y: click.y }; + } + + async focusGame({ button = 'middle', x: requestedX, y: requestedY } = {}) { + const alreadyLocked = await this.page.evaluate(() => Boolean(document.pointerLockElement)); + if (alreadyLocked) { + return { pointerLocked: true, x: this.mousePosition.x, y: this.mousePosition.y, button }; + } + const surface = await this.page.evaluate(() => { + const canvases = Array.from(document.querySelectorAll('canvas')); + const canvas = canvases.sort((left, right) => { + const leftRect = left.getBoundingClientRect(); + const rightRect = right.getBoundingClientRect(); + return rightRect.width * rightRect.height - leftRect.width * leftRect.height; + })[0]; + if (!canvas) return null; + const rect = canvas.getBoundingClientRect(); + return { left: rect.left, top: rect.top, right: rect.right, bottom: rect.bottom }; + }); + if (!surface) throw new Error('focus_game requires a visible canvas'); + + let x = Number.isFinite(Number(requestedX)) ? Number(requestedX) : this.mousePosition.x; + let y = Number.isFinite(Number(requestedY)) ? Number(requestedY) : this.mousePosition.y; + if (x < surface.left || x >= surface.right || y < surface.top || y >= surface.bottom) { + x = Math.round((surface.left + surface.right) / 2); + y = Math.round((surface.top + surface.bottom) / 2); + } + if (x !== this.mousePosition.x || y !== this.mousePosition.y) { + await this.page.mouse.move(x, y); + this.mousePosition = { x, y }; + } + await this.page.evaluate(() => { + const controller = window.__runwavePauseController; + if (!controller || !controller.preparePointerLock) { + throw new Error('pointer lock controller is unavailable'); + } + controller.preparePointerLock(); + }); + await this.page.mouse.down({ button }); + await this.page.mouse.up({ button }); + await sleep(75); + const pointerLocked = await this.page.evaluate(() => Boolean(document.pointerLockElement)); + return { pointerLocked, x, y, button }; } async moveCursor(move) { @@ -415,54 +510,41 @@ class BrowserSession { async moveView(move) { const viewport = this.page.viewportSize() || this.config.viewport || { width: 1024, height: 620 }; - const x = Math.max(0, Math.min(viewport.width - 1, this.mousePosition.x + move.dx)); - const y = Math.max(0, Math.min(viewport.height - 1, this.mousePosition.y + move.dy)); - await this.time('browser.mouse.move', { x, y, dx: move.dx, dy: move.dy }, () => this.page.mouse.move(x, y)); - await this.time('browser.mouse.dispatch_view_move_events', { x, y, dx: move.dx, dy: move.dy }, () => - this.page.evaluate(({ dx, dy, x, y }) => { - const target = document.pointerLockElement || document.activeElement || document.querySelector('canvas') || document.body; - const targets = Array.from(new Set([target, document, window].filter(Boolean))); - const eventInit = { - bubbles: true, - cancelable: true, - view: window, - clientX: x, - clientY: y, - screenX: x, - screenY: y, - movementX: dx, - movementY: dy, - buttons: 0, - }; - const defineMovement = (event) => { - for (const [name, value] of [ - ['movementX', dx], - ['movementY', dy], - ]) { - if (event[name] !== value) { - Object.defineProperty(event, name, { value, configurable: true }); - } - } - return event; - }; - - for (const eventTarget of targets) { - eventTarget.dispatchEvent(defineMovement(new MouseEvent('mousemove', eventInit))); - - if (typeof PointerEvent === 'function') { - eventTarget.dispatchEvent( - defineMovement( - new PointerEvent('pointermove', { - ...eventInit, - pointerId: 1, - pointerType: 'mouse', - isPrimary: true, - }) - ) - ); - } - } - }, { dx: move.dx, dy: move.dy, x, y }) + const pointerLocked = await this.time('browser.mouse.pointer_lock_state', () => + this.page.evaluate(() => Boolean(document.pointerLockElement)) + ); + const rawX = this.mousePosition.x + move.dx; + const rawY = this.mousePosition.y + move.dy; + if (pointerLocked) { + // Chromium produces a trusted requested delta and then a trusted inverse + // recenter event. The page-side controller suppresses that inverse, so + // games which reject synthetic MouseEvents still receive real movement. + await this.time('browser.mouse.dispatch_locked_view_move', { + dx: move.dx, + dy: move.dy, + requestedSteps: move.steps, + }, async () => { + const movementX = Math.round(Number(move.dx)); + const movementY = Math.round(Number(move.dy)); + const armed = await this.page.evaluate(({ x, y }) => { + const controller = window.__runwavePauseController; + return Boolean(controller && controller.preparePointerMove(x, y)); + }, { x: movementX, y: movementY }); + if (!armed) throw new Error('trusted pointer movement controller is unavailable'); + await this.page.mouse.move(movementX, movementY); + // Unity and other frame-polled games consume mouse deltas during their + // next animation update. On a software renderer that frame can take + // much longer than the requested action duration, so wait for the + // actual frame instead of pausing on a wall-clock guess. + await this.page.evaluate(() => new Promise((resolve) => requestAnimationFrame(() => resolve()))); + }); + return; + } + + const x = Math.max(0, Math.min(viewport.width - 1, rawX)); + const y = Math.max(0, Math.min(viewport.height - 1, rawY)); + await this.time('browser.mouse.move', { x, y, dx: move.dx, dy: move.dy }, () => + this.page.mouse.move(x, y, { steps: move.steps }) ); this.mousePosition = { x, y }; } @@ -473,6 +555,38 @@ class BrowserSession { ); } + // The MCP agent receives a frame and then may spend seconds reasoning. The + // page-side controller stops the game clock and input at the renderer, while + // leaving screenshots and state reads available during the pause. + async pause() { + if (!this.page) return false; + const changed = await this.page.evaluate(() => { + const controller = window.__runwavePauseController; + if (!controller) throw new Error('runwave pause controller is unavailable'); + return controller.pause(); + }); + this.paused = true; + return changed; + } + + async resume() { + if (!this.page) return false; + const changed = await this.page.evaluate(() => { + const controller = window.__runwavePauseController; + if (!controller) throw new Error('runwave pause controller is unavailable'); + return controller.resume(); + }); + this.paused = false; + return changed; + } + + async isPaused() { + if (!this.page) return Boolean(this.paused); + return this.page.evaluate(() => Boolean( + window.__runwavePauseController && window.__runwavePauseController.isPaused() + )); + } + async stopGameProcess() { if (!this.process || processHasClosed(this.process)) return; await this.time('browser.close.terminate_process', async () => { @@ -485,11 +599,16 @@ class BrowserSession { } async close() { + let videoPath = null; let audioVideoPath = null; let closeError = null; try { + if (this.playwrightRecorderActive) { + videoPath = await this.time('browser.close.playwright_recorder_stop', () => this.stopPlaywrightRecording()); + } if (this.audioRecorder) { audioVideoPath = await this.time('browser.close.audio_video_stop', () => this.audioRecorder.stop()); + videoPath = audioVideoPath; } if (this.context) await this.time('browser.close.context_close', () => this.context.close()); if (this.browser) await this.time('browser.close.browser_close', () => this.browser.close()); @@ -501,9 +620,10 @@ class BrowserSession { } catch (error) { if (!closeError) closeError = error; } + this.paused = false; if (closeError) throw closeError; return { - video: audioVideoPath, + video: videoPath, audioVideo: audioVideoPath || undefined, }; } @@ -516,5 +636,8 @@ module.exports = { chromiumLaunchArgs, launchHeadless, pageViewportVideoSource, + recordingBackend, + usesGstreamerRecording, + videoSize, webLaunchConfig, }; diff --git a/runwave/controller/src/protocol.js b/runwave/controller/src/protocol.js index 74c3763..0b77dd2 100644 --- a/runwave/controller/src/protocol.js +++ b/runwave/controller/src/protocol.js @@ -141,6 +141,21 @@ function optionalPositiveInteger(value) { return Number.isInteger(number) && number > 0 ? number : null; } +function recordingBackend(input = {}, env = process.env) { + if (!input.record && !input.recordAudio) return null; + const raw = String( + input.recordingBackend + ?? input.recording_backend + ?? input.recordBackend + ?? input.videoBackend + ?? env.RUNWAVE_RECORDING_BACKEND + ?? 'gstreamer' + ).trim().toLowerCase(); + if (raw === 'gstreamer' || raw === 'x11') return 'gstreamer'; + if (raw === 'playwright' || raw === 'playwright-native' || raw === 'native') return 'playwright'; + throw new Error(`unsupported recording backend: ${raw}`); +} + function linuxStartConfig(input = {}) { const launch = input.launch && typeof input.launch === 'object' ? input.launch : {}; const launchEnv = input.env ?? launch.env; @@ -182,12 +197,17 @@ function startSessionConfig(input, options = {}) { const kind = targetKind(input); const viewport = normalizeSize(input.viewport, { width: 1024, height: 620 }); const record = Boolean(input.record || input.recordAudio); + const backend = recordingBackend(input); + if (kind === 'linux' && backend && backend !== 'gstreamer') { + throw new Error(`the ${backend} recording backend is only available for web sessions`); + } const common = { kind, context: { viewport, deviceScaleFactor: optionalNumber(input.deviceScaleFactor, 1), record, + recordingBackend: backend, videoSize: record ? normalizeSize(input.videoSize || input.viewport, viewport) : null, }, defaults: { @@ -216,7 +236,7 @@ function startSessionConfig(input, options = {}) { launchUrl: web.launchUrl, web, browser: { - headless: record ? false : input.headless !== false, + headless: backend === 'gstreamer' ? false : input.headless !== false, channel: optionalString(input.channel), executablePath: optionalString(input.executablePath), chromiumArgsMode: String(input.chromiumArgsMode || process.env.RUNWAVE_CHROMIUM_ARGS_MODE || 'append').toLowerCase(), @@ -261,6 +281,7 @@ module.exports = { linuxStartConfig, webStartConfig, parseArgList, + recordingBackend, startSessionConfig, diffStartSessionConfig, isListSessionsAction, diff --git a/runwave/controller/src/step-executor.js b/runwave/controller/src/step-executor.js index bc0cb45..7159def 100644 --- a/runwave/controller/src/step-executor.js +++ b/runwave/controller/src/step-executor.js @@ -35,7 +35,7 @@ async function releasePressedKeys(browser, pressed, profiler) { } } -async function executeTimeline({ browser, events, duration, outputDir, prefix, stateExpression, profiler }) { +async function executeTimeline({ browser, events, duration, outputDir, prefix, stateExpression, beforeEndCapture, profiler }) { const pressed = new Set(); const captures = []; const startedAt = Date.now(); @@ -97,6 +97,7 @@ async function executeTimeline({ browser, events, duration, outputDir, prefix, s await browser.moveView(event.viewMove); } } else if (event.type === 'capture') { + if (beforeEndCapture && event.at === duration) await beforeEndCapture(); captures.push(await (profiler ? profiler.time('timeline.event.capture', fields, () => captureAt({ browser, outputDir, prefix, stateExpression, at: event.at, profiler }) diff --git a/runwave/controller/src/step-runner.js b/runwave/controller/src/step-runner.js index 2330802..5997596 100644 --- a/runwave/controller/src/step-runner.js +++ b/runwave/controller/src/step-runner.js @@ -27,7 +27,7 @@ function summarizeActions(step) { ].sort((left, right) => (left.start ?? 0) - (right.start ?? 0)); } -async function runStep({ input, config, browser, outputDir, nextStepIndex, actionName, profiler }) { +async function runStep({ input, config, browser, outputDir, nextStepIndex, actionName, beforeEndCapture, profiler }) { const timeSync = (event, fields, fn) => (profiler ? profiler.timeSync(event, fields, fn) : fn()); const time = (event, fields, fn) => (profiler ? profiler.time(event, fields, fn) : fn()); @@ -58,6 +58,7 @@ async function runStep({ input, config, browser, outputDir, nextStepIndex, actio outputDir, prefix, stateExpression: input.stateExpression, + beforeEndCapture, profiler: profiler ? profiler.child('step-executor') : null, }) ); diff --git a/runwave/controller/test/browser-session.test.js b/runwave/controller/test/browser-session.test.js index 9739ade..517b8de 100644 --- a/runwave/controller/test/browser-session.test.js +++ b/runwave/controller/test/browser-session.test.js @@ -1,5 +1,7 @@ const assert = require('node:assert/strict'); +const fs = require('node:fs'); const os = require('os'); +const path = require('node:path'); const test = require('node:test'); const { @@ -8,6 +10,8 @@ const { chromiumLaunchArgs, launchHeadless, pageViewportVideoSource, + recordingBackend, + usesGstreamerRecording, webLaunchConfig, } = require('../src/browser-session'); @@ -43,6 +47,71 @@ test('recording sessions force a visible headed browser', () => { assert.equal(launchHeadless({ record: false, headless: false }), false); }); +test('playwright recording stays headless and does not add x11 window arguments', () => { + const config = { + record: true, + recordingBackend: 'playwright', + viewport: { width: 1280, height: 720 }, + }; + + assert.equal(recordingBackend(config), 'playwright'); + assert.equal(usesGstreamerRecording(config), false); + assert.equal(launchHeadless(config), true); + assert.equal(chromiumLaunchArgs(config, {}).includes('--kiosk'), false); +}); + +test('recording backend rejects unknown values', () => { + assert.throws( + () => recordingBackend({ record: true, recordingBackend: 'unknown' }, {}), + /unsupported recording backend: unknown/ + ); +}); + +test('playwright recorder writes the configured viewport video and stops cleanly', async () => { + const runDir = fs.mkdtempSync(path.join(os.tmpdir(), 'runwave-playwright-recording-')); + const session = new BrowserSession( + { + url: 'about:blank', + record: true, + recordingBackend: 'playwright', + viewport: { width: 1280, height: 720 }, + }, + { runDir } + ); + const calls = []; + session.videoDir = path.join(runDir, 'video'); + fs.mkdirSync(session.videoDir); + session.page = { + screencast: { + start: async (options) => calls.push({ type: 'start', options }), + stop: async () => { + calls.push({ type: 'stop' }); + fs.writeFileSync(session.playwrightVideoPath, 'video'); + }, + }, + }; + + try { + const startedPath = await session.startPlaywrightRecording(); + const stoppedPath = await session.stopPlaywrightRecording(); + + assert.equal(startedPath, path.join(session.videoDir, '000-runwave-playwright.webm')); + assert.equal(stoppedPath, startedPath); + assert.deepEqual(calls, [ + { + type: 'start', + options: { + path: startedPath, + size: { width: 1280, height: 720 }, + }, + }, + { type: 'stop' }, + ]); + } finally { + fs.rmSync(runDir, { recursive: true, force: true }); + } +}); + test('browser launch config defaults game directories to start.sh', () => { assert.deepEqual(webLaunchConfig({ gameDir: '/tmp/web-game', port: 4123 }), { command: 'bash', @@ -82,6 +151,7 @@ test('browser clicks hold the mouse down for the normalized click interval', asy const session = new BrowserSession({ url: 'about:blank' }, { runDir: os.tmpdir() }); const calls = []; session.page = { + evaluate: async () => false, mouse: { move: async (x, y) => calls.push({ type: 'move', x, y }), down: async (options) => calls.push({ type: 'down', options, at: Date.now() }), @@ -99,3 +169,67 @@ test('browser clicks hold the mouse down for the normalized click interval', asy assert.ok(Date.now() - startedAt >= 45); assert.deepEqual(session.mousePosition, { x: 321, y: 222 }); }); + +test('pointer-locked clicks do not move the FPS camera before pressing the button', async () => { + const session = new BrowserSession({ url: 'about:blank' }, { runDir: os.tmpdir() }); + const calls = []; + session.mousePosition = { x: 400, y: 300 }; + session.page = { + evaluate: async () => true, + mouse: { + move: async (x, y) => calls.push({ type: 'move', x, y }), + down: async (options) => calls.push({ type: 'down', options }), + up: async (options) => calls.push({ type: 'up', options }), + }, + }; + + await session.click({ type: 'click', start: 0, end: 0, x: 640, y: 360, button: 'left', clickCount: 1 }); + + assert.deepEqual(calls.map((call) => call.type), ['down', 'up']); + assert.deepEqual(session.mousePosition, { x: 400, y: 300 }); +}); + +test('pointer-locked view movement dispatches trusted relative samples and suppresses recentering', async () => { + const session = new BrowserSession( + { url: 'about:blank', viewport: { width: 1280, height: 720 } }, + { runDir: os.tmpdir() } + ); + const moves = []; + const evaluations = []; + session.mousePosition = { x: 1270, y: 710 }; + session.page = { + viewportSize: () => ({ width: 1280, height: 720 }), + mouse: { move: async (x, y) => moves.push({ x, y }) }, + evaluate: async (fn, argument) => { + evaluations.push({ fn, argument }); + return true; + }, + }; + + await session.moveView({ dx: 100, dy: 50, steps: 8 }); + + assert.equal(moves.length, 1); + assert.equal(moves.reduce((sum, move) => sum + move.x, 0), 100); + assert.equal(moves.reduce((sum, move) => sum + move.y, 0), 50); + assert.deepEqual(evaluations[1].argument, { x: 100, y: 50 }); + assert.deepEqual(session.mousePosition, { x: 1270, y: 710 }); +}); + +test('unlocked view movement remains clamped to the viewport', async () => { + const session = new BrowserSession( + { url: 'about:blank', viewport: { width: 1280, height: 720 } }, + { runDir: os.tmpdir() } + ); + const moves = []; + session.mousePosition = { x: 1270, y: 710 }; + session.page = { + viewportSize: () => ({ width: 1280, height: 720 }), + mouse: { move: async (x, y) => moves.push({ x, y }) }, + evaluate: async () => false, + }; + + await session.moveView({ dx: 100, dy: 50 }); + + assert.deepEqual(moves, [{ x: 1279, y: 719 }]); + assert.deepEqual(session.mousePosition, { x: 1279, y: 719 }); +}); diff --git a/runwave/controller/test/session-cleanup.test.js b/runwave/controller/test/session-cleanup.test.js index 07586f7..ed821e5 100644 --- a/runwave/controller/test/session-cleanup.test.js +++ b/runwave/controller/test/session-cleanup.test.js @@ -215,6 +215,35 @@ test('web start configuration can launch a game directory through a local port', assert.equal(config.web.httpTimeoutMs, 60000); }); +test('playwright recording is represented as a headless web session', () => { + const config = startSessionConfig({ + action: 'start', + action_name: 'playwright-recording', + url: 'http://127.0.0.1:4123/', + viewport: { width: 1280, height: 720 }, + record: true, + recordingBackend: 'playwright', + }); + + assert.equal(config.context.record, true); + assert.equal(config.context.recordingBackend, 'playwright'); + assert.equal(config.browser.headless, true); +}); + +test('linux sessions reject the playwright-only recording backend', () => { + assert.throws( + () => startSessionConfig({ + action: 'start', + action_name: 'invalid-native-recording', + kind: 'linux', + command: './game', + record: true, + recordingBackend: 'playwright', + }), + /only available for web sessions/ + ); +}); + test('linux start rejects a live session with a different native launch command', async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'runwave-linux-target-mismatch-')); const originalStart = {