diff --git a/apps/desktop/src/ipc/methods/preview.test.ts b/apps/desktop/src/ipc/methods/preview.test.ts index 68ff5dbfef9b..400d34afb1dc 100644 --- a/apps/desktop/src/ipc/methods/preview.test.ts +++ b/apps/desktop/src/ipc/methods/preview.test.ts @@ -136,4 +136,81 @@ describe("preview IPC methods", () => { }), ).toThrow(); }); + + effectIt.effect("returns fulfilled automation press results for known keyboard failures", () => + Effect.gen(function* () { + const errors = [ + new PreviewManager.PreviewAutomationKeyboardWindowNotFocusedError({ + tabId: "tab-1", + webContentsId: 41, + }), + new PreviewManager.PreviewAutomationKeyboardFocusedFrameUnsupportedError({ + tabId: "tab-1", + webContentsId: 41, + }), + new PreviewManager.PreviewAutomationKeyboardDeliveryNotConfirmedError({ + tabId: "tab-1", + webContentsId: 41, + }), + new PreviewManager.PreviewAutomationTargetChangedError({ + operation: "press", + tabId: "tab-1", + webContentsId: 41, + }), + ]; + + for (const error of errors) { + const manager = PreviewManager.PreviewManager.of({ + automationPress: () => Effect.fail(error), + } as unknown as PreviewManager.PreviewManager["Service"]); + + expect( + yield* PreviewIpc.automationPress + .handler({ tabId: "tab-1", input: { key: "x" } }) + .pipe(Effect.provideService(PreviewManager.PreviewManager, manager)), + ).toEqual({ + _tag: "Failure", + error: { + _tag: error._tag, + ...(error._tag === "PreviewAutomationTargetChangedError" + ? { operation: error.operation } + : {}), + tabId: error.tabId, + webContentsId: error.webContentsId, + }, + }); + } + }), + ); + + effectIt.effect("returns a fulfilled automation press success", () => + Effect.gen(function* () { + const manager = PreviewManager.PreviewManager.of({ + automationPress: () => Effect.void, + } as unknown as PreviewManager.PreviewManager["Service"]); + + expect( + yield* PreviewIpc.automationPress + .handler({ tabId: "tab-1", input: { key: "x" } }) + .pipe(Effect.provideService(PreviewManager.PreviewManager, manager)), + ).toEqual({ _tag: "Success" }); + }), + ); + + effectIt.effect("rejects unrelated automation press failures", () => + Effect.gen(function* () { + const error = new PreviewManager.PreviewTabNotFoundError({ tabId: "tab-1" }); + const manager = PreviewManager.PreviewManager.of({ + automationPress: () => Effect.fail(error), + } as unknown as PreviewManager.PreviewManager["Service"]); + + const exit = yield* PreviewIpc.automationPress + .handler({ tabId: "tab-1", input: { key: "x" } }) + .pipe(Effect.provideService(PreviewManager.PreviewManager, manager), Effect.exit); + + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isSuccess(exit)) return; + expect(Cause.findErrorOption(exit.cause)).toEqual(Option.some(error)); + }), + ); }); diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 8a77770deb1e..5beb050c3db2 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -4,6 +4,7 @@ import { DesktopPreviewAutomationClickInputSchema, DesktopPreviewAutomationEvaluateInputSchema, DesktopPreviewAutomationPressInputSchema, + DesktopPreviewAutomationPressResultSchema, DesktopPreviewAutomationScrollInputSchema, DesktopPreviewAutomationStatusSchema, DesktopPreviewAutomationTypeInputSchema, @@ -377,10 +378,22 @@ export const automationType = DesktopIpc.makeIpcMethod({ export const automationPress = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PREVIEW_AUTOMATION_PRESS_CHANNEL, payload: DesktopPreviewAutomationPressInputSchema, - result: Schema.Void, + result: DesktopPreviewAutomationPressResultSchema, handler: Effect.fn("desktop.ipc.preview.automationPress")(function* ({ tabId, input }) { const manager = yield* PreviewManager.PreviewManager; - yield* manager.automationPress(tabId, input); + return yield* manager.automationPress(tabId, input).pipe( + Effect.as({ _tag: "Success" } as const), + Effect.catchTags({ + PreviewAutomationKeyboardWindowNotFocusedError: (error) => + Effect.succeed({ _tag: "Failure", error } as const), + PreviewAutomationKeyboardFocusedFrameUnsupportedError: (error) => + Effect.succeed({ _tag: "Failure", error } as const), + PreviewAutomationKeyboardDeliveryNotConfirmedError: (error) => + Effect.succeed({ _tag: "Failure", error } as const), + PreviewAutomationTargetChangedError: (error) => + Effect.succeed({ _tag: "Failure", error } as const), + }), + ); }), }); diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index d334b3635080..1b53a37bb54b 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -115,7 +115,6 @@ const { browserWindowConstructor, createFromPath, fromId, - getFocusedWebContents, mkdir, showItemInFolder, webviewSend, @@ -125,7 +124,6 @@ const { browserWindowConstructor: vi.fn(), createFromPath: vi.fn((): { readonly isEmpty: () => boolean } => ({ isEmpty: () => false })), fromId: vi.fn<(_id?: number) => Electron.WebContents | null>((_id?: number) => null), - getFocusedWebContents: vi.fn(() => null), mkdir: vi.fn((_path: string) => undefined), showItemInFolder: vi.fn(), webviewSend: vi.fn(), @@ -149,7 +147,6 @@ vi.mock("electron", () => ({ }, webContents: { fromId, - getFocusedWebContents, }, })); @@ -284,6 +281,302 @@ const makeTestPreviewWebContents = ( } as unknown as TestPreviewWebContents; }; +const keyboardInputFromPacket = (packet: Electron.KeyboardInputEvent): Electron.Input => { + const modifiers = new Set(packet.modifiers ?? []); + const shift = modifiers.has("shift"); + const keyCode = String(packet.keyCode); + const namedKeys: Readonly> = { + Enter: { key: "Enter", code: "Enter" }, + Escape: { key: "Escape", code: "Escape" }, + Backspace: { key: "Backspace", code: "Backspace" }, + Tab: { key: "Tab", code: "Tab" }, + Shift: { key: "Shift", code: "ShiftLeft" }, + Control: { key: "Control", code: "ControlLeft" }, + Alt: { key: "Alt", code: "AltLeft" }, + Meta: { key: "Meta", code: "MetaLeft" }, + Space: { key: " ", code: "Space" }, + Left: { key: "ArrowLeft", code: "ArrowLeft" }, + Right: { key: "ArrowRight", code: "ArrowRight" }, + Up: { key: "ArrowUp", code: "ArrowUp" }, + Down: { key: "ArrowDown", code: "ArrowDown" }, + }; + const printableKeys: Readonly< + Record + > = { + "`": { key: "`", shiftedKey: "~", code: "Backquote" }, + "1": { key: "1", shiftedKey: "!", code: "Digit1" }, + "2": { key: "2", shiftedKey: "@", code: "Digit2" }, + "3": { key: "3", shiftedKey: "#", code: "Digit3" }, + "4": { key: "4", shiftedKey: "$", code: "Digit4" }, + "5": { key: "5", shiftedKey: "%", code: "Digit5" }, + "6": { key: "6", shiftedKey: "^", code: "Digit6" }, + "7": { key: "7", shiftedKey: "&", code: "Digit7" }, + "8": { key: "8", shiftedKey: "*", code: "Digit8" }, + "9": { key: "9", shiftedKey: "(", code: "Digit9" }, + "0": { key: "0", shiftedKey: ")", code: "Digit0" }, + "-": { key: "-", shiftedKey: "_", code: "Minus" }, + "=": { key: "=", shiftedKey: "+", code: "Equal" }, + "\\": { key: "\\", shiftedKey: "|", code: "Backslash" }, + "[": { key: "[", shiftedKey: "{", code: "BracketLeft" }, + "]": { key: "]", shiftedKey: "}", code: "BracketRight" }, + ";": { key: ";", shiftedKey: ":", code: "Semicolon" }, + "'": { key: "'", shiftedKey: '"', code: "Quote" }, + ",": { key: ",", shiftedKey: "<", code: "Comma" }, + ".": { key: ".", shiftedKey: ">", code: "Period" }, + "/": { key: "/", shiftedKey: "?", code: "Slash" }, + }; + const named = namedKeys[keyCode]; + const printable = printableKeys[keyCode]; + const letter = /^[A-Z]$/.test(keyCode); + const key = + named?.key ?? + (printable ? (shift ? printable.shiftedKey : printable.key) : undefined) ?? + (letter && !shift ? keyCode.toLowerCase() : keyCode); + const code = named?.code ?? printable?.code ?? (letter ? `Key${keyCode}` : keyCode); + return { + type: packet.type === "keyUp" ? "keyUp" : "keyDown", + key, + code, + meta: modifiers.has("meta"), + shift, + control: modifiers.has("control") || modifiers.has("ctrl"), + alt: modifiers.has("alt"), + modifiers: packet.modifiers ?? [], + isAutoRepeat: false, + isComposing: false, + location: 0, + }; +}; + +const makeKeyboardWebContents = (options: { + readonly hostWebContents: Electron.WebContents; + readonly id?: number; + readonly initialFocusedFrame?: "main" | "child" | null; + readonly initialDevToolsOpened?: boolean; + readonly onHumanReceipt?: (phase: "down" | "up") => void; + readonly onIsDevToolsOpened?: () => void; + readonly onSendInputEvent?: (packet: Electron.KeyboardInputEvent) => void; + readonly onSetIgnoreMenuShortcuts?: (ignore: boolean) => void; + readonly sendCommand?: (method: string, params?: Record) => Promise; +}) => { + let beforeInput: ((event: Electron.Event, input: Electron.Input) => void) | undefined; + let humanInput: ((event: Electron.IpcMainEvent, signal: unknown) => void) | undefined; + let confirmDelivery = true; + let devToolsOpened = options.initialDevToolsOpened ?? false; + const activity: string[] = []; + let mainFrameProcessId = 100; + let mainFrameRoutingId = 200; + let mainFrameDetached = false; + const mainFrame = { + get detached() { + return mainFrameDetached; + }, + get processId() { + return mainFrameProcessId; + }, + get routingId() { + return mainFrameRoutingId; + }, + } as Electron.WebFrameMain; + const childFrame = { + detached: false, + processId: 101, + routingId: 201, + } as Electron.WebFrameMain; + let focusedFrame = + options.initialFocusedFrame === null + ? null + : options.initialFocusedFrame === "child" + ? childFrame + : mainFrame; + const focus = vi.fn(); + const off = vi.fn(); + const openDevTools = vi.fn(); + const reload = vi.fn(); + const setIgnoreMenuShortcuts = vi.fn((ignore: boolean) => { + activity.push(`menu:${ignore}`); + options.onSetIgnoreMenuShortcuts?.(ignore); + }); + const sendCommand = vi.fn( + options.sendCommand ?? + (async (method: string, params?: Record) => { + if (method !== "Runtime.evaluate") return undefined; + return { + result: { + value: + typeof params?.["expression"] === "string" && + params["expression"].includes("document.activeElement?.tagName") + ? false + : { ok: true }, + }, + }; + }), + ); + const sendInputEvent = vi.fn((packet: Electron.KeyboardInputEvent) => { + activity.push(`send:${packet.type}`); + options.onSendInputEvent?.(packet); + if (packet.type === "char") return; + const input = keyboardInputFromPacket(packet); + let prevented = false; + const event = { + preventDefault: vi.fn(() => { + prevented = true; + }), + } as unknown as Electron.Event; + activity.push(`before:${input.type}`); + beforeInput?.(event, input); + if (confirmDelivery && !prevented) { + queueMicrotask(() => { + const phase = packet.type === "keyUp" ? "up" : "down"; + activity.push(`receipt:${phase}`); + options.onHumanReceipt?.(phase); + humanInput?.( + { + sender: webContents, + senderFrame: mainFrame, + processId: mainFrameProcessId, + frameId: mainFrameRoutingId, + } as Electron.IpcMainEvent, + { + kind: "key", + phase, + key: input.key, + code: input.code, + meta: input.meta, + shift: input.shift, + control: input.control, + alt: input.alt, + }, + ); + }); + } + }); + const capturedImage = { + getSize: () => ({ width: 1, height: 1 }), + resize: () => capturedImage, + toPNG: () => Buffer.from("png"), + }; + const listeners = new Map void>(); + const webContents = { + id: options.id ?? 42, + hostWebContents: options.hostWebContents, + mainFrame, + get focusedFrame() { + return focusedFrame; + }, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => "https://example.com", + getTitle: () => "Example", + isLoading: () => false, + isDevToolsOpened: () => { + options.onIsDevToolsOpened?.(); + return devToolsOpened; + }, + focus, + reload, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + setAudioMuted: vi.fn(), + isCurrentlyAudible: () => false, + on: vi.fn((event: string, listener: (...args: never[]) => void) => { + if (event === "before-input-event") beforeInput = listener as typeof beforeInput; + listeners.set(event, listener); + }), + once: vi.fn(), + off, + ipc: { + on: vi.fn((channel: string, listener: typeof humanInput) => { + if (channel === "preview:human-input") humanInput = listener; + }), + off: vi.fn(), + }, + send: webviewSend, + sendInputEvent, + setIgnoreMenuShortcuts, + openDevTools, + capturePage: vi.fn(async () => capturedImage), + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + } as unknown as Electron.WebContents; + return { + activity, + focus, + off, + openDevTools, + reload, + sendCommand, + sendInputEvent, + setIgnoreMenuShortcuts, + webContents, + emitPhysicalInput(input: Electron.Input) { + const preventDefault = vi.fn(); + beforeInput?.({ preventDefault } as unknown as Electron.Event, input); + return preventDefault; + }, + emitHumanInput( + signal: unknown, + eventOptions?: { + readonly frame?: "main" | "child" | null; + readonly frameId?: number; + readonly processId?: number; + }, + ) { + const senderFrame = + eventOptions?.frame === null + ? null + : eventOptions?.frame === "child" + ? childFrame + : mainFrame; + humanInput?.( + { + sender: webContents, + senderFrame, + processId: eventOptions?.processId ?? senderFrame?.processId ?? -1, + frameId: eventOptions?.frameId ?? senderFrame?.routingId ?? -1, + } as Electron.IpcMainEvent, + signal, + ); + }, + emitNavigation(options?: { readonly processId?: number; readonly frameId?: number }) { + mainFrameProcessId = options?.processId ?? mainFrameProcessId + 1; + mainFrameRoutingId = options?.frameId ?? mainFrameRoutingId + 1; + listeners.get("did-navigate")?.(); + }, + emitNavigationStarted(options?: { + readonly isMainFrame?: boolean; + readonly isSameDocument?: boolean; + }) { + listeners.get("did-start-navigation")?.({ + isMainFrame: options?.isMainFrame ?? true, + isSameDocument: options?.isSameDocument ?? false, + } as never); + }, + emitInPageNavigation() { + listeners.get("did-navigate-in-page")?.(); + }, + setMainFrameDetached(value: boolean) { + mainFrameDetached = value; + }, + setConfirmDelivery(value: boolean) { + confirmDelivery = value; + }, + setDevToolsOpened(value: boolean) { + devToolsOpened = value; + }, + setFocusedFrame(value: "main" | "child" | null) { + focusedFrame = value === null ? null : value === "child" ? childFrame : mainFrame; + }, + }; +}; + /** Two ready tabs (41, 42) sharing one window, so they contend for the single display-media slot. */ const setupRecordingRaceTabs = (manager: PreviewManager.PreviewManager["Service"]) => Effect.gen(function* () { @@ -464,8 +757,6 @@ describe("PreviewManager", () => { beforeEach(() => { browserWindowConstructor.mockReset(); fromId.mockClear(); - getFocusedWebContents.mockReset(); - getFocusedWebContents.mockReturnValue(null); mkdir.mockClear(); writeFile.mockClear(); showItemInFolder.mockClear(); @@ -2486,6 +2777,23 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("clears an armed recording when its WebContents moves to another tab", () => + withManager((manager) => + Effect.gen(function* () { + const { grants, takeGrant } = yield* setupRecordingRaceTabs(manager); + + yield* manager.startRecording("tab_race_a"); + // The same guest is now owned by tab B. A late getDisplayMedia request + // from the old renderer must not receive tab A's armed stream. + yield* manager.registerWebview("tab_race_b", 41); + takeGrant(); + + expect(grants).toEqual([{}]); + yield* manager.stopRecording("tab_race_a"); + }), + ), + ); + effectIt.effect("continues native recording when the source warmup fails", () => withManager((manager) => Effect.gen(function* () { @@ -3499,176 +3807,739 @@ describe("PreviewManager", () => { ), ); - effectIt.effect("types in background webviews and enables native key input", () => + effectIt.effect("types through the page runtime without native text input", () => withManager((manager) => Effect.gen(function* () { - let failKeyDown = false; - let humanInput: ((_event: unknown, signal: unknown) => void) | undefined; - const sendCommand = vi.fn(async (method: string, params?: Record) => { - if ( - failKeyDown && - method === "Input.dispatchKeyEvent" && - (params?.["type"] === "keyDown" || params?.["type"] === "rawKeyDown") - ) { - throw new Error("key dispatch failed"); - } - if ( - method === "Input.dispatchKeyEvent" && - (params?.["type"] === "keyDown" || params?.["type"] === "rawKeyDown") - ) { - humanInput?.( - {}, - { - kind: "key", - key: params["key"], - code: params["code"] ?? "Digit1", - }, - ); - } - return method === "Runtime.evaluate" ? { result: { value: { ok: true } } } : undefined; - }); - const restoreFocus = vi.fn(); - const focus = vi.fn(); - getFocusedWebContents.mockReturnValue({ - id: 7, - isDestroyed: () => false, - focus: restoreFocus, - } as never); - fromId.mockReturnValue({ - id: 42, + const hostWebContents = { sendInputEvent: vi.fn() } as unknown as Electron.WebContents; + const guest = makeKeyboardWebContents({ hostWebContents }); + fromId.mockReturnValue(guest.webContents); + yield* manager.setMainWindow({ isDestroyed: () => false, - getType: () => "webview", - getURL: () => "https://example.com", - getTitle: () => "Example", - isLoading: () => false, - isDevToolsOpened: () => false, - focus, - getZoomFactor: () => 1, - setZoomFactor: vi.fn(), - setAudioMuted: vi.fn(), - isCurrentlyAudible: () => false, - on: vi.fn(), - off: vi.fn(), - ipc: { - on: vi.fn((channel: string, listener: typeof humanInput) => { - if (channel === "preview:human-input") humanInput = listener; - }), - off: vi.fn(), - }, - send: webviewSend, - navigationHistory: { canGoBack: () => false, canGoForward: () => false }, - setWindowOpenHandler: vi.fn(), - debugger: { - isAttached: () => false, - attach: vi.fn(), - sendCommand, - on: vi.fn(), - off: vi.fn(), - }, + isFocused: () => true, + once: vi.fn(), + webContents: hostWebContents, } as never); - yield* manager.createTab("tab_input"); yield* manager.registerWebview("tab_input", 42); - yield* manager.automationType("tab_input", { text: "hello", clear: true }); + + yield* manager.automationType("tab_input", { text: "hé🙂", clear: true }); yield* manager.automationType("tab_input", { text: "", clear: true }); - yield* manager.automationPress("tab_input", { key: "x" }); - const calls = sendCommand.mock.calls; + const calls = guest.sendCommand.mock.calls; const methods = calls.map(([method]) => method); - const enableIndex = methods.indexOf("Input.setIgnoreInputEvents"); - const focusOnIndex = calls.findIndex( - ([method, params]) => - method === "Emulation.setFocusEmulationEnabled" && params?.["enabled"] === true, - ); - const keyDownIndex = calls.findIndex( - ([method, params]) => - method === "Input.dispatchKeyEvent" && params?.["type"] === "keyDown", - ); - const keyUpIndex = calls.findIndex( - ([method, params]) => method === "Input.dispatchKeyEvent" && params?.["type"] === "keyUp", - ); - const focusOffIndex = calls.findIndex( - ([method, params]) => - method === "Emulation.setFocusEmulationEnabled" && params?.["enabled"] === false, - ); - const typeEvaluation = sendCommand.mock.calls.find( - ([method, params]) => - method === "Runtime.evaluate" && - typeof params === "object" && - params !== null && - "expression" in params && - typeof params.expression === "string" && - params.expression.includes('document.execCommand("insertText"'), - ); - expect(typeEvaluation).toBeDefined(); - const clearOnlyEvaluation = sendCommand.mock.calls.find( - ([method, params]) => - method === "Runtime.evaluate" && - typeof params === "object" && - params !== null && - "expression" in params && - typeof params.expression === "string" && - params.expression.includes('const text = ""') && - params.expression.includes("Object.getOwnPropertyDescriptor"), - ); - expect(clearOnlyEvaluation).toBeDefined(); - expect(methods).not.toContain("Input.insertText"); - expect(enableIndex).toBeGreaterThanOrEqual(0); - expect(focus).toHaveBeenCalledOnce(); - expect(restoreFocus).toHaveBeenCalledOnce(); - expect(methods).toContain("Page.bringToFront"); - expect(enableIndex).toBeLessThan(focusOnIndex); - expect(focusOnIndex).toBeLessThan(keyDownIndex); - expect(keyDownIndex).toBeLessThan(keyUpIndex); - expect(keyUpIndex).toBeLessThan(focusOffIndex); expect( - calls.filter( + calls.find( ([method, params]) => - method === "Input.dispatchKeyEvent" && params?.["type"] === "keyUp", + method === "Runtime.evaluate" && + typeof params?.["expression"] === "string" && + params["expression"].includes('const text = "hé🙂"') && + params["expression"].includes('document.execCommand("insertText"'), ), - ).toHaveLength(1); - expect(sendCommand).toHaveBeenCalledWith("Input.setIgnoreInputEvents", { ignore: false }); - - sendCommand.mockClear(); - failKeyDown = true; - const failedPress = yield* Effect.exit(manager.automationPress("tab_input", { key: "y" })); - - expect(Exit.isFailure(failedPress)).toBe(true); - expect(sendCommand).toHaveBeenCalledWith("Input.dispatchKeyEvent", { - type: "keyUp", - key: "y", - code: "KeyY", - modifiers: 0, - windowsVirtualKeyCode: 89, - location: 0, - isKeypad: false, - }); - expect(sendCommand).toHaveBeenCalledWith("Emulation.setFocusEmulationEnabled", { - enabled: false, - }); - expect(restoreFocus).toHaveBeenCalledTimes(2); + ).toBeDefined(); expect( - sendCommand.mock.calls.filter( + calls.find( ([method, params]) => - method === "Input.dispatchKeyEvent" && params?.["type"] === "keyUp", + method === "Runtime.evaluate" && + typeof params?.["expression"] === "string" && + params["expression"].includes('const text = ""') && + params["expression"].includes("Object.getOwnPropertyDescriptor"), ), - ).toHaveLength(1); + ).toBeDefined(); + expect(methods).not.toContain("Input.insertText"); + expect(guest.sendInputEvent).not.toHaveBeenCalled(); + }), + ), + ); + + effectIt.effect("sends native key packets to a never-focused guest and confirms delivery", () => + withManager((manager) => + Effect.gen(function* () { + const hostSendInputEvent = vi.fn(); + const hostFocus = vi.fn(); + const hostWebContents = { + focus: hostFocus, + sendInputEvent: hostSendInputEvent, + } as unknown as Electron.WebContents; + const guest = makeKeyboardWebContents({ hostWebContents }); + fromId.mockReturnValue(guest.webContents); + yield* manager.setMainWindow({ + isDestroyed: () => false, + isFocused: () => true, + once: vi.fn(), + webContents: hostWebContents, + } as never); + yield* manager.createTab("tab_input"); + yield* manager.registerWebview("tab_input", 42); + + yield* manager.automationPress("tab_input", { key: "x" }); + yield* manager.automationPress("tab_input", { key: "Enter" }); + yield* manager.automationPress("tab_input", { key: "z", modifiers: ["Meta"] }); + yield* manager.automationPress("tab_input", { key: "Escape" }); + yield* manager.automationPress("tab_input", { key: "Escape" }); + yield* manager.automationPress("tab_input", { key: "Escape" }); + + expect(guest.sendInputEvent.mock.calls.map(([packet]) => packet)).toEqual([ + { type: "rawKeyDown", keyCode: "X", modifiers: [] }, + { type: "char", keyCode: "x", modifiers: [] }, + { type: "keyUp", keyCode: "X", modifiers: [] }, + { type: "rawKeyDown", keyCode: "Enter", modifiers: [] }, + { type: "char", keyCode: "\r", modifiers: [] }, + { type: "keyUp", keyCode: "Enter", modifiers: [] }, + { type: "rawKeyDown", keyCode: "Z", modifiers: ["meta"] }, + { type: "keyUp", keyCode: "Z", modifiers: ["meta"] }, + { type: "rawKeyDown", keyCode: "Escape", modifiers: [] }, + { type: "keyUp", keyCode: "Escape", modifiers: [] }, + { type: "rawKeyDown", keyCode: "Escape", modifiers: [] }, + { type: "keyUp", keyCode: "Escape", modifiers: [] }, + { type: "rawKeyDown", keyCode: "Escape", modifiers: [] }, + { type: "keyUp", keyCode: "Escape", modifiers: [] }, + ]); + expect(guest.activity.slice(0, 9)).toEqual([ + "menu:true", + "send:rawKeyDown", + "before:keyDown", + "send:char", + "send:keyUp", + "before:keyUp", + "receipt:down", + "receipt:up", + "menu:false", + ]); + expect(guest.focus).not.toHaveBeenCalled(); + expect(hostFocus).not.toHaveBeenCalled(); + expect(hostSendInputEvent).not.toHaveBeenCalled(); + expect(guest.setIgnoreMenuShortcuts.mock.calls).toEqual( + Array.from({ length: 6 }, () => [[true], [false]]).flat(), + ); + const methods = guest.sendCommand.mock.calls.map(([method]) => method); + expect(methods).not.toContain("Input.dispatchKeyEvent"); + expect(methods).not.toContain("Page.bringToFront"); + expect(methods).not.toContain("Emulation.setFocusEmulationEnabled"); + }), + ), + ); + + effectIt.effect("confirms both phases of named modifier presses", () => + withManager((manager) => + Effect.gen(function* () { + const hostSendInputEvent = vi.fn(); + const hostWebContents = { + sendInputEvent: hostSendInputEvent, + } as unknown as Electron.WebContents; + const guest = makeKeyboardWebContents({ hostWebContents }); + fromId.mockReturnValue(guest.webContents); + yield* manager.setMainWindow({ + isDestroyed: () => false, + isFocused: () => true, + once: vi.fn(), + webContents: hostWebContents, + } as never); + yield* manager.createTab("tab_modifiers"); + yield* manager.registerWebview("tab_modifiers", 42); + + for (const key of ["Shift", "Control", "Alt", "Meta"] as const) { + yield* manager.automationPress("tab_modifiers", { key }); + } + yield* manager.automationPress("tab_modifiers", { key: "x" }); + + expect(guest.sendInputEvent.mock.calls.map(([packet]) => packet)).toEqual([ + { type: "rawKeyDown", keyCode: "Shift", modifiers: ["shift"] }, + { type: "keyUp", keyCode: "Shift", modifiers: [] }, + { type: "rawKeyDown", keyCode: "Control", modifiers: ["control"] }, + { type: "keyUp", keyCode: "Control", modifiers: [] }, + { type: "rawKeyDown", keyCode: "Alt", modifiers: ["alt"] }, + { type: "keyUp", keyCode: "Alt", modifiers: [] }, + { type: "rawKeyDown", keyCode: "Meta", modifiers: ["meta"] }, + { type: "keyUp", keyCode: "Meta", modifiers: [] }, + { type: "rawKeyDown", keyCode: "X", modifiers: [] }, + { type: "char", keyCode: "x", modifiers: [] }, + { type: "keyUp", keyCode: "X", modifiers: [] }, + ]); + expect(guest.activity.filter((event) => event.startsWith("receipt:"))).toEqual( + Array.from({ length: 5 }, () => ["receipt:down", "receipt:up"]).flat(), + ); + expect(guest.setIgnoreMenuShortcuts.mock.calls).toEqual( + Array.from({ length: 5 }, () => [[true], [false]]).flat(), + ); + expect(hostSendInputEvent).not.toHaveBeenCalled(); + expect(guest.reload).not.toHaveBeenCalled(); + }), + ), + ); + + effectIt.effect("does not quarantine a fully acknowledged key after human takeover", () => + withManager((manager) => + Effect.gen(function* () { + const hostFocus = vi.fn(); + const hostWebContents = { + focus: hostFocus, + sendInputEvent: vi.fn(), + } as unknown as Electron.WebContents; + let interruptAtNextTargetCheck = false; + let interrupted = false; + let guest: ReturnType; + guest = makeKeyboardWebContents({ + hostWebContents, + onHumanReceipt: (phase) => { + if (phase === "up") interruptAtNextTargetCheck = true; + }, + }); + const physicalInput = (type: "keyDown" | "keyUp"): Electron.Input => ({ + type, + key: "q", + code: "KeyQ", + meta: false, + shift: false, + control: false, + alt: false, + modifiers: [], + isAutoRepeat: false, + isComposing: false, + location: 0, + }); + const physicalSignal = (phase: "down" | "up") => ({ + kind: "key" as const, + phase, + key: "q", + code: "KeyQ", + meta: false, + shift: false, + control: false, + alt: false, + }); + fromId.mockImplementation(() => { + if (interruptAtNextTargetCheck && !interrupted) { + interrupted = true; + guest.emitPhysicalInput(physicalInput("keyDown")); + } + return guest.webContents; + }); + yield* manager.setMainWindow({ + isDestroyed: () => false, + isFocused: () => true, + once: vi.fn(), + webContents: hostWebContents, + } as never); + yield* manager.createTab("tab_receipt_race"); + yield* manager.registerWebview("tab_receipt_race", 42); + + const interruptedExit = yield* Effect.exit( + manager.automationPress("tab_receipt_race", { key: "x" }), + ); + expect(Exit.isFailure(interruptedExit)).toBe(true); + if (Exit.isFailure(interruptedExit)) { + expect(Option.getOrThrow(Cause.findErrorOption(interruptedExit.cause))).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + operation: "press", + }); + } + + // Both key phases were acknowledged before takeover, so a clean retry + // in the same document must not be blocked by delivery quarantine. + guest.emitPhysicalInput(physicalInput("keyUp")); + guest.emitHumanInput(physicalSignal("down")); + guest.emitHumanInput(physicalSignal("up")); + for (let attempt = 0; attempt < 3; attempt++) yield* Effect.yieldNow; + yield* manager.automationPress("tab_receipt_race", { key: "y" }); + expect(guest.sendInputEvent).toHaveBeenCalledTimes(6); + expect(guest.setIgnoreMenuShortcuts.mock.calls).toEqual([[true], [false], [true], [false]]); + expect(guest.focus).not.toHaveBeenCalled(); + expect(hostFocus).not.toHaveBeenCalled(); + }), + ), + ); + + effectIt.effect("rejects keyboard input when a child frame owns focus", () => + withManager((manager) => + Effect.gen(function* () { + const hostFocus = vi.fn(); + const hostWebContents = { + focus: hostFocus, + sendInputEvent: vi.fn(), + } as unknown as Electron.WebContents; + const guest = makeKeyboardWebContents({ + hostWebContents, + sendCommand: async (method, params) => { + if (method !== "Runtime.evaluate") return undefined; + return { + result: { + value: + typeof params?.["expression"] === "string" && + params["expression"].includes("document.activeElement?.tagName") + ? true + : { ok: true }, + }, + }; + }, + }); + fromId.mockReturnValue(guest.webContents); + yield* manager.setMainWindow({ + isDestroyed: () => false, + isFocused: () => true, + once: vi.fn(), + webContents: hostWebContents, + } as never); + yield* manager.createTab("tab_frame"); + yield* manager.registerWebview("tab_frame", 42); + + const exit = yield* Effect.exit(manager.automationPress("tab_frame", { key: "x" })); + + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({ + _tag: "PreviewAutomationKeyboardFocusedFrameUnsupportedError", + }); + } + expect(guest.sendInputEvent).not.toHaveBeenCalled(); + expect(guest.setIgnoreMenuShortcuts).not.toHaveBeenCalled(); + expect(guest.focus).not.toHaveBeenCalled(); + expect(hostFocus).not.toHaveBeenCalled(); + }), + ), + ); + + effectIt.effect("fails clearly when native keyboard delivery is unavailable", () => + withManager((manager) => + Effect.gen(function* () { + let focused = false; + const hostFocus = vi.fn(); + const hostWebContents = { + focus: hostFocus, + sendInputEvent: vi.fn(), + } as unknown as Electron.WebContents; + const guest = makeKeyboardWebContents({ hostWebContents }); + fromId.mockReturnValue(guest.webContents); + yield* manager.setMainWindow({ + isDestroyed: () => false, + isFocused: () => focused, + once: vi.fn(), + webContents: hostWebContents, + } as never); + yield* manager.createTab("tab_input"); + yield* manager.registerWebview("tab_input", 42); + guest.sendCommand.mockClear(); - sendCommand.mockClear(); - failKeyDown = false; - yield* manager.automationPress("tab_input", { key: "!" }); - expect(sendCommand).toHaveBeenCalledWith("Input.dispatchKeyEvent", { + const unfocused = yield* Effect.exit(manager.automationPress("tab_input", { key: "x" })); + expect(Exit.isFailure(unfocused)).toBe(true); + if (Exit.isFailure(unfocused)) { + expect(Option.getOrThrow(Cause.findErrorOption(unfocused.cause))).toMatchObject({ + _tag: "PreviewAutomationKeyboardWindowNotFocusedError", + }); + } + expect(guest.sendInputEvent).not.toHaveBeenCalled(); + expect(guest.setIgnoreMenuShortcuts).not.toHaveBeenCalled(); + expect(guest.sendCommand).not.toHaveBeenCalled(); + expect(guest.focus).not.toHaveBeenCalled(); + expect(hostFocus).not.toHaveBeenCalled(); + + focused = true; + yield* manager.automationPress("tab_input", { key: "x" }); + expect(guest.sendInputEvent).toHaveBeenCalledTimes(3); + expect(guest.setIgnoreMenuShortcuts.mock.calls).toEqual([[true], [false]]); + expect(guest.focus).not.toHaveBeenCalled(); + expect(hostFocus).not.toHaveBeenCalled(); + + guest.sendInputEvent.mockClear(); + guest.setIgnoreMenuShortcuts.mockClear(); + guest.setConfirmDelivery(false); + const unconfirmed = yield* manager + .automationPress("tab_input", { key: "x" }) + .pipe(Effect.exit, Effect.forkChild({ startImmediately: true })); + yield* settle(() => guest.sendInputEvent.mock.calls.length === 3); + yield* TestClock.adjust(1_000); + const unconfirmedExit = yield* Fiber.join(unconfirmed); + expect(Exit.isFailure(unconfirmedExit)).toBe(true); + if (Exit.isFailure(unconfirmedExit)) { + expect(Option.getOrThrow(Cause.findErrorOption(unconfirmedExit.cause))).toMatchObject({ + _tag: "PreviewAutomationKeyboardDeliveryNotConfirmedError", + }); + } + expect(guest.setIgnoreMenuShortcuts.mock.calls).toEqual([[true]]); + + const sentPackets = guest.sendInputEvent.mock.calls.length; + const quarantined = yield* Effect.exit(manager.automationPress("tab_input", { key: "y" })); + expect(Exit.isFailure(quarantined)).toBe(true); + if (Exit.isFailure(quarantined)) { + expect(Option.getOrThrow(Cause.findErrorOption(quarantined.cause))).toMatchObject({ + _tag: "PreviewAutomationKeyboardDeliveryNotConfirmedError", + }); + } + expect(guest.sendInputEvent).toHaveBeenCalledTimes(sentPackets); + + const physicalKey = guest.emitPhysicalInput({ type: "keyDown", - key: "!", - code: "Digit1", - modifiers: 0, - windowsVirtualKeyCode: 49, + key: "a", + code: "KeyA", + meta: false, + shift: false, + control: false, + alt: false, + modifiers: [], + isAutoRepeat: false, + isComposing: false, location: 0, - isKeypad: false, - text: "!", - unmodifiedText: "!", }); - expect(restoreFocus).toHaveBeenCalledTimes(3); + expect(physicalKey).not.toHaveBeenCalled(); + expect(guest.setIgnoreMenuShortcuts.mock.calls).toEqual([[true], [false]]); + expect(guest.focus).not.toHaveBeenCalled(); + expect(hostFocus).not.toHaveBeenCalled(); + }), + ), + ); + + effectIt.effect("restores menu shortcuts when raw key-down dispatch throws", () => + withManager((manager) => + Effect.gen(function* () { + let failDispatch = true; + const hostFocus = vi.fn(); + const hostWebContents = { + focus: hostFocus, + sendInputEvent: vi.fn(), + } as unknown as Electron.WebContents; + const guest = makeKeyboardWebContents({ + hostWebContents, + onSendInputEvent: (packet) => { + if (failDispatch && packet.type === "rawKeyDown") { + failDispatch = false; + throw new Error("dispatch failed"); + } + }, + }); + fromId.mockReturnValue(guest.webContents); + yield* manager.setMainWindow({ + isDestroyed: () => false, + isFocused: () => true, + once: vi.fn(), + webContents: hostWebContents, + } as never); + yield* manager.createTab("tab_input"); + yield* manager.registerWebview("tab_input", 42); + + const failed = yield* Effect.exit(manager.automationPress("tab_input", { key: "x" })); + expect(Exit.isFailure(failed)).toBe(true); + expect(guest.sendInputEvent.mock.calls.map(([packet]) => packet.type)).toEqual([ + "rawKeyDown", + ]); + expect(guest.setIgnoreMenuShortcuts.mock.calls).toEqual([[true], [false]]); + + yield* manager.automationPress("tab_input", { key: "x" }); + expect(guest.sendInputEvent.mock.calls.map(([packet]) => packet.type)).toEqual([ + "rawKeyDown", + "rawKeyDown", + "char", + "keyUp", + ]); + expect(guest.setIgnoreMenuShortcuts.mock.calls).toEqual([[true], [false], [true], [false]]); + expect(guest.focus).not.toHaveBeenCalled(); + expect(hostFocus).not.toHaveBeenCalled(); + }), + ), + ); + + effectIt.effect("recovers uncertain keyboard delivery only after committed navigation", () => + withManager((manager) => + Effect.gen(function* () { + const hostFocus = vi.fn(); + const hostWebContents = { + focus: hostFocus, + sendInputEvent: vi.fn(), + } as unknown as Electron.WebContents; + const guest = makeKeyboardWebContents({ hostWebContents }); + guest.setConfirmDelivery(false); + fromId.mockReturnValue(guest.webContents); + yield* manager.setMainWindow({ + isDestroyed: () => false, + isFocused: () => true, + once: vi.fn(), + webContents: hostWebContents, + } as never); + yield* manager.createTab("tab_input"); + yield* manager.registerWebview("tab_input", 42); + + const failed = yield* manager + .automationPress("tab_input", { key: "x" }) + .pipe(Effect.exit, Effect.forkChild({ startImmediately: true })); + yield* settle(() => guest.sendInputEvent.mock.calls.length === 3); + yield* TestClock.adjust(1_000); + expect(Exit.isFailure(yield* Fiber.join(failed))).toBe(true); + + guest.emitNavigationStarted(); + guest.emitInPageNavigation(); + for (const key of ["y", "z"] as const) { + const blocked = yield* Effect.exit(manager.automationPress("tab_input", { key })); + expect(Exit.isFailure(blocked)).toBe(true); + } + expect(guest.sendInputEvent).toHaveBeenCalledTimes(3); + expect(guest.setIgnoreMenuShortcuts.mock.calls).toEqual([[true]]); + + guest.emitNavigation(); + guest.setConfirmDelivery(true); + yield* manager.automationPress("tab_input", { key: "y" }); + + expect(guest.sendInputEvent).toHaveBeenCalledTimes(6); + expect(guest.setIgnoreMenuShortcuts.mock.calls).toEqual([[true], [false], [true], [false]]); + }), + ), + ); + + effectIt.effect("keeps keyboard input isolated to the selected guest", () => + withManager((manager) => + Effect.gen(function* () { + const hostFocus = vi.fn(); + const hostWebContents = { + focus: hostFocus, + sendInputEvent: vi.fn(), + } as unknown as Electron.WebContents; + const first = makeKeyboardWebContents({ hostWebContents, id: 41 }); + const second = makeKeyboardWebContents({ hostWebContents, id: 42 }); + const webContentsById = new Map([ + [41, first.webContents], + [42, second.webContents], + ]); + fromId.mockImplementation((id) => + id === undefined ? null : (webContentsById.get(id) ?? null), + ); + yield* manager.setMainWindow({ + isDestroyed: () => false, + isFocused: () => true, + once: vi.fn(), + webContents: hostWebContents, + } as never); + yield* manager.createTab("tab_first"); + yield* manager.createTab("tab_second"); + yield* manager.registerWebview("tab_first", 41); + yield* manager.registerWebview("tab_second", 42); + + yield* Effect.all( + [ + manager.automationPress("tab_first", { key: "x" }), + manager.automationPress("tab_second", { key: "y" }), + ], + { concurrency: 2, discard: true }, + ); + + expect(first.sendInputEvent).toHaveBeenCalledTimes(3); + expect(second.sendInputEvent).toHaveBeenCalledTimes(3); + expect(first.focus).not.toHaveBeenCalled(); + expect(second.focus).not.toHaveBeenCalled(); + expect(hostFocus).not.toHaveBeenCalled(); + }), + ), + ); + + effectIt.effect("rejects a same-id guest replacement while keyboard input is queued", () => + withManager((manager) => + Effect.gen(function* () { + const hostWebContents = { sendInputEvent: vi.fn() } as unknown as Electron.WebContents; + let startBlockedEvaluate = false; + let releaseEvaluate: (() => void) | undefined; + let reportEvaluateStarted: (() => void) | undefined; + const evaluateStarted = new Promise((resolve) => { + reportEvaluateStarted = resolve; + }); + const evaluateRelease = new Promise((resolve) => { + releaseEvaluate = resolve; + }); + const first = makeKeyboardWebContents({ + hostWebContents, + sendCommand: async (method) => { + if (method === "Runtime.evaluate" && startBlockedEvaluate) { + reportEvaluateStarted?.(); + await evaluateRelease; + return { result: { value: { ok: true } } }; + } + return method === "Runtime.evaluate" ? { result: { value: { ok: true } } } : undefined; + }, + }); + const replacement = makeKeyboardWebContents({ hostWebContents }); + let currentWebContents = first.webContents; + fromId.mockImplementation(() => currentWebContents); + yield* manager.setMainWindow({ + isDestroyed: () => false, + isFocused: () => true, + once: vi.fn(), + webContents: hostWebContents, + } as never); + yield* manager.createTab("tab_input"); + yield* manager.registerWebview("tab_input", 42); + + startBlockedEvaluate = true; + const active = yield* manager + .automationEvaluate("tab_input", { expression: "blocked" }) + .pipe(Effect.exit, Effect.forkChild({ startImmediately: true })); + yield* Effect.promise(() => evaluateStarted); + const queued = yield* manager + .automationPress("tab_input", { key: "y" }) + .pipe(Effect.exit, Effect.forkChild({ startImmediately: true })); + for (let attempt = 0; attempt < 3; attempt++) yield* Effect.yieldNow; + + currentWebContents = replacement.webContents; + yield* manager.registerWebview("tab_input", 42); + releaseEvaluate?.(); + yield* Fiber.join(active); + const queuedExit = yield* Fiber.join(queued); + + expect(Exit.isFailure(queuedExit)).toBe(true); + if (Exit.isFailure(queuedExit)) { + expect(Option.getOrThrow(Cause.findErrorOption(queuedExit.cause))).toMatchObject({ + _tag: "PreviewAutomationTargetChangedError", + operation: "press", + tabId: "tab_input", + webContentsId: 42, + }); + } + expect(first.sendInputEvent).not.toHaveBeenCalled(); + expect(replacement.sendInputEvent).not.toHaveBeenCalled(); + }), + ), + ); + + effectIt.effect("rechecks a same-id replacement at the native send boundary", () => + withManager((manager) => + Effect.gen(function* () { + const hostWebContents = { sendInputEvent: vi.fn() } as unknown as Electron.WebContents; + const replacement = makeKeyboardWebContents({ hostWebContents }); + let currentWebContents: Electron.WebContents; + const first = makeKeyboardWebContents({ + hostWebContents, + onSetIgnoreMenuShortcuts: (ignore) => { + if (ignore) currentWebContents = replacement.webContents; + }, + }); + currentWebContents = first.webContents; + fromId.mockImplementation(() => currentWebContents); + yield* manager.setMainWindow({ + isDestroyed: () => false, + isFocused: () => true, + once: vi.fn(), + webContents: hostWebContents, + } as never); + yield* manager.createTab("tab_input"); + yield* manager.registerWebview("tab_input", 42); + + const exit = yield* Effect.exit(manager.automationPress("tab_input", { key: "x" })); + + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({ + _tag: "PreviewAutomationTargetChangedError", + operation: "press", + tabId: "tab_input", + webContentsId: 42, + }); + } + expect(first.sendInputEvent).not.toHaveBeenCalled(); + expect(replacement.sendInputEvent).not.toHaveBeenCalled(); + expect(first.setIgnoreMenuShortcuts.mock.calls).toEqual([[true], [false]]); + }), + ), + ); + + effectIt.effect("does not dispatch a key after a human pointer claims control", () => + withManager((manager) => + Effect.gen(function* () { + const hostWebContents = { sendInputEvent: vi.fn() } as unknown as Electron.WebContents; + let pointerInjected = false; + let guest: ReturnType; + guest = makeKeyboardWebContents({ + hostWebContents, + onSetIgnoreMenuShortcuts: (ignore) => { + if (!ignore || pointerInjected) return; + pointerInjected = true; + guest.emitHumanInput({ kind: "pointer", x: 12, y: 24, button: 0 }); + }, + }); + fromId.mockReturnValue(guest.webContents); + yield* manager.setMainWindow({ + isDestroyed: () => false, + isFocused: () => true, + once: vi.fn(), + webContents: hostWebContents, + } as never); + yield* manager.createTab("tab_pointer_before_key"); + yield* manager.registerWebview("tab_pointer_before_key", 42); + + const exit = yield* Effect.exit( + manager.automationPress("tab_pointer_before_key", { key: "x" }), + ); + yield* TestClock.adjust(750); + + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + operation: "press", + }); + } + expect(guest.sendInputEvent).not.toHaveBeenCalled(); + expect(guest.setIgnoreMenuShortcuts.mock.calls).toEqual([[true], [false]]); + }), + ), + ); + + effectIt.effect("rejects queued keyboard input after physical input takes control", () => + withManager((manager) => + Effect.gen(function* () { + const hostWebContents = { sendInputEvent: vi.fn() } as unknown as Electron.WebContents; + let releaseEvaluate: (() => void) | undefined; + let reportEvaluateStarted: (() => void) | undefined; + const evaluateStarted = new Promise((resolve) => { + reportEvaluateStarted = resolve; + }); + const evaluateRelease = new Promise((resolve) => { + releaseEvaluate = resolve; + }); + const guest = makeKeyboardWebContents({ + hostWebContents, + sendCommand: async (method, params) => { + if (method === "Runtime.evaluate" && params?.["expression"] === "blocked") { + reportEvaluateStarted?.(); + await evaluateRelease; + return { result: { value: { ok: true } } }; + } + return method === "Runtime.evaluate" ? { result: { value: { ok: true } } } : undefined; + }, + }); + fromId.mockReturnValue(guest.webContents); + let humanHasControl = false; + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + if (state.controller === "human") humanHasControl = true; + }), + ); + yield* manager.setMainWindow({ + isDestroyed: () => false, + isFocused: () => true, + once: vi.fn(), + webContents: hostWebContents, + } as never); + yield* manager.createTab("tab_input"); + yield* manager.registerWebview("tab_input", 42); + + const active = yield* manager + .automationEvaluate("tab_input", { expression: "blocked" }) + .pipe(Effect.exit, Effect.forkChild({ startImmediately: true })); + yield* Effect.promise(() => evaluateStarted); + const queued = yield* manager + .automationPress("tab_input", { key: "x" }) + .pipe(Effect.exit, Effect.forkChild({ startImmediately: true })); + for (let attempt = 0; attempt < 3; attempt++) yield* Effect.yieldNow; + guest.emitHumanInput({ kind: "pointer", x: 12, y: 24, button: 0 }); + yield* settle(() => humanHasControl); + releaseEvaluate?.(); + yield* Fiber.join(active); + const queuedExit = yield* Fiber.join(queued); + + expect(Exit.isFailure(queuedExit)).toBe(true); + if (Exit.isFailure(queuedExit)) { + expect(Option.getOrThrow(Cause.findErrorOption(queuedExit.cause))).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + operation: "press", + tabId: "tab_input", + webContentsId: 42, + }); + } + expect(guest.sendInputEvent).not.toHaveBeenCalled(); }), ), ); @@ -3676,7 +4547,13 @@ describe("PreviewManager", () => { effectIt.effect("still interrupts agent control for a different human pointer event", () => withManager((manager) => Effect.gen(function* () { - let humanInput: ((_event: unknown, signal: unknown) => void) | undefined; + const mainFrame = { + detached: false, + processId: 100, + routingId: 200, + } as Electron.WebFrameMain; + let guestWebContents: Electron.WebContents; + let humanInput: ((event: Electron.IpcMainEvent, signal: unknown) => void) | undefined; const sendCommand = vi.fn(async (method: string) => { if (method === "Runtime.evaluate") { return { @@ -3686,12 +4563,21 @@ describe("PreviewManager", () => { }; } if (method === "Input.dispatchMouseEvent") { - humanInput?.({}, { kind: "pointer", x: 400, y: 300, button: 0 }); + humanInput?.( + { + sender: guestWebContents, + senderFrame: mainFrame, + processId: mainFrame.processId, + frameId: mainFrame.routingId, + } as Electron.IpcMainEvent, + { kind: "pointer", x: 400, y: 300, button: 0 }, + ); } return undefined; }); - fromId.mockReturnValue({ + guestWebContents = { id: 42, + mainFrame, isDestroyed: () => false, getType: () => "webview", getURL: () => "https://example.com", @@ -3720,7 +4606,8 @@ describe("PreviewManager", () => { on: vi.fn(), off: vi.fn(), }, - } as never); + } as never; + fromId.mockReturnValue(guestWebContents); yield* manager.createTab("tab_1"); yield* manager.registerWebview("tab_1", 42); @@ -3747,7 +4634,6 @@ describe("PreviewManager", () => { }), ), ); - effectIt.effect("derives evaluation detail kind and length from the same non-empty source", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 8af2a460fb3e..c0d286da194d 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -122,6 +122,7 @@ const DIAGNOSTIC_BUFFER_LIMIT = 200; const MAX_ARTIFACT_SITE_SLUG_LENGTH = 80; const AGENT_CURSOR_MOVE_MS = 160; const AGENT_CURSOR_CLICK_LEAD_MS = 40; +const AGENT_KEY_RECEIPT_TIMEOUT_MS = 1_000; const requestRecordingCaptureExpression = (tabId: string): string => `globalThis[${JSON.stringify(DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER)}]?.(${JSON.stringify(tabId)}) === true`; const encodeUnknownJson = Schema.encodeUnknownEffect(Schema.fromJsonString(Schema.Unknown)); @@ -396,15 +397,32 @@ const nextZoomLevel = (current: number, direction: "in" | "out"): number => { type Listener = (tabId: string, state: PreviewTabState) => Effect.Effect; type RecordingFrameListener = (frame: DesktopPreviewRecordingFrame) => Effect.Effect; +interface PreviewKeyIdentity { + readonly kind: "key"; + readonly key: string; + readonly code: string; + readonly meta: boolean; + readonly shift: boolean; + readonly control: boolean; + readonly alt: boolean; +} + type PreviewInputSignal = | { readonly kind: "pointer"; readonly x: number; readonly y: number; readonly button: number } - | { readonly kind: "key"; readonly key: string; readonly code: string }; + | (PreviewKeyIdentity & { readonly phase: "down" | "up" }); interface ManagedListeners { readonly attachmentId: symbol; readonly cancelFaviconCapture: () => void; + readonly pendingAgentKeys: Set; + readonly pendingKeyInputs: Array; readonly scope: Scope.Closeable; + readonly tabId: string; readonly webContents: Electron.WebContents; + active: boolean; + documentGeneration: number; + keyboardDeliveryUncertainGeneration: number | undefined; + menuShortcutLease: symbol | undefined; } type FrameCaptureConsumer = "picture-in-picture" | "recording"; @@ -437,8 +455,12 @@ interface PickSession { interface BrowserControlSession { readonly webContentsId: number; + readonly webContents: Electron.WebContents; + readonly attachmentId: symbol; + readonly tabId: string; readonly semaphore: Semaphore.Semaphore; readonly scope: Scope.Closeable; + active: boolean; readonly onMessage: ( event: Electron.Event, method: string, @@ -455,8 +477,36 @@ interface BrowserDiagnostics { type PointerEventListener = (event: DesktopPreviewPointerEvent) => Effect.Effect; interface ExpectedAgentInput { + readonly id: symbol; readonly signal: PreviewInputSignal; readonly expiresAt: number; + readonly attachmentId?: symbol; + readonly nativeKey?: ActiveAgentKey; + readonly receipt?: Deferred.Deferred; +} + +interface KeyboardDocumentIdentity { + readonly frame: Electron.WebFrameMain; + readonly frameId: number; + readonly generation: number; + readonly processId: number; +} + +interface ActiveAgentKey { + readonly attachmentId: symbol; + readonly document: KeyboardDocumentIdentity; + readonly id: symbol; + readonly phase: "down" | "up"; + readonly signal: PreviewKeyIdentity; + readonly webContents: Electron.WebContents; + accepted: boolean; + valid: boolean; +} + +interface PendingKeyInput { + readonly document: KeyboardDocumentIdentity; + readonly nativeKey?: ActiveAgentKey; + readonly signal: PreviewInputSignal & { readonly kind: "key" }; } const APP_FORWARDED_SHORTCUTS: ReadonlyArray<{ @@ -464,15 +514,16 @@ const APP_FORWARDED_SHORTCUTS: ReadonlyArray<{ meta: boolean; shift: boolean; control: boolean; + alt: boolean; }> = Object.freeze([ // mod+shift+J → preview.toggle - { key: "j", meta: true, shift: true, control: false }, + { key: "j", meta: true, shift: true, control: false, alt: false }, // mod+K → command palette - { key: "k", meta: true, shift: false, control: false }, + { key: "k", meta: true, shift: false, control: false, alt: false }, // mod+, → settings (macOS convention) - { key: ",", meta: true, shift: false, control: false }, + { key: ",", meta: true, shift: false, control: false, alt: false }, // mod+W → close tab/panel - { key: "w", meta: true, shift: false, control: false }, + { key: "w", meta: true, shift: false, control: false, alt: false }, ]); /** @@ -554,7 +605,17 @@ const isPreviewInputSignal = (value: unknown): value is PreviewInputSignal => { "key" in value && typeof value.key === "string" && "code" in value && - typeof value.code === "string" + typeof value.code === "string" && + "meta" in value && + typeof value.meta === "boolean" && + "shift" in value && + typeof value.shift === "boolean" && + "control" in value && + typeof value.control === "boolean" && + "alt" in value && + typeof value.alt === "boolean" && + "phase" in value && + (value.phase === "down" || value.phase === "up") ); }; @@ -571,10 +632,68 @@ const inputSignalsMatch = (left: PreviewInputSignal, right: PreviewInputSignal): left.kind === "key" && right.kind === "key" && left.key === right.key && - left.code === right.code + left.code === right.code && + left.meta === right.meta && + left.shift === right.shift && + left.control === right.control && + left.alt === right.alt && + left.phase === right.phase ); }; +const inputMatchesActiveAgentKey = (expected: ActiveAgentKey, input: Electron.Input): boolean => + input.type === (expected.phase === "down" ? "keyDown" : "keyUp") && + input.key === expected.signal.key && + input.code === expected.signal.code && + input.meta === expected.signal.meta && + input.shift === expected.signal.shift && + input.control === expected.signal.control && + input.alt === expected.signal.alt; + +const keySignalFromInput = ( + input: Electron.Input & { readonly type: "keyDown" | "keyUp" }, +): PreviewInputSignal & { readonly kind: "key" } => ({ + kind: "key", + phase: input.type === "keyDown" ? "down" : "up", + key: input.key, + code: input.code, + meta: input.meta, + shift: input.shift, + control: input.control, + alt: input.alt, +}); + +const captureKeyboardDocument = ( + attachment: ManagedListeners, + wc: Electron.WebContents, +): KeyboardDocumentIdentity => { + const frame = wc.mainFrame; + return { + frame, + frameId: frame.routingId, + generation: attachment.documentGeneration, + processId: frame.processId, + }; +}; + +const isKeyboardDocumentCurrent = ( + attachment: ManagedListeners, + wc: Electron.WebContents, + document: KeyboardDocumentIdentity, +): boolean => + attachment.documentGeneration === document.generation && + isKeyboardDocumentFrameCurrent(wc, document); + +const isKeyboardDocumentFrameCurrent = ( + wc: Electron.WebContents, + document: KeyboardDocumentIdentity, +): boolean => + !wc.isDestroyed() && + wc.mainFrame === document.frame && + !document.frame.detached && + document.frame.processId === document.processId && + document.frame.routingId === document.frameId; + const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function* ( artifactDirectory: string, pictureInPicturePreloadPath: string, @@ -621,7 +740,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function >(new Map()); const pictureInPictureAspectRatiosRef = yield* Ref.make>(new Map()); const pictureInPictureMutationSemaphore = yield* Semaphore.make(1); + const webviewRegistrationSemaphore = yield* Semaphore.make(1); const closingTabIdsRef = yield* Ref.make>(new Set()); + const activeAgentKeys = new Map(); + const uncertainKeyboardDocuments = new WeakMap(); // Tab recording uses `setDisplayMediaRequestHandler` because Electron's legacy // `getMediaSourceId` + `chromeMediaSource: "tab"` capture path was removed upstream // (electron#44618) and now always rejects with NotAllowedError. @@ -649,6 +771,27 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function try: evaluate, catch: (cause) => new PreviewOperationError({ ...errorContext, cause }), }); + const releaseMenuShortcutLease = ( + attachment: ManagedListeners, + wc: Electron.WebContents, + expectedLease?: symbol, + ): void => { + const lease = attachment.menuShortcutLease; + if (lease === undefined || (expectedLease !== undefined && lease !== expectedLease)) return; + attachment.menuShortcutLease = undefined; + if (wc.isDestroyed()) return; + try { + wc.setIgnoreMenuShortcuts(false); + } catch (cause) { + runFork( + Effect.logDebug("Failed to restore preview menu shortcuts.", { + cause, + tabId: attachment.tabId, + webContentsId: wc.id, + }), + ); + } + }; const currentIso = DateTime.now.pipe(Effect.map(DateTime.formatIso)); const currentMillis = Clock.currentTimeMillis; const encodeJson = (errorContext: PreviewOperationContext, value: unknown) => @@ -961,7 +1104,14 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return yield* new PreviewWebviewNotInitializedError({ tabId }); } const wc = webContents.fromId(tab.webContentsId); - if (!wc) { + const attachment = (yield* Ref.get(attachedRef)).get(tab.webContentsId); + if ( + !wc || + wc.isDestroyed() || + !attachment?.active || + attachment.tabId !== tabId || + attachment.webContents !== wc + ) { return yield* new PreviewWebContentsNotFoundError({ tabId, webContentsId: tab.webContentsId, @@ -997,10 +1147,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); const tabIdForWebContents = Effect.fnUntraced(function* (webContentsId: number) { - const tabs = yield* SynchronizedRef.get(tabsRef); - return ( - Array.from(tabs.entries()).find(([, tab]) => tab.webContentsId === webContentsId)?.[0] ?? null - ); + const attachment = (yield* Ref.get(attachedRef)).get(webContentsId); + return attachment?.active ? attachment.tabId : null; }); const pushBounded = (buffer: ReadonlyArray, entry: A): ReadonlyArray => @@ -1136,17 +1284,35 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const detachControlSession = Effect.fn("PreviewManager.detachControlSession")(function* ( webContentsId: number, + expected?: { + readonly attachmentId: symbol; + readonly tabId: string; + readonly webContents: Electron.WebContents; + }, ) { - const control = yield* SynchronizedRef.modify(controlSessionsRef, (sessions) => [ - sessions.get(webContentsId), - replaceMap(sessions, (copy) => { - copy.delete(webContentsId); - }), - ]); + const control = yield* SynchronizedRef.modify(controlSessionsRef, (sessions) => { + const current = sessions.get(webContentsId); + if ( + expected && + (current?.attachmentId !== expected.attachmentId || + current.tabId !== expected.tabId || + current.webContents !== expected.webContents) + ) { + return [undefined, sessions] as const; + } + if (current) current.active = false; + return [ + current, + replaceMap(sessions, (copy) => { + copy.delete(webContentsId); + }), + ]; + }); if (control) { yield* Scope.close(control.scope, Exit.void).pipe(Effect.ignore); return; } + if (expected) return; yield* Ref.update(diagnosticsRef, (diagnostics) => replaceMap(diagnostics, (copy) => { copy.delete(webContentsId); @@ -1156,7 +1322,27 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const ensureControlSession = Effect.fn("PreviewManager.ensureControlSession")(function* ( wc: Electron.WebContents, + tabId: string, + operation: string, + expectedAttachment?: ManagedListeners, ) { + const attachment = expectedAttachment ?? (yield* Ref.get(attachedRef)).get(wc.id); + const currentAttachment = (yield* Ref.get(attachedRef)).get(wc.id); + if ( + !attachment || + !attachment.active || + attachment.webContents !== wc || + attachment.tabId !== tabId || + currentAttachment?.attachmentId !== attachment.attachmentId || + currentAttachment?.tabId !== tabId || + currentAttachment?.webContents !== wc + ) { + return yield* new PreviewAutomationTargetChangedError({ + operation, + tabId, + webContentsId: wc.id, + }); + } return yield* SynchronizedRef.modifyEffect( controlSessionsRef, ( @@ -1166,7 +1352,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function PreviewManagerError > => { const existing = sessions.get(wc.id); - if (existing) return Effect.succeed([existing, sessions] as const); + if ( + attachment.active && + existing?.active && + existing.webContents === wc && + existing.attachmentId === attachment.attachmentId && + existing.tabId === tabId + ) { + return Effect.succeed([existing, sessions] as const); + } + const currentSessions = existing + ? replaceMap(sessions, (copy) => { + copy.delete(wc.id); + }) + : sessions; if (wc.isDevToolsOpened()) { return Effect.fail( new PreviewAutomationDevToolsOpenError({ @@ -1182,6 +1381,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); } const createControlSession = Effect.fn("PreviewManager.createControlSession")(function* () { + if (existing) { + existing.active = false; + yield* Scope.close(existing.scope, Exit.void).pipe(Effect.ignore); + } const semaphore = yield* Semaphore.make(1); const scope = yield* Scope.fork(parentScope, "sequential"); const handleDebuggerMessage = Effect.fnUntraced(function* ( @@ -1253,8 +1456,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); const control: BrowserControlSession = { webContentsId: wc.id, + webContents: wc, + attachmentId: attachment.attachmentId, + tabId, semaphore, scope, + active: true, onMessage, }; const initialize = Effect.fn("PreviewManager.initializeControlSession")(function* () { @@ -1281,12 +1488,35 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ), { concurrency: "unbounded", discard: true }, ); - return [ - control, - replaceMap(sessions, (copy) => { - copy.set(wc.id, control); - }), - ] as const; + const [tabs, attachments] = yield* Effect.all([ + SynchronizedRef.get(tabsRef), + Ref.get(attachedRef), + ]); + return yield* Effect.suspend(() => { + if ( + !attachment.active || + !control.active || + wc.isDestroyed() || + webContents.fromId(wc.id) !== wc || + tabs.get(tabId)?.webContentsId !== wc.id || + attachments.get(wc.id) !== attachment || + attachment.tabId !== tabId + ) { + return Effect.fail( + new PreviewAutomationTargetChangedError({ + operation, + tabId, + webContentsId: wc.id, + }), + ); + } + return Effect.succeed([ + control, + replaceMap(currentSessions, (copy) => { + copy.set(wc.id, control); + }), + ] as const); + }); }); return yield* initialize().pipe( Effect.onError(() => Scope.close(scope, Exit.void).pipe(Effect.ignore)), @@ -1337,7 +1567,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, wc: Electron.WebContents, action: string, - use: (send: SendCommand, sendCleanup: SendCommand) => Effect.Effect, + use: ( + send: SendCommand, + assertCurrent: Effect.Effect, + attachment: ManagedListeners, + isNativeTargetCurrent: () => boolean, + ) => Effect.Effect, ) { const sequence = yield* nextCounter(actionSequenceRef); const startedAt = yield* currentIso; @@ -1349,55 +1584,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function startedAt, }; yield* pushAction(tabId, actionEvent); - const epoch = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0; - const control = yield* ensureControlSession(wc); - const execute = Effect.fn("PreviewManager.executeControlAction")(function* () { - yield* update(tabId, { controller: "agent" }); - const send: SendCommand = Effect.fn("PreviewManager.sendCommand")( - function* (method, commandParams) { - const before = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0; - if (before !== epoch) { - return yield* new PreviewAutomationControlInterruptedError({ - operation: action, - tabId, - webContentsId: wc.id, - }); - } - const result = yield* attemptPromise( - { operation: `${action}.${method}`, tabId, webContentsId: wc.id }, - () => wc.debugger.sendCommand(method, commandParams), - ); - const after = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0; - if (after !== epoch) { - return yield* new PreviewAutomationControlInterruptedError({ - operation: action, - tabId, - webContentsId: wc.id, - }); - } - return result; - }, - ); - // Cleanup commands must still run after human input invalidates the action's - // control epoch. Otherwise a partially dispatched input can leave Chromium - // with a held key or focus emulation enabled for subsequent actions. - const sendCleanup: SendCommand = Effect.fn("PreviewManager.sendCleanupCommand")( - function* (method, commandParams) { - return yield* attemptPromise( - { - operation: `${action}.cleanup.${method}`, - tabId, - webContentsId: wc.id, - }, - () => wc.debugger.sendCommand(method, commandParams), - ); - }, - ); - return yield* use(send, sendCleanup); - }); + let controllerClaimed = false; + let finalized = false; const finalize = Effect.fn("PreviewManager.finalizeControlAction")(function* ( exit: Exit.Exit, ) { + if (finalized) return; + finalized = true; const completedAt = yield* currentIso; if (exit._tag === "Success") { yield* replaceAction(tabId, { @@ -1425,9 +1618,81 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); } const tabs = yield* SynchronizedRef.get(tabsRef); - if (tabs.has(tabId)) yield* update(tabId, { controller: "none" }); + if (controllerClaimed && tabs.get(tabId)?.controller === "agent") { + yield* update(tabId, { controller: "none" }); + } }); - return yield* control.semaphore.withPermit(execute().pipe(Effect.onExit(finalize))); + const run = Effect.gen(function* () { + const epoch = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0; + const attachment = (yield* Ref.get(attachedRef)).get(wc.id); + if (!attachment || !attachment.active || attachment.webContents !== wc) { + return yield* new PreviewAutomationTargetChangedError({ + operation: action, + tabId, + webContentsId: wc.id, + }); + } + const control = yield* ensureControlSession(wc, tabId, action, attachment); + const assertCurrent = Effect.fn("PreviewManager.assertCurrentControlTarget")(function* () { + const [tabs, attachments, controls, epochs] = yield* Effect.all([ + SynchronizedRef.get(tabsRef), + Ref.get(attachedRef), + SynchronizedRef.get(controlSessionsRef), + Ref.get(controlEpochRef), + ]); + if ((epochs.get(tabId) ?? 0) !== epoch) { + return yield* new PreviewAutomationControlInterruptedError({ + operation: action, + tabId, + webContentsId: wc.id, + }); + } + if ( + wc.isDestroyed() || + tabs.get(tabId)?.webContentsId !== wc.id || + webContents.fromId(wc.id) !== wc || + attachments.get(wc.id)?.attachmentId !== attachment.attachmentId || + attachments.get(wc.id)?.tabId !== tabId || + attachments.get(wc.id)?.webContents !== wc || + !attachment.active || + controls.get(wc.id) !== control || + !control.active || + control.webContents !== wc || + control.attachmentId !== attachment.attachmentId || + control.tabId !== tabId + ) { + return yield* new PreviewAutomationTargetChangedError({ + operation: action, + tabId, + webContentsId: wc.id, + }); + } + }); + const execute = Effect.fn("PreviewManager.executeControlAction")(function* () { + yield* assertCurrent(); + yield* update(tabId, { controller: "agent" }); + controllerClaimed = true; + const send: SendCommand = Effect.fn("PreviewManager.sendCommand")( + function* (method, commandParams) { + yield* assertCurrent(); + const result = yield* attemptPromise( + { operation: `${action}.${method}`, tabId, webContentsId: wc.id }, + () => wc.debugger.sendCommand(method, commandParams), + ); + yield* assertCurrent(); + return result; + }, + ); + const isNativeTargetCurrent = () => + attachment.active && + control.active && + !wc.isDestroyed() && + webContents.fromId(wc.id) === wc; + return yield* use(send, assertCurrent(), attachment, isNativeTargetCurrent); + }); + return yield* control.semaphore.withPermit(execute().pipe(Effect.onExit(finalize))); + }); + return yield* run.pipe(Effect.onExit((exit) => (finalized ? Effect.void : finalize(exit)))); }); const evaluateWithDebugger = ( @@ -1512,15 +1777,60 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (session) yield* session.cancel; }); + const rememberKeyboardDeliveryHazard = (attachment: ManagedListeners): void => { + const wc = attachment.webContents; + const hasPotentialHazard = + attachment.keyboardDeliveryUncertainGeneration === attachment.documentGeneration || + attachment.menuShortcutLease !== undefined || + attachment.pendingKeyInputs.length > 0 || + Array.from(attachment.pendingAgentKeys).some((pending) => pending.accepted); + if (!hasPotentialHazard || wc.isDestroyed()) return; + const document = captureKeyboardDocument(attachment, wc); + const hasCurrentPendingInput = attachment.pendingKeyInputs.some((pending) => + isKeyboardDocumentCurrent(attachment, wc, pending.document), + ); + const hasAcceptedAgentKey = Array.from(attachment.pendingAgentKeys).some( + (pending) => pending.accepted && isKeyboardDocumentCurrent(attachment, wc, pending.document), + ); + if ( + attachment.keyboardDeliveryUncertainGeneration === attachment.documentGeneration || + attachment.menuShortcutLease !== undefined || + hasCurrentPendingInput || + hasAcceptedAgentKey + ) { + uncertainKeyboardDocuments.set(wc, document); + } + }; + + const markKeyboardDeliveryUncertain = ( + attachment: ManagedListeners, + wc: Electron.WebContents, + document: KeyboardDocumentIdentity, + ): void => { + if (!isKeyboardDocumentCurrent(attachment, wc, document)) return; + attachment.keyboardDeliveryUncertainGeneration = document.generation; + if (attachment.active) uncertainKeyboardDocuments.set(wc, document); + for (const pending of attachment.pendingAgentKeys) pending.valid = false; + }; + const detachListeners = Effect.fn("PreviewManager.detachListeners")(function* ( webContentsId: number, + expected?: ManagedListeners, ) { - const managed = yield* Ref.modify(attachedRef, (attached) => [ - attached.get(webContentsId), - replaceMap(attached, (copy) => { - copy.delete(webContentsId); - }), - ]); + const managed = yield* Ref.modify(attachedRef, (attached) => { + const current = attached.get(webContentsId); + if (expected && current !== expected) return [undefined, attached] as const; + if (current) { + current.active = false; + rememberKeyboardDeliveryHazard(current); + } + return [ + current, + replaceMap(attached, (copy) => { + copy.delete(webContentsId); + }), + ]; + }); if (managed) { managed.cancelFaviconCapture(); yield* Scope.close(managed.scope, Exit.void).pipe(Effect.ignore); @@ -1534,7 +1844,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function shortcut.key.toLowerCase() === input.key.toLowerCase() && shortcut.meta === input.meta && shortcut.shift === input.shift && - shortcut.control === input.control, + shortcut.control === input.control && + shortcut.alt === input.alt, ); const computeNavStatus = (wc: Electron.WebContents): PreviewNavStatus => { @@ -1546,39 +1857,80 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }; const consumeExpectedAgentInput = Effect.fn("PreviewManager.consumeExpectedAgentInput")( - function* (tabId: string, signal: PreviewInputSignal) { + function* ( + tabId: string, + attachmentId: symbol, + signal: PreviewInputSignal, + nativeKey?: ActiveAgentKey, + ) { const now = yield* currentMillis; - return yield* Ref.modify(expectedAgentInputsRef, (allExpected) => { + const matched = yield* Ref.modify(expectedAgentInputsRef, (allExpected) => { const pending = (allExpected.get(tabId) ?? []).filter( (expected) => expected.expiresAt > now, ); - const index = pending.findIndex((expected) => inputSignalsMatch(expected.signal, signal)); - const matched = index >= 0; - const nextPending = matched + const index = pending.findIndex( + (expected) => + (expected.attachmentId === undefined || expected.attachmentId === attachmentId) && + expected.nativeKey === nativeKey && + (nativeKey === undefined || (nativeKey.valid && nativeKey.accepted)) && + inputSignalsMatch(expected.signal, signal), + ); + const match = pending[index]; + const nextPending = match ? pending.filter((_, pendingIndex) => pendingIndex !== index) : pending; return [ - matched, + match, replaceMap(allExpected, (copy) => { if (nextPending.length === 0) copy.delete(tabId); else copy.set(tabId, nextPending); }), ] as const; }); + if (matched?.receipt) yield* Deferred.succeed(matched.receipt, undefined).pipe(Effect.ignore); + return matched !== undefined; }, ); const expectAgentInput = Effect.fn("PreviewManager.expectAgentInput")(function* ( tabId: string, signal: PreviewInputSignal, + options?: { + readonly attachmentId?: symbol; + readonly nativeKey?: ActiveAgentKey; + readonly receipt?: Deferred.Deferred; + }, ) { const now = yield* currentMillis; + const id = Symbol(); yield* Ref.update(expectedAgentInputsRef, (allExpected) => replaceMap(allExpected, (copy) => { const pending = (allExpected.get(tabId) ?? []).filter( (expected) => expected.expiresAt > now, ); - copy.set(tabId, [...pending, { signal, expiresAt: now + 1_000 }]); + copy.set(tabId, [ + ...pending, + { + id, + signal, + expiresAt: now + AGENT_KEY_RECEIPT_TIMEOUT_MS, + ...options, + }, + ]); + }), + ); + return id; + }); + + const removeExpectedAgentInput = Effect.fn("PreviewManager.removeExpectedAgentInput")(function* ( + tabId: string, + id: symbol, + ) { + yield* Ref.update(expectedAgentInputsRef, (allExpected) => + replaceMap(allExpected, (copy) => { + const pending = (allExpected.get(tabId) ?? []).filter((expected) => expected.id !== id); + if (pending.length === 0) copy.delete(tabId); + else copy.set(tabId, pending); }), ); }); @@ -1602,6 +1954,26 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function activeCapture?.controller.abort(); activeCapture = null; }; + const uncertainKeyboardDocument = uncertainKeyboardDocuments.get(wc); + const inheritsKeyboardDeliveryHazard = + uncertainKeyboardDocument !== undefined && + isKeyboardDocumentFrameCurrent(wc, uncertainKeyboardDocument); + if (uncertainKeyboardDocument && !inheritsKeyboardDeliveryHazard) { + uncertainKeyboardDocuments.delete(wc); + } + const attachment: ManagedListeners = { + attachmentId, + cancelFaviconCapture, + pendingAgentKeys: new Set(), + pendingKeyInputs: [], + scope, + tabId, + webContents: wc, + active: true, + documentGeneration: 0, + keyboardDeliveryUncertainGeneration: inheritsKeyboardDeliveryHazard ? 0 : undefined, + menuShortcutLease: undefined, + }; const syncState = Effect.fn("PreviewManager.syncWebContentsState")(function* ( preserveLoadFailure: boolean, confirmedNavigation = false, @@ -1651,7 +2023,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (Option.isSome(next)) yield* emitIfCurrent(tabId, next.value); }); const sync = () => runFork(syncState(true)); - const syncNavigation = () => runFork(syncState(false, true)); + const syncNavigation = () => { + if (!attachment.active) return; + releaseMenuShortcutLease(attachment, wc); + uncertainKeyboardDocuments.delete(wc); + attachment.documentGeneration += 1; + attachment.pendingKeyInputs.length = 0; + for (const pending of attachment.pendingAgentKeys) pending.valid = false; + runFork(syncState(false, true)); + }; const syncInPageNavigation = () => runFork(syncState(false)); const navigationStarted = ( event: Electron.Event, @@ -1774,12 +2154,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }), ); }; - const handleHumanInput = Effect.fn("PreviewManager.handleHumanInput")(function* ( - rawSignal?: unknown, - ) { - if (isPreviewInputSignal(rawSignal) && (yield* consumeExpectedAgentInput(tabId, rawSignal))) { - return; - } + const claimHumanControl = Effect.fn("PreviewManager.claimHumanControl")(function* () { + for (const pending of attachment.pendingAgentKeys) pending.valid = false; yield* Ref.update(controlEpochRef, (epochs) => replaceMap(epochs, (copy) => { copy.set(tabId, (epochs.get(tabId) ?? 0) + 1); @@ -1792,8 +2168,81 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function yield* update(tabId, { controller: "none" }); } }); - const humanInput = (_event: unknown, rawSignal?: unknown): void => { - runFork(handleHumanInput(rawSignal)); + const handleHumanInput = Effect.fn("PreviewManager.handleHumanInput")(function* ( + rawSignal?: unknown, + ) { + if (isPreviewInputSignal(rawSignal)) { + if (yield* consumeExpectedAgentInput(tabId, attachmentId, rawSignal)) return; + if (rawSignal.kind === "key" && rawSignal.phase === "up") return; + } + yield* claimHumanControl(); + }); + const humanInput = (event: Electron.IpcMainEvent, rawSignal?: unknown): void => { + const senderFrame = event.senderFrame; + const processId = event.processId; + const frameId = event.frameId; + if ( + event.sender !== wc || + !senderFrame || + senderFrame.detached || + senderFrame !== wc.mainFrame || + processId !== senderFrame.processId || + frameId !== senderFrame.routingId || + !isPreviewInputSignal(rawSignal) + ) { + return; + } + if (rawSignal.kind === "pointer") { + runFork(handleHumanInput(rawSignal)); + return; + } + const matchesReceipt = (pending: PendingKeyInput): boolean => + isKeyboardDocumentCurrent(attachment, wc, pending.document) && + pending.document.processId === processId && + pending.document.frameId === frameId && + inputSignalsMatch(pending.signal, rawSignal); + const pending = attachment.pendingKeyInputs[0]; + if (!pending || !matchesReceipt(pending)) { + if (rawSignal.phase === "up") { + const physicalMatchIndex = attachment.pendingKeyInputs.findIndex((candidate) => { + if (candidate.nativeKey) return false; + return matchesReceipt(candidate); + }); + const agentIndex = attachment.pendingKeyInputs.findIndex( + (candidate) => candidate.nativeKey !== undefined, + ); + if (physicalMatchIndex >= 0 && (agentIndex === -1 || physicalMatchIndex < agentIndex)) { + attachment.pendingKeyInputs.splice(0, physicalMatchIndex + 1); + return; + } + } + const pendingAgentDocument = + attachment.pendingKeyInputs.find( + (candidate) => + candidate.nativeKey !== undefined && + isKeyboardDocumentCurrent(attachment, wc, candidate.document), + )?.document ?? + Array.from(attachment.pendingAgentKeys).find((candidate) => + isKeyboardDocumentCurrent(attachment, wc, candidate.document), + )?.document; + if (pendingAgentDocument) { + markKeyboardDeliveryUncertain(attachment, wc, pendingAgentDocument); + } + attachment.pendingKeyInputs.length = 0; + if (rawSignal.phase === "down") runFork(claimHumanControl()); + return; + } + attachment.pendingKeyInputs.shift(); + if (!pending.nativeKey) return; + runFork( + consumeExpectedAgentInput(tabId, attachmentId, rawSignal, pending.nativeKey).pipe( + Effect.tap((matched) => + matched + ? Effect.void + : Effect.sync(() => markKeyboardDeliveryUncertain(attachment, wc, pending.document)), + ), + ), + ); }; const mouseNavigate = (_event: unknown, payload?: unknown): void => { const direction = @@ -1811,16 +2260,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }).pipe(Effect.ignore), ); }; - const forwardShortcut = Effect.fn("PreviewManager.forwardShortcut")(function* ( - event: Electron.Event, - input: Electron.Input, - ) { - const mainWindow = yield* Ref.get(mainWindowRef); - if (!isAppShortcut(input) || Option.isNone(mainWindow) || mainWindow.value.isDestroyed()) { - return; + const forwardShortcut = (event: Electron.Event, input: Electron.Input): boolean => { + const mainWindow = currentMainWindow; + if (!isAppShortcut(input) || !mainWindow || mainWindow.isDestroyed()) { + return false; } event.preventDefault(); - mainWindow.value.webContents.sendInputEvent({ + mainWindow.webContents.sendInputEvent({ type: "keyDown", keyCode: input.key, modifiers: [ @@ -1830,7 +2276,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ...(input.alt ? (["alt"] as const) : []), ], }); - }); + return true; + }; // A popup opens with Electron's default handler, so the page inside it could // otherwise spawn native windows without limit. Nothing in an OAuth flow // opens a second popup, so the chain stops at the first one. @@ -1838,6 +2285,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function window.webContents.setWindowOpenHandler(() => ({ action: "deny" })); }; const beforeInput = (event: Electron.Event, input: Electron.Input): void => { + const activeAgentKey = activeAgentKeys.get(attachmentId); + if ( + activeAgentKey && + activeAgentKey.attachmentId === attachmentId && + activeAgentKey.webContents === wc && + isKeyboardDocumentCurrent(attachment, wc, activeAgentKey.document) && + inputMatchesActiveAgentKey(activeAgentKey, input) + ) { + if (activeAgentKey.valid) { + activeAgentKey.accepted = true; + attachment.pendingKeyInputs.push({ + document: activeAgentKey.document, + nativeKey: activeAgentKey, + signal: keySignalFromInput(input as Electron.Input & { type: "keyDown" | "keyUp" }), + }); + } + return; + } + if (activeAgentKey) { + activeAgentKey.valid = false; + event.preventDefault(); + releaseMenuShortcutLease(attachment, wc); + if (input.type === "keyDown") runFork(claimHumanControl()); + return; + } + if (input.type === "keyDown" || input.type === "keyUp") { + for (const pending of attachment.pendingAgentKeys) pending.valid = false; + if (input.type === "keyDown") runFork(claimHumanControl()); + } + releaseMenuShortcutLease(attachment, wc); if (isPreviewRefreshShortcut(input)) { event.preventDefault(); runFork( @@ -1847,12 +2324,29 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); return; } - runFork(forwardShortcut(event, input)); + if (forwardShortcut(event, input)) return; + if ( + (input.type === "keyDown" || input.type === "keyUp") && + wc.focusedFrame === wc.mainFrame + ) { + const signal = keySignalFromInput( + input as Electron.Input & { readonly type: "keyDown" | "keyUp" }, + ); + attachment.pendingKeyInputs.push({ + document: captureKeyboardDocument(attachment, wc), + signal, + }); + if (attachment.pendingKeyInputs.length > 20) { + attachment.pendingKeyInputs.length = 0; + markKeyboardDeliveryUncertain(attachment, wc, captureKeyboardDocument(attachment, wc)); + } + } }; yield* Scope.addFinalizer( scope, attempt({ operation: "detachListeners", tabId, webContentsId: wc.id }, () => { cancelFaviconCapture(); + releaseMenuShortcutLease(attachment, wc); wc.off("did-start-navigation", navigationStarted); wc.off("did-navigate", syncNavigation); wc.off("did-navigate-in-page", syncInPageNavigation); @@ -1897,11 +2391,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); yield* Ref.update(attachedRef, (attached) => replaceMap(attached, (copy) => { - copy.set(wc.id, { attachmentId, cancelFaviconCapture, scope, webContents: wc }); + copy.set(wc.id, attachment); }), ); }); yield* install().pipe(Effect.onError(() => Scope.close(scope, Exit.void).pipe(Effect.ignore))); + return attachment; }); const setMainWindow = Effect.fn("PreviewManager.setMainWindow")(function* ( @@ -2000,6 +2495,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function discard: true, }, ); + const currentTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if (!currentTab) return; + if (currentTab.webContentsId != null) { + const attachment = (yield* Ref.get(attachedRef)).get(currentTab.webContentsId); + if (attachment?.tabId === tabId) { + yield* Effect.all( + [ + detachControlSession(currentTab.webContentsId, attachment), + detachListeners(currentTab.webContentsId, attachment), + ], + { concurrency: 2, discard: true }, + ); + } + } const tab = yield* SynchronizedRef.modify(tabsRef, (tabs) => { const current = tabs.get(tabId); if (!current) return [Option.none(), tabs] as const; @@ -2012,12 +2521,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); if (Option.isNone(tab)) return; const closedTab = tab.value; - if (closedTab.webContentsId != null) { - yield* Effect.all( - [detachControlSession(closedTab.webContentsId), detachListeners(closedTab.webContentsId)], - { concurrency: 2, discard: true }, - ); - } const updatedAt = yield* currentIso; const closed: PreviewTabState = { ...closedTab, @@ -2081,7 +2584,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const annotationTheme = yield* Ref.get(annotationThemeRef); const currentAttachment = attached.get(webContentsId); yield* keepFrameCaptureWebContentsUnthrottled(tabId, wc); - if (tab.webContentsId === webContentsId && currentAttachment?.webContents === wc) { + if ( + tab.webContentsId === webContentsId && + currentAttachment?.active && + currentAttachment.tabId === tabId && + currentAttachment.webContents === wc + ) { // The guest we already own re-announced itself, so nothing about the tab // changed. Only push its zoom back down — Chromium may have just handed // this guest the app window's zoom level. @@ -2091,23 +2599,38 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); return; } - const replacedWebContentsId = - tab.webContentsId != null && - (tab.webContentsId !== webContentsId || currentAttachment?.webContents !== wc) - ? tab.webContentsId - : null; - if (replacedWebContentsId !== null) { - // The replaced guest can no longer redeem a display-media grant. - clearPendingRecording(tabId); - yield* Effect.all( - [ - detachControlSession(replacedWebContentsId), - detachListeners(replacedWebContentsId), - cancelPickElement(tabId), - ], - { concurrency: 3, discard: true }, - ); + const replacedAttachments = new Set(); + if (tab.webContentsId != null) { + const tabAttachment = attached.get(tab.webContentsId); + if (tabAttachment?.tabId === tabId) replacedAttachments.add(tabAttachment); } + if ( + currentAttachment && + (currentAttachment.tabId !== tabId || currentAttachment.webContents !== wc) + ) { + replacedAttachments.add(currentAttachment); + } + if (replacedAttachments.size > 0) { + for (const displacedTabId of new Set([ + tabId, + ...Array.from(replacedAttachments, (attachment) => attachment.tabId), + ])) { + clearPendingRecording(displacedTabId); + } + } + yield* Effect.forEach( + replacedAttachments, + (replacedAttachment) => + Effect.all( + [ + detachControlSession(replacedAttachment.webContents.id, replacedAttachment), + detachListeners(replacedAttachment.webContents.id, replacedAttachment), + cancelPickElement(replacedAttachment.tabId), + ], + { concurrency: 3, discard: true }, + ), + { concurrency: "unbounded", discard: true }, + ); const currentTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); if ( !currentTab || @@ -2129,7 +2652,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function yield* attempt({ operation: "registerWebview.restoreAudioMuted", tabId, webContentsId }, () => wc.setAudioMuted(currentTab.audioMuted), ); - yield* attachListeners(tabId, wc); + const registeredAttachment = yield* attachListeners(tabId, wc); const readAudible = attempt( { operation: "registerWebview.readAudible", tabId, webContentsId }, () => wc.isCurrentlyAudible(), @@ -2145,7 +2668,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function (yield* Ref.get(closingTabIdsRef)).has(tabId) ) { return [ - Option.none<{ readonly state: PreviewTabState; readonly pendingUrl: string | null }>(), + Option.none<{ + readonly state: PreviewTabState; + readonly pendingUrl: string | null; + readonly displaced: ReadonlyArray; + }>(), tabs, ] as const; } @@ -2160,25 +2687,47 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function audible: attachedAudible, updatedAt: registeredAt, }; + const displaced = Array.from(tabs.entries()).flatMap(([ownerTabId, owner]) => { + if (ownerTabId === tabId || owner.webContentsId !== webContentsId) return []; + return [ + [ + ownerTabId, + { + ...owner, + webContentsId: null, + controller: "none" as const, + updatedAt: registeredAt, + }, + ] as const, + ]; + }); return [ Option.some({ state: next, pendingUrl, + displaced, }), replaceMap(tabs, (copy) => { + for (const [ownerTabId, owner] of displaced) copy.set(ownerTabId, owner); copy.set(tabId, next); }), ] as const; }), ); if (Option.isNone(registration)) { - yield* Effect.all([detachControlSession(webContentsId), detachListeners(webContentsId)], { - concurrency: 2, - discard: true, - }); + yield* Effect.all( + [ + detachControlSession(webContentsId, registeredAttachment), + detachListeners(webContentsId, registeredAttachment), + ], + { concurrency: 2, discard: true }, + ); return yield* new PreviewTabNotFoundError({ tabId }); } - const { state: registered, pendingUrl } = registration.value; + const { state: registered, pendingUrl, displaced } = registration.value; + yield* Effect.forEach(displaced, ([ownerTabId, owner]) => emitIfCurrent(ownerTabId, owner), { + discard: true, + }); // A zoom or mute action that landed while this attach was in flight // addressed the guest this one replaced, so settle the new guest on the // committed values. @@ -2218,9 +2767,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function webContentsId: number, ) { const expectedGeneration = tabLifecycleGenerations.get(tabId); - return yield* withTabLifecycleLock( - tabId, - registerWebviewUnlocked(tabId, webContentsId, expectedGeneration), + return yield* webviewRegistrationSemaphore.withPermit( + withTabLifecycleLock( + tabId, + registerWebviewUnlocked(tabId, webContentsId, expectedGeneration), + ), ); }); @@ -2283,14 +2834,16 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ) { return; } - yield* Effect.all( - [ - detachControlSession(webContentsId), - detachListeners(webContentsId), - cancelPickElement(tabId), - ], - { concurrency: 3, discard: true }, - ); + if (expectedAttachment?.tabId === tabId) { + yield* Effect.all( + [ + detachControlSession(webContentsId, expectedAttachment), + detachListeners(webContentsId, expectedAttachment), + cancelPickElement(tabId), + ], + { concurrency: 3, discard: true }, + ); + } const detached = yield* SynchronizedRef.modify(tabsRef, (tabs) => { const current = tabs.get(tabId); if (current?.webContentsId !== webContentsId) { @@ -2342,23 +2895,56 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const hardReload = (tabId: string) => withWebContents("hardReload", tabId, (wc) => wc.reloadIgnoringCache()); - const openDevTools = Effect.fn("PreviewManager.openDevTools")(function* (tabId: string) { + const openDevToolsUnlocked = Effect.fn("PreviewManager.openDevToolsUnlocked")(function* ( + tabId: string, + ) { const wc = yield* requireWebContents(tabId); + const attachment = (yield* Ref.get(attachedRef)).get(wc.id); + if (!attachment?.active || attachment.tabId !== tabId || attachment.webContents !== wc) { + return yield* new PreviewAutomationTargetChangedError({ + operation: "openDevTools", + tabId, + webContentsId: wc.id, + }); + } + let targetChanged = false; + const withExactTarget = (use: () => void) => + attempt({ operation: "openDevTools", tabId, webContentsId: wc.id }, () => { + if (!attachment.active || webContents.fromId(wc.id) !== wc) { + targetChanged = true; + return; + } + use(); + }); if (wc.isDevToolsOpened()) { - yield* attempt({ operation: "openDevTools.focus", tabId, webContentsId: wc.id }, () => - wc.devToolsWebContents?.focus(), - ); + yield* withExactTarget(() => wc.devToolsWebContents?.focus()); + if (targetChanged) { + return yield* new PreviewAutomationTargetChangedError({ + operation: "openDevTools", + tabId, + webContentsId: wc.id, + }); + } return; } - yield* detachControlSession(wc.id); - yield* attempt({ operation: "openDevTools", tabId, webContentsId: wc.id }, () => { + yield* detachControlSession(wc.id, attachment); + yield* withExactTarget(() => { wc.once("devtools-closed", () => { if (!wc.isDestroyed()) runFork(restoreControlSession(tabId, wc)); }); wc.openDevTools({ mode: "detach" }); }); + if (targetChanged) { + return yield* new PreviewAutomationTargetChangedError({ + operation: "openDevTools", + tabId, + webContentsId: wc.id, + }); + } }); + const openDevTools = (tabId: string) => withTabLifecycleLock(tabId, openDevToolsUnlocked(tabId)); + const setAnnotationTheme = Effect.fn("PreviewManager.setAnnotationTheme")(function* ( theme: DesktopPreviewAnnotationTheme, ) { @@ -2578,7 +3164,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc: Electron.WebContents, colorScheme: DesktopPreviewColorScheme, ) { - yield* ensureControlSession(wc); + yield* ensureControlSession(wc, tabId, "setColorScheme"); yield* attemptPromise({ operation: "applyColorScheme", tabId, webContentsId: wc.id }, () => wc.debugger.sendCommand("Emulation.setEmulatedMedia", { features: [ @@ -2600,10 +3186,14 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function Effect.gen(function* () { const beforeAttach = (yield* SynchronizedRef.get(tabsRef)).get(tabId); if (beforeAttach?.webContentsId !== wc.id) return; - yield* ensureControlSession(wc); + const attachment = (yield* Ref.get(attachedRef)).get(wc.id); + const control = yield* ensureControlSession(wc, tabId, "restoreControlSession", attachment); const afterAttach = (yield* SynchronizedRef.get(tabsRef)).get(tabId); - if (afterAttach?.webContentsId !== wc.id) { - yield* detachControlSession(wc.id); + if ( + afterAttach?.webContentsId !== wc.id || + (yield* Ref.get(attachedRef)).get(wc.id) !== attachment + ) { + yield* detachControlSession(wc.id, control); return; } if (afterAttach.colorScheme !== "system") { @@ -3856,51 +4446,298 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc: Electron.WebContents, input: PreviewAutomationPressInput, send: SendCommand, - sendCleanup: SendCommand, + assertCurrent: Effect.Effect, + attachment: ManagedListeners, + isNativeTargetCurrent: () => boolean, ) { + yield* assertCurrent; + if (attachment.keyboardDeliveryUncertainGeneration === attachment.documentGeneration) { + return yield* new PreviewAutomationKeyboardDeliveryNotConfirmedError({ + tabId, + webContentsId: wc.id, + }); + } + const mainWindow = currentMainWindow; + // Electron only routes native WebContents keyboard packets reliably while + // their containing BrowserWindow is active. Never focus either target + // here: doing so would steal the user's composer focus. Background/remote + // callers get a typed failure and can retry after the window is active. + if (!mainWindow || mainWindow.isDestroyed() || !mainWindow.isFocused()) { + return yield* new PreviewAutomationKeyboardWindowNotFocusedError({ + tabId, + webContentsId: wc.id, + }); + } + if (wc.hostWebContents !== mainWindow.webContents) { + return yield* new PreviewAutomationTargetChangedError({ + operation: "press", + tabId, + webContentsId: wc.id, + }); + } yield* prepareAutomationInput(send, false); - const keySequence = makePreviewAutomationKeySequence(input, { - isMac: hostPlatform === "darwin", + yield* assertCurrent; + const focusedFrame = yield* evaluateWithDebugger( + tabId, + send, + `(() => { + const tagName = document.activeElement?.tagName; + return tagName === "IFRAME" || tagName === "FRAME"; + })()`, + true, + ); + if (focusedFrame) { + return yield* new PreviewAutomationKeyboardFocusedFrameUnsupportedError({ + tabId, + webContentsId: wc.id, + }); + } + + const receiptDocument = captureKeyboardDocument(attachment, wc); + if (receiptDocument.frame.detached) { + return yield* new PreviewAutomationTargetChangedError({ + operation: "press", + tabId, + webContentsId: wc.id, + }); + } + const keySequence = makePreviewAutomationKeySequence(input); + const makeActiveKey = ( + phase: "down" | "up", + signal: typeof keySequence.keyDownSignal, + ): ActiveAgentKey => ({ + attachmentId: attachment.attachmentId, + document: receiptDocument, + id: Symbol(), + phase, + signal, + webContents: wc, + accepted: false, + valid: true, }); - const previouslyFocused = yield* attempt( - { operation: "automationPress.getFocusedWebContents", tabId, webContentsId: wc.id }, - () => webContents.getFocusedWebContents(), + const keyDownMarker = makeActiveKey("down", keySequence.keyDownSignal); + const keyUpMarker = makeActiveKey("up", keySequence.keyUpSignal); + attachment.pendingAgentKeys.add(keyDownMarker); + attachment.pendingAgentKeys.add(keyUpMarker); + const keyDownReceipt = yield* Deferred.make(); + const keyUpReceipt = yield* Deferred.make(); + const keyDownExpectationId = yield* expectAgentInput( + tabId, + { ...keySequence.keyDownSignal, phase: "down" }, + { + attachmentId: attachment.attachmentId, + nativeKey: keyDownMarker, + receipt: keyDownReceipt, + }, ); - let keyDownAttempted = false; - const releaseInput = Effect.gen(function* () { - if (keyDownAttempted) { - yield* sendCleanup("Input.dispatchKeyEvent", keySequence.keyUp).pipe(Effect.ignore); + const keyUpExpectationId = yield* expectAgentInput( + tabId, + { ...keySequence.keyUpSignal, phase: "up" }, + { + attachmentId: attachment.attachmentId, + nativeKey: keyUpMarker, + receipt: keyUpReceipt, + }, + ); + const sendNativeKey = (marker: ActiveAgentKey, event: Electron.KeyboardInputEvent): void => { + activeAgentKeys.set(attachment.attachmentId, marker); + try { + wc.sendInputEvent(event); + } finally { + if (activeAgentKeys.get(attachment.attachmentId) === marker) { + activeAgentKeys.delete(attachment.attachmentId); + } } - yield* sendCleanup("Emulation.setFocusEmulationEnabled", { enabled: false }).pipe( - Effect.ignore, - ); - if (previouslyFocused && previouslyFocused.id !== wc.id && !previouslyFocused.isDestroyed()) { - yield* attempt( - { - operation: "automationPress.restoreFocusedWebContents", + }; + let keyDownAttempted = false; + let nativeKeyDispatched = false; + let nativePhaseReceiptsComplete = false; + const menuShortcutLease = Symbol(); + const restoreMenuShortcuts = attempt( + { operation: "automationPress.restoreMenuShortcuts", tabId, webContentsId: wc.id }, + () => { + if ( + !attachment.active || + attachment.menuShortcutLease !== menuShortcutLease || + !isKeyboardDocumentCurrent(attachment, wc, receiptDocument) + ) { + return; + } + releaseMenuShortcutLease(attachment, wc, menuShortcutLease); + }, + ).pipe(Effect.ignore); + const releaseInput = attempt( + { operation: "automationPress.releaseNativeKey", tabId, webContentsId: wc.id }, + () => { + if ( + !keyDownAttempted || + wc.isDestroyed() || + !isNativeTargetCurrent() || + !isKeyboardDocumentCurrent(attachment, wc, receiptDocument) + ) { + return; + } + attachment.menuShortcutLease = menuShortcutLease; + wc.setIgnoreMenuShortcuts(true); + sendNativeKey(keyUpMarker, keySequence.keyUp); + }, + ).pipe(Effect.ignore); + let preflightError: PreviewManagerError | undefined; + const dispatch = attempt( + { operation: "automationPress.dispatchNativeKey", tabId, webContentsId: wc.id }, + () => { + if ( + !isNativeTargetCurrent() || + currentMainWindow !== mainWindow || + mainWindow.isDestroyed() || + wc.hostWebContents !== mainWindow.webContents || + !isKeyboardDocumentCurrent(attachment, wc, receiptDocument) + ) { + preflightError = new PreviewAutomationTargetChangedError({ + operation: "press", tabId, - webContentsId: previouslyFocused.id, - }, - () => previouslyFocused.focus(), - ).pipe(Effect.ignore); - } - }); - - // Focus the guest WebContents itself, not its containing BrowserWindow. This - // activates native keyboard behavior for hidden/background previews without - // changing which thread is mounted in the UI. Restore the previous renderer - // after dispatch so automation never leaves the app's input focus behind. + webContentsId: wc.id, + }); + return; + } + if (!mainWindow.isFocused()) { + preflightError = new PreviewAutomationKeyboardWindowNotFocusedError({ + tabId, + webContentsId: wc.id, + }); + return; + } + // Keep native menu routing disabled until the page's key-up receipt. + // Chromium cannot deliver key-up before it handles the key-down ACK. + attachment.menuShortcutLease = menuShortcutLease; + wc.setIgnoreMenuShortcuts(true); + if (!keyDownMarker.valid || !keyUpMarker.valid) { + releaseMenuShortcutLease(attachment, wc, menuShortcutLease); + preflightError = new PreviewAutomationControlInterruptedError({ + operation: "press", + tabId, + webContentsId: wc.id, + }); + return; + } + if ( + !isNativeTargetCurrent() || + currentMainWindow !== mainWindow || + mainWindow.isDestroyed() || + wc.hostWebContents !== mainWindow.webContents || + !isKeyboardDocumentCurrent(attachment, wc, receiptDocument) + ) { + releaseMenuShortcutLease(attachment, wc, menuShortcutLease); + preflightError = new PreviewAutomationTargetChangedError({ + operation: "press", + tabId, + webContentsId: wc.id, + }); + return; + } + const nativeFocusedFrame = wc.focusedFrame; + if (nativeFocusedFrame != null && nativeFocusedFrame !== wc.mainFrame) { + releaseMenuShortcutLease(attachment, wc, menuShortcutLease); + preflightError = new PreviewAutomationKeyboardFocusedFrameUnsupportedError({ + tabId, + webContentsId: wc.id, + }); + return; + } + sendNativeKey(keyDownMarker, keySequence.keyDown); + keyDownAttempted = true; + nativeKeyDispatched = true; + if (!keyDownMarker.accepted || !keyDownMarker.valid) { + preflightError = new PreviewAutomationKeyboardDeliveryNotConfirmedError({ + tabId, + webContentsId: wc.id, + }); + return; + } + if (keySequence.char) sendNativeKey(keyDownMarker, keySequence.char); + if (!keyDownMarker.valid || !keyUpMarker.valid) { + preflightError = new PreviewAutomationControlInterruptedError({ + operation: "press", + tabId, + webContentsId: wc.id, + }); + return; + } + sendNativeKey(keyUpMarker, keySequence.keyUp); + keyDownAttempted = false; + if (!keyUpMarker.accepted || !keyUpMarker.valid) { + preflightError = new PreviewAutomationKeyboardDeliveryNotConfirmedError({ + tabId, + webContentsId: wc.id, + }); + } + }, + ).pipe( + Effect.flatMap(() => (preflightError ? Effect.fail(preflightError) : Effect.void)), + Effect.ensuring(releaseInput), + ); yield* Effect.gen(function* () { - yield* attempt( - { operation: "automationPress.focusWebContents", tabId, webContentsId: wc.id }, - () => wc.focus(), - ); - yield* send("Page.bringToFront"); - yield* send("Emulation.setFocusEmulationEnabled", { enabled: true }); - yield* expectAgentInput(tabId, keySequence.signal); - keyDownAttempted = true; - yield* send("Input.dispatchKeyEvent", keySequence.keyDown); - }).pipe(Effect.ensuring(releaseInput)); + yield* assertCurrent; + yield* dispatch; + const received = yield* Effect.all( + [Deferred.await(keyDownReceipt), Deferred.await(keyUpReceipt)], + { discard: true }, + ).pipe(Effect.timeoutOption(AGENT_KEY_RECEIPT_TIMEOUT_MS)); + if (Option.isNone(received)) { + if (!keyDownMarker.valid || !keyUpMarker.valid) { + return yield* new PreviewAutomationControlInterruptedError({ + operation: "press", + tabId, + webContentsId: wc.id, + }); + } + return yield* new PreviewAutomationKeyboardDeliveryNotConfirmedError({ + tabId, + webContentsId: wc.id, + }); + } + // Receipt completion is a delivery fact, independent of whether human + // input claims control before this action finishes its final checks. + nativePhaseReceiptsComplete = true; + yield* assertCurrent; + const interrupted = !keyDownMarker.valid || !keyUpMarker.valid; + if (interrupted) { + return yield* new PreviewAutomationControlInterruptedError({ + operation: "press", + tabId, + webContentsId: wc.id, + }); + } + }).pipe( + Effect.ensuring( + Effect.all( + [ + removeExpectedAgentInput(tabId, keyDownExpectationId), + removeExpectedAgentInput(tabId, keyUpExpectationId), + Effect.sync(() => { + attachment.pendingAgentKeys.delete(keyDownMarker); + attachment.pendingAgentKeys.delete(keyUpMarker); + if ( + nativeKeyDispatched && + !nativePhaseReceiptsComplete && + isKeyboardDocumentCurrent(attachment, wc, receiptDocument) + ) { + markKeyboardDeliveryUncertain(attachment, wc, receiptDocument); + } + }), + ], + { discard: true }, + ).pipe( + Effect.andThen( + Effect.suspend(() => + nativePhaseReceiptsComplete || !nativeKeyDispatched + ? restoreMenuShortcuts + : Effect.void, + ), + ), + ), + ), + ); }); const automationPress = Effect.fn("PreviewManager.automationPress")(function* ( @@ -3908,8 +4745,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function input: PreviewAutomationPressInput, ) { const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "press", (send, sendCleanup) => - performAutomationPress(tabId, wc, input, send, sendCleanup), + yield* withControlSession( + tabId, + wc, + "press", + (send, assertCurrent, attachment, isNativeTargetCurrent) => + performAutomationPress( + tabId, + wc, + input, + send, + assertCurrent, + attachment, + isNativeTargetCurrent, + ), ); }); @@ -4425,6 +5274,63 @@ export class PreviewAutomationTimeoutError extends Schema.TaggedErrorClass()( + "PreviewAutomationKeyboardWindowNotFocusedError", + { + tabId: Schema.String, + webContentsId: Schema.Number, + }, +) { + override get message(): string { + return `Preview tab ${this.tabId} cannot receive keyboard input while its desktop window is unfocused`; + } +} + +export class PreviewAutomationKeyboardFocusedFrameUnsupportedError extends Schema.TaggedErrorClass()( + "PreviewAutomationKeyboardFocusedFrameUnsupportedError", + { + tabId: Schema.String, + webContentsId: Schema.Number, + }, +) { + override get message(): string { + return `Preview tab ${this.tabId} cannot receive keyboard input while a frame is focused`; + } +} + +export class PreviewAutomationKeyboardDeliveryNotConfirmedError extends Schema.TaggedErrorClass()( + "PreviewAutomationKeyboardDeliveryNotConfirmedError", + { + tabId: Schema.String, + webContentsId: Schema.Number, + }, +) { + override get message(): string { + return `Preview tab ${this.tabId} did not confirm keyboard input delivery`; + } +} + +export const PreviewAutomationKeyboardError = Schema.Union([ + PreviewAutomationKeyboardWindowNotFocusedError, + PreviewAutomationKeyboardFocusedFrameUnsupportedError, + PreviewAutomationKeyboardDeliveryNotConfirmedError, +]); +export type PreviewAutomationKeyboardError = typeof PreviewAutomationKeyboardError.Type; +export const isPreviewAutomationKeyboardError = Schema.is(PreviewAutomationKeyboardError); + +export class PreviewAutomationTargetChangedError extends Schema.TaggedErrorClass()( + "PreviewAutomationTargetChangedError", + { + operation: Schema.String, + tabId: Schema.String, + webContentsId: Schema.Number, + }, +) { + override get message(): string { + return `Preview automation ${this.operation} stopped because tab ${this.tabId} changed targets`; + } +} + export class PreviewAutomationControlInterruptedError extends Schema.TaggedErrorClass()( "PreviewAutomationControlInterruptedError", { @@ -4457,6 +5363,8 @@ export const PreviewManagerError = Schema.Union([ PreviewAutomationInvalidSelectorError, PreviewAutomationResultTooLargeError, PreviewAutomationTimeoutError, + PreviewAutomationKeyboardError, + PreviewAutomationTargetChangedError, PreviewAutomationControlInterruptedError, ]); export type PreviewManagerError = typeof PreviewManagerError.Type; diff --git a/apps/desktop/src/preview/PickPreload.ts b/apps/desktop/src/preview/PickPreload.ts index 6155c4119ec8..3ba2d46144b9 100644 --- a/apps/desktop/src/preview/PickPreload.ts +++ b/apps/desktop/src/preview/PickPreload.ts @@ -93,17 +93,23 @@ const reportHumanPointerInput = (event: PointerEvent): void => { }); }; -const reportHumanKeyInput = (event: KeyboardEvent): void => { +const reportHumanKeyInput = (event: KeyboardEvent, phase: "down" | "up"): void => { if (!event.isTrusted) return; ipcRenderer.send(HUMAN_INPUT_CHANNEL, { kind: "key", + phase, key: event.key, code: event.code, + meta: event.metaKey, + shift: event.shiftKey, + control: event.ctrlKey, + alt: event.altKey, }); }; window.addEventListener("pointerdown", reportHumanPointerInput, true); -window.addEventListener("keydown", reportHumanKeyInput, true); +window.addEventListener("keydown", (event) => reportHumanKeyInput(event, "down"), true); +window.addEventListener("keyup", (event) => reportHumanKeyInput(event, "up"), true); // Mouse thumb buttons: `button === 3` is Back, `button === 4` is Forward. const MOUSE_BUTTON_BACK = 3; diff --git a/apps/desktop/src/preview/PreviewKeyboard.test.ts b/apps/desktop/src/preview/PreviewKeyboard.test.ts index 7a9a7373fe32..96260b1bfb64 100644 --- a/apps/desktop/src/preview/PreviewKeyboard.test.ts +++ b/apps/desktop/src/preview/PreviewKeyboard.test.ts @@ -2,96 +2,153 @@ import { describe, expect, it } from "vite-plus/test"; import { makePreviewAutomationKeySequence } from "./PreviewKeyboard.ts"; +const expectedSignal = (key: string, code: string, modifiers: ReadonlyArray) => ({ + kind: "key" as const, + key, + code, + meta: modifiers.includes("meta"), + shift: modifiers.includes("shift"), + control: modifiers.includes("control"), + alt: modifiers.includes("alt"), +}); + +const expectedSignals = ( + key: string, + code: string, + keyDownModifiers: ReadonlyArray, + keyUpModifiers = keyDownModifiers, +) => ({ + keyDownSignal: expectedSignal(key, code, keyDownModifiers), + keyUpSignal: expectedSignal(key, code, keyUpModifiers), +}); + describe("preview keyboard packets", () => { - it("includes the Chromium virtual key code and Enter text", () => { + it("sends Enter with a carriage-return char packet", () => { expect(makePreviewAutomationKeySequence({ key: "Enter" })).toEqual({ - keyDown: { - type: "keyDown", - key: "Enter", - code: "Enter", - modifiers: 0, - windowsVirtualKeyCode: 13, - location: 0, - isKeypad: false, - text: "\r", - unmodifiedText: "\r", - }, - keyUp: { - type: "keyUp", - key: "Enter", - code: "Enter", - modifiers: 0, - windowsVirtualKeyCode: 13, - location: 0, - isKeypad: false, - }, - signal: { kind: "key", key: "Enter", code: "Enter" }, + keyDown: { type: "rawKeyDown", keyCode: "Enter", modifiers: [] }, + char: { type: "char", keyCode: "\r", modifiers: [] }, + keyUp: { type: "keyUp", keyCode: "Enter", modifiers: [] }, + ...expectedSignals("Enter", "Enter", []), }); }); - it("dispatches printable keys as text key-down events", () => { - const sequence = makePreviewAutomationKeySequence({ key: "z" }); - expect(sequence.keyDown).toMatchObject({ - type: "keyDown", - key: "z", - code: "KeyZ", - windowsVirtualKeyCode: 90, - text: "z", + it("keeps Shift on every Shift+Enter packet", () => { + expect(makePreviewAutomationKeySequence({ key: "Enter", modifiers: ["Shift"] })).toEqual({ + keyDown: { type: "rawKeyDown", keyCode: "Enter", modifiers: ["shift"] }, + char: { type: "char", keyCode: "\r", modifiers: ["shift"] }, + keyUp: { type: "keyUp", keyCode: "Enter", modifiers: ["shift"] }, + ...expectedSignals("Enter", "Enter", ["shift"]), }); - expect(sequence.keyUp).not.toHaveProperty("text"); }); - it("suppresses text and uses raw key-down for shortcuts", () => { - expect( - makePreviewAutomationKeySequence({ key: "a", modifiers: ["Meta"] }, { isMac: true }).keyDown, - ).toEqual({ - type: "rawKeyDown", - key: "a", - code: "KeyA", - modifiers: 4, - windowsVirtualKeyCode: 65, - location: 0, - isKeypad: false, - commands: ["selectAll"], + for (const modifier of ["Control", "Alt", "Meta"] as const) { + it(`suppresses Enter text for ${modifier} chords`, () => { + const nativeModifier = modifier.toLowerCase(); + expect(makePreviewAutomationKeySequence({ key: "Enter", modifiers: [modifier] })).toEqual({ + keyDown: { type: "rawKeyDown", keyCode: "Enter", modifiers: [nativeModifier] }, + keyUp: { type: "keyUp", keyCode: "Enter", modifiers: [nativeModifier] }, + ...expectedSignals("Enter", "Enter", [nativeModifier]), + }); + }); + } + + const namedModifiers = [ + { key: "Shift", code: "ShiftLeft", modifier: "shift" }, + { key: "Control", code: "ControlLeft", modifier: "control" }, + { key: "Alt", code: "AltLeft", modifier: "alt" }, + { key: "Meta", code: "MetaLeft", modifier: "meta" }, + ] as const; + + for (const { key, code, modifier } of namedModifiers) { + it(`sets and clears the ${key} flag when pressing the named modifier`, () => { + expect(makePreviewAutomationKeySequence({ key })).toEqual({ + keyDown: { type: "rawKeyDown", keyCode: key, modifiers: [modifier] }, + keyUp: { type: "keyUp", keyCode: key, modifiers: [] }, + ...expectedSignals(key, code, [modifier], []), + }); + }); + + it(`preserves other modifiers and removes duplicate ${key} flags on key-up`, () => { + const otherModifier = key === "Alt" ? "Control" : "Alt"; + const otherNativeModifier = otherModifier.toLowerCase(); + expect( + makePreviewAutomationKeySequence({ + key, + modifiers: [otherModifier, key, key], + }), + ).toEqual({ + keyDown: { + type: "rawKeyDown", + keyCode: key, + modifiers: [otherNativeModifier, modifier], + }, + keyUp: { type: "keyUp", keyCode: key, modifiers: [otherNativeModifier] }, + ...expectedSignals(key, code, [otherNativeModifier, modifier], [otherNativeModifier]), + }); + }); + } + + it("separates printable key events from text insertion", () => { + expect(makePreviewAutomationKeySequence({ key: "z" })).toEqual({ + keyDown: { type: "rawKeyDown", keyCode: "Z", modifiers: [] }, + char: { type: "char", keyCode: "z", modifiers: [] }, + keyUp: { type: "keyUp", keyCode: "Z", modifiers: [] }, + ...expectedSignals("z", "KeyZ", []), }); }); - it("maps common macOS editing shortcuts without changing other platforms", () => { - expect( - makePreviewAutomationKeySequence({ key: "z", modifiers: ["Shift", "Meta"] }, { isMac: true }) - .keyDown.commands, - ).toEqual(["redo"]); - expect( - makePreviewAutomationKeySequence({ key: "a", modifiers: ["Meta"] }).keyDown, - ).not.toHaveProperty("commands"); + it("uses native modifier chords without inserting text", () => { + expect(makePreviewAutomationKeySequence({ key: "a", modifiers: ["Meta"] })).toEqual({ + keyDown: { type: "rawKeyDown", keyCode: "A", modifiers: ["meta"] }, + keyUp: { type: "keyUp", keyCode: "A", modifiers: ["meta"] }, + ...expectedSignals("a", "KeyA", ["meta"]), + }); }); - it("resolves shifted printable keys to their browser values", () => { - const sequence = makePreviewAutomationKeySequence({ key: "1", modifiers: ["Shift"] }); - expect(sequence.keyDown).toMatchObject({ - key: "!", - code: "Digit1", - modifiers: 8, - windowsVirtualKeyCode: 49, - text: "!", + it("keeps editing-chord modifiers on each native packet", () => { + expect(makePreviewAutomationKeySequence({ key: "z", modifiers: ["Shift", "Meta"] })).toEqual({ + keyDown: { type: "rawKeyDown", keyCode: "Z", modifiers: ["shift", "meta"] }, + keyUp: { type: "keyUp", keyCode: "Z", modifiers: ["shift", "meta"] }, + ...expectedSignals("Z", "KeyZ", ["shift", "meta"]), }); - expect(sequence.signal).toEqual({ kind: "key", key: "!", code: "Digit1" }); }); - it("keeps shifted key values while suppressing text for modified chords", () => { - const sequence = makePreviewAutomationKeySequence({ - key: "1", - modifiers: ["Control", "Shift"], + it("maps shifted printable keys to a base key plus Shift", () => { + expect(makePreviewAutomationKeySequence({ key: "!" })).toEqual({ + keyDown: { type: "rawKeyDown", keyCode: "1", modifiers: ["shift"] }, + char: { type: "char", keyCode: "!", modifiers: ["shift"] }, + keyUp: { type: "keyUp", keyCode: "1", modifiers: ["shift"] }, + ...expectedSignals("!", "Digit1", ["shift"]), }); - expect(sequence.keyDown).toEqual({ - type: "rawKeyDown", - key: "!", - code: "Digit1", - modifiers: 10, - windowsVirtualKeyCode: 49, - location: 0, - isKeypad: false, + }); + + it("does not insert text for modified shifted keys", () => { + expect(makePreviewAutomationKeySequence({ key: "1", modifiers: ["Control", "Shift"] })).toEqual( + { + keyDown: { type: "rawKeyDown", keyCode: "1", modifiers: ["control", "shift"] }, + keyUp: { type: "keyUp", keyCode: "1", modifiers: ["control", "shift"] }, + ...expectedSignals("!", "Digit1", ["control", "shift"]), + }, + ); + }); + + it("uses Electron accelerator names for arrows and function keys", () => { + expect(makePreviewAutomationKeySequence({ key: "ArrowLeft" }).keyDown.keyCode).toBe("Left"); + expect(makePreviewAutomationKeySequence({ key: "F12" }).keyDown.keyCode).toBe("F12"); + }); + + it("uses a literal space only for the char packet", () => { + expect(makePreviewAutomationKeySequence({ key: "Space" })).toEqual({ + keyDown: { type: "rawKeyDown", keyCode: "Space", modifiers: [] }, + char: { type: "char", keyCode: " ", modifiers: [] }, + keyUp: { type: "keyUp", keyCode: "Space", modifiers: [] }, + ...expectedSignals(" ", "Space", []), }); - expect(sequence.signal).toEqual({ kind: "key", key: "!", code: "Digit1" }); + }); + + it("does not forward unchecked Unicode keys to Electron", () => { + expect(() => makePreviewAutomationKeySequence({ key: "é" } as never)).toThrow( + "Use preview_type for Unicode text.", + ); }); }); diff --git a/apps/desktop/src/preview/PreviewKeyboard.ts b/apps/desktop/src/preview/PreviewKeyboard.ts index 0d231b86f4cc..a027e155c829 100644 --- a/apps/desktop/src/preview/PreviewKeyboard.ts +++ b/apps/desktop/src/preview/PreviewKeyboard.ts @@ -1,134 +1,100 @@ import type { PreviewAutomationPressInput } from "@t3tools/contracts"; +import type { KeyboardInputEvent } from "electron"; interface KeyDefinition { readonly code: string; readonly key: string; - readonly keyCode: number; + readonly nativeKeyCode: string; readonly text?: string; - readonly location?: number; readonly shiftedKey?: string; } -export interface PreviewAutomationKeyEvent { - readonly [key: string]: unknown; - readonly type: "keyDown" | "rawKeyDown" | "keyUp"; +interface PreviewAutomationKeySignal { + readonly kind: "key"; readonly key: string; readonly code: string; - readonly modifiers: number; - readonly windowsVirtualKeyCode: number; - readonly location: number; - readonly isKeypad: boolean; - readonly text?: string; - readonly unmodifiedText?: string; - readonly commands?: ReadonlyArray; + readonly meta: boolean; + readonly shift: boolean; + readonly control: boolean; + readonly alt: boolean; } export interface PreviewAutomationKeySequence { - readonly keyDown: PreviewAutomationKeyEvent; - readonly keyUp: PreviewAutomationKeyEvent; - readonly signal: { - readonly kind: "key"; - readonly key: string; - readonly code: string; - }; + readonly keyDown: KeyboardInputEvent; + readonly char?: KeyboardInputEvent; + readonly keyUp: KeyboardInputEvent; + readonly keyDownSignal: PreviewAutomationKeySignal; + readonly keyUpSignal: PreviewAutomationKeySignal; } +type PreviewAutomationModifier = "alt" | "control" | "meta" | "shift"; + const NAMED_KEYS: Readonly> = { - Escape: { code: "Escape", key: "Escape", keyCode: 27 }, - Backspace: { code: "Backspace", key: "Backspace", keyCode: 8 }, - Tab: { code: "Tab", key: "Tab", keyCode: 9 }, - Enter: { code: "Enter", key: "Enter", keyCode: 13, text: "\r" }, - Shift: { code: "ShiftLeft", key: "Shift", keyCode: 16, location: 1 }, - Control: { code: "ControlLeft", key: "Control", keyCode: 17, location: 1 }, - Alt: { code: "AltLeft", key: "Alt", keyCode: 18, location: 1 }, - Meta: { code: "MetaLeft", key: "Meta", keyCode: 91, location: 1 }, - CapsLock: { code: "CapsLock", key: "CapsLock", keyCode: 20 }, - Space: { code: "Space", key: " ", keyCode: 32, text: " " }, - PageUp: { code: "PageUp", key: "PageUp", keyCode: 33 }, - PageDown: { code: "PageDown", key: "PageDown", keyCode: 34 }, - End: { code: "End", key: "End", keyCode: 35 }, - Home: { code: "Home", key: "Home", keyCode: 36 }, - ArrowLeft: { code: "ArrowLeft", key: "ArrowLeft", keyCode: 37 }, - ArrowUp: { code: "ArrowUp", key: "ArrowUp", keyCode: 38 }, - ArrowRight: { code: "ArrowRight", key: "ArrowRight", keyCode: 39 }, - ArrowDown: { code: "ArrowDown", key: "ArrowDown", keyCode: 40 }, - Insert: { code: "Insert", key: "Insert", keyCode: 45 }, - Delete: { code: "Delete", key: "Delete", keyCode: 46 }, + Escape: { code: "Escape", key: "Escape", nativeKeyCode: "Escape" }, + Backspace: { code: "Backspace", key: "Backspace", nativeKeyCode: "Backspace" }, + Tab: { code: "Tab", key: "Tab", nativeKeyCode: "Tab" }, + Enter: { code: "Enter", key: "Enter", nativeKeyCode: "Enter", text: "\r" }, + Shift: { code: "ShiftLeft", key: "Shift", nativeKeyCode: "Shift" }, + Control: { code: "ControlLeft", key: "Control", nativeKeyCode: "Control" }, + Alt: { code: "AltLeft", key: "Alt", nativeKeyCode: "Alt" }, + Meta: { code: "MetaLeft", key: "Meta", nativeKeyCode: "Meta" }, + CapsLock: { code: "CapsLock", key: "CapsLock", nativeKeyCode: "CapsLock" }, + Space: { code: "Space", key: " ", nativeKeyCode: "Space", text: " " }, + PageUp: { code: "PageUp", key: "PageUp", nativeKeyCode: "PageUp" }, + PageDown: { code: "PageDown", key: "PageDown", nativeKeyCode: "PageDown" }, + End: { code: "End", key: "End", nativeKeyCode: "End" }, + Home: { code: "Home", key: "Home", nativeKeyCode: "Home" }, + ArrowLeft: { code: "ArrowLeft", key: "ArrowLeft", nativeKeyCode: "Left" }, + ArrowUp: { code: "ArrowUp", key: "ArrowUp", nativeKeyCode: "Up" }, + ArrowRight: { code: "ArrowRight", key: "ArrowRight", nativeKeyCode: "Right" }, + ArrowDown: { code: "ArrowDown", key: "ArrowDown", nativeKeyCode: "Down" }, + Insert: { code: "Insert", key: "Insert", nativeKeyCode: "Insert" }, + Delete: { code: "Delete", key: "Delete", nativeKeyCode: "Delete" }, }; const PRINTABLE_KEYS: ReadonlyArray = [ - { code: "Backquote", key: "`", shiftedKey: "~", keyCode: 192 }, - { code: "Digit1", key: "1", shiftedKey: "!", keyCode: 49 }, - { code: "Digit2", key: "2", shiftedKey: "@", keyCode: 50 }, - { code: "Digit3", key: "3", shiftedKey: "#", keyCode: 51 }, - { code: "Digit4", key: "4", shiftedKey: "$", keyCode: 52 }, - { code: "Digit5", key: "5", shiftedKey: "%", keyCode: 53 }, - { code: "Digit6", key: "6", shiftedKey: "^", keyCode: 54 }, - { code: "Digit7", key: "7", shiftedKey: "&", keyCode: 55 }, - { code: "Digit8", key: "8", shiftedKey: "*", keyCode: 56 }, - { code: "Digit9", key: "9", shiftedKey: "(", keyCode: 57 }, - { code: "Digit0", key: "0", shiftedKey: ")", keyCode: 48 }, - { code: "Minus", key: "-", shiftedKey: "_", keyCode: 189 }, - { code: "Equal", key: "=", shiftedKey: "+", keyCode: 187 }, - { code: "Backslash", key: "\\", shiftedKey: "|", keyCode: 220 }, - { code: "BracketLeft", key: "[", shiftedKey: "{", keyCode: 219 }, - { code: "BracketRight", key: "]", shiftedKey: "}", keyCode: 221 }, - { code: "Semicolon", key: ";", shiftedKey: ":", keyCode: 186 }, - { code: "Quote", key: "'", shiftedKey: '"', keyCode: 222 }, - { code: "Comma", key: ",", shiftedKey: "<", keyCode: 188 }, - { code: "Period", key: ".", shiftedKey: ">", keyCode: 190 }, - { code: "Slash", key: "/", shiftedKey: "?", keyCode: 191 }, + { code: "Backquote", key: "`", shiftedKey: "~", nativeKeyCode: "`" }, + { code: "Digit1", key: "1", shiftedKey: "!", nativeKeyCode: "1" }, + { code: "Digit2", key: "2", shiftedKey: "@", nativeKeyCode: "2" }, + { code: "Digit3", key: "3", shiftedKey: "#", nativeKeyCode: "3" }, + { code: "Digit4", key: "4", shiftedKey: "$", nativeKeyCode: "4" }, + { code: "Digit5", key: "5", shiftedKey: "%", nativeKeyCode: "5" }, + { code: "Digit6", key: "6", shiftedKey: "^", nativeKeyCode: "6" }, + { code: "Digit7", key: "7", shiftedKey: "&", nativeKeyCode: "7" }, + { code: "Digit8", key: "8", shiftedKey: "*", nativeKeyCode: "8" }, + { code: "Digit9", key: "9", shiftedKey: "(", nativeKeyCode: "9" }, + { code: "Digit0", key: "0", shiftedKey: ")", nativeKeyCode: "0" }, + { code: "Minus", key: "-", shiftedKey: "_", nativeKeyCode: "-" }, + { code: "Equal", key: "=", shiftedKey: "+", nativeKeyCode: "=" }, + { code: "Backslash", key: "\\", shiftedKey: "|", nativeKeyCode: "\\" }, + { code: "BracketLeft", key: "[", shiftedKey: "{", nativeKeyCode: "[" }, + { code: "BracketRight", key: "]", shiftedKey: "}", nativeKeyCode: "]" }, + { code: "Semicolon", key: ";", shiftedKey: ":", nativeKeyCode: ";" }, + { code: "Quote", key: "'", shiftedKey: '"', nativeKeyCode: "'" }, + { code: "Comma", key: ",", shiftedKey: "<", nativeKeyCode: "," }, + { code: "Period", key: ".", shiftedKey: ">", nativeKeyCode: "." }, + { code: "Slash", key: "/", shiftedKey: "?", nativeKeyCode: "/" }, ]; -/** - * Chromium does not infer macOS editing commands from synthetic Meta chords. - * Keep the common browser editing/navigation shortcuts explicit so dispatched - * key events behave like their physical-key equivalents. - */ -const MAC_EDITING_COMMANDS: Readonly> = { - "Meta+Backspace": "deleteToBeginningOfLine", - "Meta+ArrowUp": "moveToBeginningOfDocument", - "Meta+ArrowDown": "moveToEndOfDocument", - "Meta+ArrowLeft": "moveToLeftEndOfLine", - "Meta+ArrowRight": "moveToRightEndOfLine", - "Shift+Meta+ArrowUp": "moveToBeginningOfDocumentAndModifySelection", - "Shift+Meta+ArrowDown": "moveToEndOfDocumentAndModifySelection", - "Shift+Meta+ArrowLeft": "moveToLeftEndOfLineAndModifySelection", - "Shift+Meta+ArrowRight": "moveToRightEndOfLineAndModifySelection", - "Meta+KeyA": "selectAll", - "Meta+KeyC": "copy", - "Meta+KeyX": "cut", - "Meta+KeyV": "paste", - "Meta+KeyZ": "undo", - "Shift+Meta+KeyZ": "redo", -}; -const SHORTCUT_MODIFIER_ORDER = ["Shift", "Control", "Alt", "Meta"] as const; - -const macEditingCommands = ( - code: string, - modifiers: PreviewAutomationPressInput["modifiers"], -): ReadonlyArray => { - const shortcut = [ - ...SHORTCUT_MODIFIER_ORDER.filter((modifier) => modifiers?.includes(modifier)), - code, - ].join("+"); - const command = MAC_EDITING_COMMANDS[shortcut]; - return command ? [command] : []; +const MODIFIER_FOR_KEY: Readonly> = { + Alt: "alt", + Control: "control", + Meta: "meta", + Shift: "shift", }; -const modifierMask = (modifiers: PreviewAutomationPressInput["modifiers"]): number => - (modifiers ?? []).reduce((value, modifier) => { - switch (modifier) { - case "Alt": - return value | 1; - case "Control": - return value | 2; - case "Meta": - return value | 4; - case "Shift": - return value | 8; - } - }, 0); +const makeSignal = ( + definition: KeyDefinition, + modifiers: ReadonlyArray, +): PreviewAutomationKeySignal => ({ + kind: "key", + key: definition.key, + code: definition.code, + meta: modifiers.includes("meta"), + shift: modifiers.includes("shift"), + control: modifiers.includes("control"), + alt: modifiers.includes("alt"), +}); function resolveKeyDefinition(input: PreviewAutomationPressInput): KeyDefinition { const named = NAMED_KEYS[input.key]; @@ -136,15 +102,14 @@ function resolveKeyDefinition(input: PreviewAutomationPressInput): KeyDefinition const functionKey = /^F([1-9]|1[0-2])$/.exec(input.key); if (functionKey) { - const number = Number(functionKey[1]); - return { code: input.key, key: input.key, keyCode: 111 + number }; + return { code: input.key, key: input.key, nativeKeyCode: input.key }; } if (/^[a-z]$/i.test(input.key)) { const upper = input.key.toUpperCase(); const shifted = input.modifiers?.includes("Shift") ?? false; const key = shifted || input.key === upper ? upper : input.key; - return { code: `Key${upper}`, key, keyCode: upper.charCodeAt(0), text: key }; + return { code: `Key${upper}`, key, nativeKeyCode: upper, text: key }; } const printable = PRINTABLE_KEYS.find( @@ -159,45 +124,56 @@ function resolveKeyDefinition(input: PreviewAutomationPressInput): KeyDefinition return { ...printable, key, text: key }; } - return { - code: input.key.length > 1 ? input.key : "", - key: input.key, - keyCode: 0, - ...(input.key.length === 1 ? { text: input.key } : {}), - }; + throw new Error( + `Unsupported preview automation key ${JSON.stringify(input.key)}. Use preview_type for Unicode text.`, + ); } /** - * Build Chromium CDP key packets using the same required fields and down-event - * choice as Playwright's pinned Chromium keyboard implementation. + * Build Electron native key packets. `keyDown` is separate from `char`: the + * former emits keyboard events, while the latter inserts printable text. */ export function makePreviewAutomationKeySequence( input: PreviewAutomationPressInput, - options?: { readonly isMac?: boolean }, ): PreviewAutomationKeySequence { const definition = resolveKeyDefinition(input); - const modifiers = modifierMask(input.modifiers); + const explicitModifiers = Array.from( + new Set((input.modifiers ?? []).map((modifier) => modifier.toLowerCase())), + ) as Array; + const needsImplicitShift = + /^[A-Z]$/.test(definition.key) || definition.shiftedKey === definition.key; + if (needsImplicitShift && !explicitModifiers.includes("shift")) explicitModifiers.push("shift"); + const pressedModifier = MODIFIER_FOR_KEY[definition.key]; + const keyDownModifiers = [...explicitModifiers]; + if (pressedModifier && !keyDownModifiers.includes(pressedModifier)) { + keyDownModifiers.push(pressedModifier); + } + const keyUpModifiers = pressedModifier + ? explicitModifiers.filter((modifier) => modifier !== pressedModifier) + : [...explicitModifiers]; const suppressText = input.modifiers?.some((modifier) => modifier !== "Shift") ?? false; - const text = suppressText ? "" : (definition.text ?? ""); - const location = definition.location ?? 0; - const commands = options?.isMac ? macEditingCommands(definition.code, input.modifiers) : []; - const shared = { - key: definition.key, - code: definition.code, - modifiers, - windowsVirtualKeyCode: definition.keyCode, - location, - isKeypad: location === 3, - }; return { keyDown: { - type: text ? "keyDown" : "rawKeyDown", - ...shared, - ...(text ? { text, unmodifiedText: text } : {}), - ...(commands.length > 0 ? { commands } : {}), + type: "rawKeyDown", + keyCode: definition.nativeKeyCode, + modifiers: keyDownModifiers, + }, + ...(!suppressText && definition.text + ? { + char: { + type: "char" as const, + keyCode: definition.text, + modifiers: keyDownModifiers, + }, + } + : {}), + keyUp: { + type: "keyUp", + keyCode: definition.nativeKeyCode, + modifiers: keyUpModifiers, }, - keyUp: { type: "keyUp", ...shared }, - signal: { kind: "key", key: definition.key, code: definition.code }, + keyDownSignal: makeSignal(definition, keyDownModifiers), + keyUpSignal: makeSignal(definition, keyUpModifiers), }; } diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 54c2e1d9cf68..bfdfe962c882 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -57,6 +57,7 @@ import { PreviewAutomationRecordingNotActiveError, PreviewAutomationTargetUnavailableError, PreviewAutomationViewportTimeoutError, + unwrapPreviewAutomationPressResult, } from "./previewAutomationErrors"; import { explicitlySuppressesPreviewMiniPlayer, @@ -653,10 +654,11 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "press": { const ready = await requireReadyTab(); - return await ready.bridge.automation.press( + const result = await ready.bridge.automation.press( ready.runtimeTabId, request.input as Parameters[1], ); + return unwrapPreviewAutomationPressResult(result); } case "scroll": { const ready = await requireReadyTab(); diff --git a/apps/web/src/components/preview/previewAutomationErrors.ts b/apps/web/src/components/preview/previewAutomationErrors.ts index dcf35de53f2d..db33a7713568 100644 --- a/apps/web/src/components/preview/previewAutomationErrors.ts +++ b/apps/web/src/components/preview/previewAutomationErrors.ts @@ -1,4 +1,6 @@ import { + DesktopPreviewAutomationPressErrorSchema, + type DesktopPreviewAutomationPressResult, EnvironmentId, type PreviewAutomationHost, PreviewAutomationOperation, @@ -134,6 +136,75 @@ export class PreviewAutomationTargetNotEditableHostError extends Schema.TaggedEr } } +const previewAutomationKeyboardHostErrorFields = { + requestId: TrimmedNonEmptyString, + operation: PreviewAutomationOperation, + environmentId: EnvironmentId, + threadId: ThreadId, + tabId: Schema.NullOr(PreviewTabId), +}; + +export class PreviewAutomationKeyboardWindowNotFocusedHostError extends Schema.TaggedErrorClass()( + "PreviewAutomationKeyboardWindowNotFocusedHostError", + { + ...previewAutomationKeyboardHostErrorFields, + }, +) { + get responseTag() { + return "PreviewAutomationExecutionError" as const; + } + + override get message(): string { + return `Preview automation ${this.operation} request ${this.requestId} cannot send keyboard input because the desktop window is not focused.`; + } +} + +export class PreviewAutomationKeyboardFocusedFrameUnsupportedHostError extends Schema.TaggedErrorClass()( + "PreviewAutomationKeyboardFocusedFrameUnsupportedHostError", + { + ...previewAutomationKeyboardHostErrorFields, + }, +) { + get responseTag() { + return "PreviewAutomationExecutionError" as const; + } + + override get message(): string { + return `Preview automation ${this.operation} request ${this.requestId} cannot send keyboard input to a focused frame.`; + } +} + +export class PreviewAutomationKeyboardDeliveryNotConfirmedHostError extends Schema.TaggedErrorClass()( + "PreviewAutomationKeyboardDeliveryNotConfirmedHostError", + { + ...previewAutomationKeyboardHostErrorFields, + }, +) { + get responseTag() { + return "PreviewAutomationExecutionError" as const; + } + + override get message(): string { + return `Preview automation ${this.operation} request ${this.requestId} did not receive keyboard delivery confirmation from tab ${this.tabId ?? "unassigned"}.`; + } +} + +export const PreviewAutomationKeyboardHostError = Schema.Union([ + PreviewAutomationKeyboardWindowNotFocusedHostError, + PreviewAutomationKeyboardFocusedFrameUnsupportedHostError, + PreviewAutomationKeyboardDeliveryNotConfirmedHostError, +]); +export type PreviewAutomationKeyboardHostError = typeof PreviewAutomationKeyboardHostError.Type; +export const isPreviewAutomationKeyboardHostError = Schema.is(PreviewAutomationKeyboardHostError); + +const isPreviewAutomationPressError = Schema.is(DesktopPreviewAutomationPressErrorSchema); + +export function unwrapPreviewAutomationPressResult( + result: DesktopPreviewAutomationPressResult, +): void { + if (result._tag === "Failure") throw result.error; +} + const targetNotEditableDiagnostics = ( cause: unknown, ): { @@ -183,6 +254,28 @@ export class PreviewAutomationOperationError extends Schema.TaggedErrorClass
["error"],
+): unknown => {
+  try {
+    unwrapPreviewAutomationPressResult({ _tag: "Failure", error });
+  } catch (cause) {
+    return cause;
+  }
+  throw new Error("Expected the preview press result to fail.");
+};
+
 const request = (
   requestId: string,
   overrides: Partial = {},
@@ -324,6 +337,82 @@ describe("previewAutomationRequestConsumer", () => {
     });
   });
 
+  it.each([
+    {
+      tag: "PreviewAutomationKeyboardWindowNotFocusedError",
+      message:
+        "Preview automation press request request-press cannot send keyboard input because the desktop window is not focused.",
+    },
+    {
+      tag: "PreviewAutomationKeyboardFocusedFrameUnsupportedError",
+      message:
+        "Preview automation press request request-press cannot send keyboard input to a focused frame.",
+    },
+    {
+      tag: "PreviewAutomationKeyboardDeliveryNotConfirmedError",
+      message:
+        "Preview automation press request request-press did not receive keyboard delivery confirmation from tab tab-1.",
+    },
+  ] as const)("preserves the $tag message", ({ tag, message }) => {
+    expect(
+      serializePreviewAutomationError(
+        unwrapPressFailure({
+          _tag: tag,
+          tabId: "tab-1",
+          webContentsId: 42,
+        }),
+        {
+          requestId: "request-press",
+          operation: "press",
+          environmentId,
+          threadId,
+          tabId,
+        },
+      ),
+    ).toEqual({
+      _tag: "PreviewAutomationExecutionError",
+      message,
+      detail: {
+        requestId: "request-press",
+        operation: "press",
+        environmentId: "environment-1",
+        threadId: "thread-1",
+        tabId: "tab-1",
+      },
+    });
+  });
+
+  it("maps a replaced desktop keyboard target to an unavailable response", () => {
+    expect(
+      serializePreviewAutomationError(
+        unwrapPressFailure({
+          _tag: "PreviewAutomationTargetChangedError",
+          operation: "press",
+          tabId: "tab-1",
+          webContentsId: 42,
+        }),
+        {
+          requestId: "request-press",
+          operation: "press",
+          environmentId,
+          threadId,
+          tabId,
+        },
+      ),
+    ).toMatchObject({
+      _tag: "PreviewAutomationTabNotFoundError",
+      detail: { tabId: "tab-1", bridgeAvailable: true },
+    });
+  });
+
+  it("accepts a fulfilled desktop keyboard success", () => {
+    expect(
+      unwrapPreviewAutomationPressResult({
+        _tag: "Success",
+      }),
+    ).toBeUndefined();
+  });
+
   it("correlates unexpected failures without exposing cause details", () => {
     const cause = new Error("private bridge token: preview-secret");
     const context = {
diff --git a/packages/contracts/src/ipc.test.ts b/packages/contracts/src/ipc.test.ts
index 20db75368a9a..33e81ce49936 100644
--- a/packages/contracts/src/ipc.test.ts
+++ b/packages/contracts/src/ipc.test.ts
@@ -1,7 +1,17 @@
 import * as Schema from "effect/Schema";
 import { describe, expect, it } from "vite-plus/test";
 
-import { DesktopEnvironmentBootstrapSchema } from "./ipc.ts";
+import {
+  DesktopEnvironmentBootstrapSchema,
+  DesktopPreviewAutomationPressResultSchema,
+} from "./ipc.ts";
+
+const encodeAutomationPressResult = Schema.encodeUnknownSync(
+  DesktopPreviewAutomationPressResultSchema,
+);
+const decodeAutomationPressResult = Schema.decodeUnknownSync(
+  DesktopPreviewAutomationPressResultSchema,
+);
 
 describe("DesktopEnvironmentBootstrapSchema", () => {
   const decode = Schema.decodeUnknownSync(DesktopEnvironmentBootstrapSchema);
@@ -36,3 +46,62 @@ describe("DesktopEnvironmentBootstrapSchema", () => {
     ).toBeNull();
   });
 });
+
+describe("DesktopPreviewAutomationPressResultSchema", () => {
+  const roundTrip = (input: unknown) => {
+    const encoded = encodeAutomationPressResult(input);
+    return decodeAutomationPressResult(encoded);
+  };
+
+  it.each([
+    { _tag: "Success" },
+    {
+      _tag: "Failure",
+      error: {
+        _tag: "PreviewAutomationKeyboardWindowNotFocusedError",
+        tabId: "tab-1",
+        webContentsId: 41,
+      },
+    },
+    {
+      _tag: "Failure",
+      error: {
+        _tag: "PreviewAutomationKeyboardFocusedFrameUnsupportedError",
+        tabId: "tab-1",
+        webContentsId: 41,
+      },
+    },
+    {
+      _tag: "Failure",
+      error: {
+        _tag: "PreviewAutomationKeyboardDeliveryNotConfirmedError",
+        tabId: "tab-1",
+        webContentsId: 41,
+      },
+    },
+    {
+      _tag: "Failure",
+      error: {
+        _tag: "PreviewAutomationTargetChangedError",
+        operation: "press",
+        tabId: "tab-1",
+        webContentsId: 41,
+      },
+    },
+  ])("round-trips $error._tag", (result) => {
+    expect(roundTrip(result)).toEqual(result);
+  });
+
+  it("rejects unknown fulfilled failures", () => {
+    expect(() =>
+      decodeAutomationPressResult({
+        _tag: "Failure",
+        error: {
+          _tag: "PreviewOperationError",
+          tabId: "tab-1",
+          webContentsId: 41,
+        },
+      }),
+    ).toThrow();
+  });
+});
diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts
index 4c124b456239..081b326f4582 100644
--- a/packages/contracts/src/ipc.ts
+++ b/packages/contracts/src/ipc.ts
@@ -1032,6 +1032,41 @@ export const DesktopPreviewAutomationPressInputSchema = Schema.Struct({
   input: PreviewAutomationPressInput,
 });
 
+const DesktopPreviewAutomationKeyboardErrorFields = {
+  tabId: DesktopPreviewTabIdSchema,
+  webContentsId: Schema.Int.check(Schema.isGreaterThan(0)),
+};
+
+export const DesktopPreviewAutomationPressErrorSchema = Schema.Union([
+  Schema.TaggedStruct(
+    "PreviewAutomationKeyboardWindowNotFocusedError",
+    DesktopPreviewAutomationKeyboardErrorFields,
+  ),
+  Schema.TaggedStruct(
+    "PreviewAutomationKeyboardFocusedFrameUnsupportedError",
+    DesktopPreviewAutomationKeyboardErrorFields,
+  ),
+  Schema.TaggedStruct(
+    "PreviewAutomationKeyboardDeliveryNotConfirmedError",
+    DesktopPreviewAutomationKeyboardErrorFields,
+  ),
+  Schema.TaggedStruct("PreviewAutomationTargetChangedError", {
+    operation: Schema.String,
+    ...DesktopPreviewAutomationKeyboardErrorFields,
+  }),
+]);
+export type DesktopPreviewAutomationPressError =
+  typeof DesktopPreviewAutomationPressErrorSchema.Type;
+
+export const DesktopPreviewAutomationPressResultSchema = Schema.Union([
+  Schema.TaggedStruct("Success", {}),
+  Schema.TaggedStruct("Failure", {
+    error: DesktopPreviewAutomationPressErrorSchema,
+  }),
+]);
+export type DesktopPreviewAutomationPressResult =
+  typeof DesktopPreviewAutomationPressResultSchema.Type;
+
 export const DesktopPreviewAutomationScrollInputSchema = Schema.Struct({
   tabId: DesktopPreviewTabIdSchema,
   input: PreviewAutomationScrollInput,
@@ -1222,7 +1257,10 @@ export interface DesktopPreviewBridge {
     snapshot: (tabId: string) => Promise;
     click: (tabId: string, input: PreviewAutomationClickInput) => Promise;
     type: (tabId: string, input: PreviewAutomationTypeInput) => Promise;
-    press: (tabId: string, input: PreviewAutomationPressInput) => Promise;
+    press: (
+      tabId: string,
+      input: PreviewAutomationPressInput,
+    ) => Promise;
     scroll: (tabId: string, input: PreviewAutomationScrollInput) => Promise;
     evaluate: (tabId: string, input: PreviewAutomationEvaluateInput) => Promise;
     waitFor: (tabId: string, input: PreviewAutomationWaitForInput) => Promise;
diff --git a/packages/contracts/src/previewAutomation.test.ts b/packages/contracts/src/previewAutomation.test.ts
new file mode 100644
index 000000000000..b9938e49b0a2
--- /dev/null
+++ b/packages/contracts/src/previewAutomation.test.ts
@@ -0,0 +1,74 @@
+import { Schema } from "effect";
+import { describe, expect, it } from "vite-plus/test";
+
+import { PreviewAutomationPressInput, PreviewAutomationPressKey } from "./previewAutomation.ts";
+
+const decodePressKey = Schema.decodeUnknownSync(PreviewAutomationPressKey);
+const decodePressInput = Schema.decodeUnknownSync(PreviewAutomationPressInput);
+
+describe("preview automation press key", () => {
+  it.each(["Enter", "Space", "ArrowDown", "Delete", "F1", "F12", "!", "~", "0", "A", "z"] as const)(
+    "accepts %s",
+    (key) => {
+      expect(decodePressKey(key)).toBe(key);
+      expect(decodePressInput({ key })).toEqual({ key });
+    },
+  );
+
+  it("accepts every current named key", () => {
+    const namedKeys = [
+      "Escape",
+      "Backspace",
+      "Tab",
+      "Enter",
+      "Shift",
+      "Control",
+      "Alt",
+      "Meta",
+      "CapsLock",
+      "Space",
+      "PageUp",
+      "PageDown",
+      "End",
+      "Home",
+      "ArrowLeft",
+      "ArrowUp",
+      "ArrowRight",
+      "ArrowDown",
+      "Insert",
+      "Delete",
+    ];
+
+    for (const key of namedKeys) {
+      expect(decodePressKey(key)).toBe(key);
+    }
+  });
+
+  it("accepts F1 through F12", () => {
+    for (let index = 1; index <= 12; index += 1) {
+      const key = `F${index}`;
+      expect(decodePressKey(key)).toBe(key);
+    }
+  });
+
+  it("accepts every printable ASCII character except space", () => {
+    for (let codePoint = 33; codePoint <= 126; codePoint += 1) {
+      const key = String.fromCodePoint(codePoint);
+      expect(decodePressKey(key)).toBe(key);
+    }
+  });
+
+  it("directs Unicode text to preview_type in the agent-facing schema", () => {
+    const jsonSchema = Schema.toJsonSchemaDocument(PreviewAutomationPressInput).schema;
+
+    expect(JSON.stringify(jsonSchema)).toContain("Use preview_type for Unicode text.");
+  });
+
+  it.each([" ", "Return", "F0", "F13", "\u00e9", "\u{1f642}"])(
+    "rejects unsupported key %s",
+    (key) => {
+      expect(() => decodePressKey(key)).toThrow();
+      expect(() => decodePressInput({ key })).toThrow();
+    },
+  );
+});
diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts
index e33615fa4c05..fe067367911a 100644
--- a/packages/contracts/src/previewAutomation.ts
+++ b/packages/contracts/src/previewAutomation.ts
@@ -359,19 +359,148 @@ export const PreviewAutomationTypeInput = Schema.Struct({
   });
 export type PreviewAutomationTypeInput = typeof PreviewAutomationTypeInput.Type;
 
+const PREVIEW_AUTOMATION_PRESS_KEYS = [
+  "Escape",
+  "Backspace",
+  "Tab",
+  "Enter",
+  "Shift",
+  "Control",
+  "Alt",
+  "Meta",
+  "CapsLock",
+  "Space",
+  "PageUp",
+  "PageDown",
+  "End",
+  "Home",
+  "ArrowLeft",
+  "ArrowUp",
+  "ArrowRight",
+  "ArrowDown",
+  "Insert",
+  "Delete",
+  "F1",
+  "F2",
+  "F3",
+  "F4",
+  "F5",
+  "F6",
+  "F7",
+  "F8",
+  "F9",
+  "F10",
+  "F11",
+  "F12",
+  "!",
+  '"',
+  "#",
+  "$",
+  "%",
+  "&",
+  "'",
+  "(",
+  ")",
+  "*",
+  "+",
+  ",",
+  "-",
+  ".",
+  "/",
+  "0",
+  "1",
+  "2",
+  "3",
+  "4",
+  "5",
+  "6",
+  "7",
+  "8",
+  "9",
+  ":",
+  ";",
+  "<",
+  "=",
+  ">",
+  "?",
+  "@",
+  "A",
+  "B",
+  "C",
+  "D",
+  "E",
+  "F",
+  "G",
+  "H",
+  "I",
+  "J",
+  "K",
+  "L",
+  "M",
+  "N",
+  "O",
+  "P",
+  "Q",
+  "R",
+  "S",
+  "T",
+  "U",
+  "V",
+  "W",
+  "X",
+  "Y",
+  "Z",
+  "[",
+  "\\",
+  "]",
+  "^",
+  "_",
+  "`",
+  "a",
+  "b",
+  "c",
+  "d",
+  "e",
+  "f",
+  "g",
+  "h",
+  "i",
+  "j",
+  "k",
+  "l",
+  "m",
+  "n",
+  "o",
+  "p",
+  "q",
+  "r",
+  "s",
+  "t",
+  "u",
+  "v",
+  "w",
+  "x",
+  "y",
+  "z",
+  "{",
+  "|",
+  "}",
+  "~",
+] as const;
+
+const PREVIEW_AUTOMATION_PRESS_KEY_DESCRIPTION =
+  "Named key, F1 through F12, or one printable ASCII character from ! through ~. Use Space for the space key. Use preview_type for Unicode text.";
+
+export const PreviewAutomationPressKey = Schema.Literals(PREVIEW_AUTOMATION_PRESS_KEYS).annotate({
+  description: PREVIEW_AUTOMATION_PRESS_KEY_DESCRIPTION,
+});
+export type PreviewAutomationPressKey = typeof PreviewAutomationPressKey.Type;
+
 export const PreviewAutomationPressInput = Schema.Struct({
   ...PreviewAutomationTabTargetFields,
-  key: Schema.String.check(Schema.isTrimmed())
-    .check(
-      Schema.isNonEmpty({
-        description:
-          "Keyboard key name such as Enter, Escape, Tab, ArrowDown, Backspace, or a single character.",
-      }),
-    )
-    .annotateKey({
-      description:
-        "Keyboard key name such as Enter, Escape, Tab, ArrowDown, Backspace, or a single character.",
-    }),
+  key: PreviewAutomationPressKey.annotateKey({
+    description: PREVIEW_AUTOMATION_PRESS_KEY_DESCRIPTION,
+  }),
   modifiers: Schema.optional(
     Schema.Array(Schema.Literals(["Alt", "Control", "Meta", "Shift"])).annotate({
       description: "Modifier keys held while pressing key.",