diff --git a/src/firefox/dom.ts b/src/firefox/dom.ts index 707b1161..197448b4 100644 --- a/src/firefox/dom.ts +++ b/src/firefox/dom.ts @@ -4,6 +4,134 @@ import { By, Key, WebDriver, WebElement } from 'selenium-webdriver'; +/** + * Key names accepted by press_key, mapped to the WebDriver unicode code points + * that selenium exposes as `Key.*`. Names are matched case-insensitively. + * `enter` and `return` are deliberately distinct, matching selenium: `return` is + * the main keyboard key, `enter` is the numpad one. + */ +const KEY_MAP: Record = { + cancel: Key.CANCEL, + help: Key.HELP, + backspace: Key.BACK_SPACE, + tab: Key.TAB, + clear: Key.CLEAR, + return: Key.RETURN, + enter: Key.ENTER, + pause: Key.PAUSE, + escape: Key.ESCAPE, + esc: Key.ESCAPE, + space: Key.SPACE, + pageup: Key.PAGE_UP, + pagedown: Key.PAGE_DOWN, + end: Key.END, + home: Key.HOME, + arrowleft: Key.ARROW_LEFT, + left: Key.ARROW_LEFT, + arrowup: Key.ARROW_UP, + up: Key.ARROW_UP, + arrowright: Key.ARROW_RIGHT, + right: Key.ARROW_RIGHT, + arrowdown: Key.ARROW_DOWN, + down: Key.ARROW_DOWN, + insert: Key.INSERT, + delete: Key.DELETE, + semicolon: Key.SEMICOLON, + equals: Key.EQUALS, + numpad0: Key.NUMPAD0, + numpad1: Key.NUMPAD1, + numpad2: Key.NUMPAD2, + numpad3: Key.NUMPAD3, + numpad4: Key.NUMPAD4, + numpad5: Key.NUMPAD5, + numpad6: Key.NUMPAD6, + numpad7: Key.NUMPAD7, + numpad8: Key.NUMPAD8, + numpad9: Key.NUMPAD9, + multiply: Key.MULTIPLY, + add: Key.ADD, + separator: Key.SEPARATOR, + subtract: Key.SUBTRACT, + decimal: Key.DECIMAL, + divide: Key.DIVIDE, + f1: Key.F1, + f2: Key.F2, + f3: Key.F3, + f4: Key.F4, + f5: Key.F5, + f6: Key.F6, + f7: Key.F7, + f8: Key.F8, + f9: Key.F9, + f10: Key.F10, + f11: Key.F11, + f12: Key.F12, +}; + +const MODIFIER_MAP: Record = { + ctrl: Key.CONTROL, + control: Key.CONTROL, + alt: Key.ALT, + shift: Key.SHIFT, + meta: Key.META, + cmd: Key.META, + command: Key.META, + win: Key.META, + super: Key.META, +}; + +export interface KeyCombo { + modifiers: string[]; + key: string; +} + +/** + * Parse a combination such as "ctrl+shift+t" into its modifiers and its single + * non-modifier key. Any number of modifiers is allowed, but exactly one key is: + * "ctrl+k+l" is rejected rather than silently dropping one of the two. + * Unrecognised names are rejected too, so that a mistyped key name does not turn + * into typed text. + * @param combo Key name, single character, or "+"-separated combination + */ +export function parseKeyCombo(combo: string): KeyCombo { + const trimmed = combo.trim(); + // A lone character is taken literally so that "+" itself can be pressed. + const parts = [...trimmed].length === 1 ? [trimmed] : trimmed.split('+'); + + const modifiers: string[] = []; + let key: string | undefined; + + for (const part of parts) { + const name = part.trim(); + if (name === '') { + continue; + } + + const modifier = MODIFIER_MAP[name.toLowerCase()]; + if (modifier) { + modifiers.push(modifier); + continue; + } + + const mapped = KEY_MAP[name.toLowerCase()]; + if (mapped === undefined && [...name].length > 1) { + throw new Error(`press_key: unknown key "${name}" in "${combo}"`); + } + if (key !== undefined) { + throw new Error( + `press_key: "${combo}" has more than one non-modifier key. Use any number of modifiers but a single key, for example "ctrl+shift+t".` + ); + } + key = mapped ?? name; + } + + if (key === undefined) { + throw new Error(`press_key: no key specified in "${combo}"`); + } + + return { modifiers, key }; +} + export class DomInteractions { constructor( private driver: WebDriver, @@ -288,6 +416,38 @@ export class DomInteractions { await this.waitForEventsAfterAction(); } + /** + * Press a single key, optionally with modifiers. + * @param key Key name or combination, such as "Escape", "F5" or "ctrl+shift+t" + * @param uid Element UID to send the key to. Defaults to the focused element. + */ + async pressKey(key: string, uid?: string): Promise { + const { modifiers, key: mainKey } = parseKeyCombo(key); + + if (uid) { + if (!this.resolveUid) { + throw new Error('pressKey: resolveUid callback not set. Ensure snapshot is initialized.'); + } + const el = await this.resolveUid(uid); + await el.sendKeys(Key.chord(...modifiers, mainKey)); + } else { + // Without an element to target, drive the WebDriver actions keyboard + // source directly so the key reaches whatever currently has focus. + const actions = this.driver.actions({ async: true }); + for (const modifier of modifiers) { + actions.keyDown(modifier); + } + actions.keyDown(mainKey); + actions.keyUp(mainKey); + for (const modifier of [...modifiers].reverse()) { + actions.keyUp(modifier); + } + await actions.perform(); + } + + await this.waitForEventsAfterAction(); + } + /** * Wait for events to propagate after user action * Gives the page time to respond to interactions diff --git a/src/firefox/index.ts b/src/firefox/index.ts index 652d6ec9..a83831e9 100644 --- a/src/firefox/index.ts +++ b/src/firefox/index.ts @@ -225,6 +225,13 @@ export class FirefoxClient { return await this.dom.uploadFileByUid(uid, filePath); } + async pressKey(key: string, uid?: string): Promise { + if (!this.dom) { + throw new Error('Not connected'); + } + return await this.dom.pressKey(key, uid); + } + // ============================================================================ // Console // ============================================================================ diff --git a/src/tools/input.ts b/src/tools/input.ts index 37ff43e1..b8290a31 100644 --- a/src/tools/input.ts +++ b/src/tools/input.ts @@ -147,6 +147,30 @@ export const uploadFileByUidTool = { }, }; +export const pressKeyTool = { + name: 'press_key', + description: + 'Press a single key, optionally with modifiers, to submit, dismiss, navigate or trigger a shortcut. Not for entering text: use fill_by_uid instead.', + annotations: { + readOnlyHint: false, + }, + inputSchema: { + type: 'object', + properties: { + key: { + type: 'string', + description: + 'One key, optionally preceded by "+"-separated modifiers, such as "Escape", "F5" or "ctrl+shift+t". Modifiers: ctrl, alt, shift, meta. Named keys: Enter (numpad), Return, Tab, Backspace, Delete, Insert, Space, Escape, Home, End, PageUp, PageDown, ArrowUp, ArrowDown, ArrowLeft, ArrowRight, F1-F12, Numpad0-Numpad9, Clear, Pause, Help, Cancel, Semicolon, Equals, Add, Subtract, Multiply, Divide, Decimal, Separator. Anything else must be a single character.', + }, + uid: { + type: 'string', + description: 'Element UID from snapshot (default: the focused element)', + }, + }, + required: ['key'], + }, +}; + // Handlers export async function handleClickByUid(args: unknown): Promise { try { @@ -327,6 +351,31 @@ export async function handleUploadFileByUid(args: unknown): Promise { + try { + const { key, uid } = args as { key: string; uid?: string }; + + if (!key || typeof key !== 'string') { + throw new Error('key parameter is required and must be a string'); + } + + const { getFirefox } = await import('../index.js'); + const firefox = await getFirefox(); + + try { + await firefox.pressKey(key, uid); + return successResponse(uid ? `press_key ${key} on ${uid}` : `press_key ${key}`); + } catch (error) { + if (uid) { + throw handleUidError(error as Error, uid); + } + throw error; + } + } catch (error) { + return errorResponse(error as Error); + } +} + export const module = defineModule({ name: 'input', description: 'Interact with the page via UID-based clicks, typing, drag, and uploads.', @@ -337,5 +386,6 @@ export const module = defineModule({ [dragByUidToUidTool, handleDragByUidToUid], [fillFormByUidTool, handleFillFormByUid], [uploadFileByUidTool, handleUploadFileByUid], + [pressKeyTool, handlePressKey], ], }); diff --git a/tests/firefox/key-combo.test.ts b/tests/firefox/key-combo.test.ts new file mode 100644 index 00000000..2cb124f1 --- /dev/null +++ b/tests/firefox/key-combo.test.ts @@ -0,0 +1,114 @@ +/** + * Unit tests for press_key combination parsing + */ + +import { describe, it, expect } from 'vitest'; +import { Key } from 'selenium-webdriver'; +import { parseKeyCombo } from '../../src/firefox/dom.js'; + +describe('parseKeyCombo', () => { + describe('named keys', () => { + it('maps return and enter to the distinct selenium keys', () => { + expect(Key.RETURN).not.toBe(Key.ENTER); + expect(parseKeyCombo('Return')).toEqual({ modifiers: [], key: Key.RETURN }); + expect(parseKeyCombo('Enter')).toEqual({ modifiers: [], key: Key.ENTER }); + }); + + it('matches key names case-insensitively', () => { + expect(parseKeyCombo('escape').key).toBe(Key.ESCAPE); + expect(parseKeyCombo('Escape').key).toBe(Key.ESCAPE); + expect(parseKeyCombo('ESCAPE').key).toBe(Key.ESCAPE); + }); + + it('accepts short aliases', () => { + expect(parseKeyCombo('esc').key).toBe(Key.ESCAPE); + expect(parseKeyCombo('up').key).toBe(Key.ARROW_UP); + expect(parseKeyCombo('ArrowUp').key).toBe(Key.ARROW_UP); + expect(parseKeyCombo('down').key).toBe(Key.ARROW_DOWN); + expect(parseKeyCombo('left').key).toBe(Key.ARROW_LEFT); + expect(parseKeyCombo('right').key).toBe(Key.ARROW_RIGHT); + }); + + it('supports function, navigation and numpad keys', () => { + expect(parseKeyCombo('F1').key).toBe(Key.F1); + expect(parseKeyCombo('F12').key).toBe(Key.F12); + expect(parseKeyCombo('PageDown').key).toBe(Key.PAGE_DOWN); + expect(parseKeyCombo('Home').key).toBe(Key.HOME); + expect(parseKeyCombo('Backspace').key).toBe(Key.BACK_SPACE); + expect(parseKeyCombo('Space').key).toBe(Key.SPACE); + expect(parseKeyCombo('Numpad7').key).toBe(Key.NUMPAD7); + expect(parseKeyCombo('Subtract').key).toBe(Key.SUBTRACT); + }); + }); + + describe('modifiers', () => { + it('separates modifiers from the key', () => { + expect(parseKeyCombo('ctrl+l')).toEqual({ modifiers: [Key.CONTROL], key: 'l' }); + }); + + it('keeps every modifier of a combination, in order', () => { + expect(parseKeyCombo('ctrl+shift+t')).toEqual({ + modifiers: [Key.CONTROL, Key.SHIFT], + key: 't', + }); + }); + + it('accepts a modifier after the key', () => { + expect(parseKeyCombo('F4+alt')).toEqual({ modifiers: [Key.ALT], key: Key.F4 }); + }); + + it('accepts modifier aliases', () => { + expect(parseKeyCombo('control+a').modifiers).toEqual([Key.CONTROL]); + for (const alias of ['meta', 'cmd', 'command', 'win', 'super']) { + expect(parseKeyCombo(`${alias}+a`).modifiers).toEqual([Key.META]); + } + }); + + it('combines modifiers with named keys', () => { + expect(parseKeyCombo('alt+ArrowLeft')).toEqual({ + modifiers: [Key.ALT], + key: Key.ARROW_LEFT, + }); + }); + + it('ignores whitespace around each part', () => { + expect(parseKeyCombo(' ctrl + shift + t ')).toEqual({ + modifiers: [Key.CONTROL, Key.SHIFT], + key: 't', + }); + }); + }); + + describe('single characters', () => { + it('passes an unmapped single character through unchanged', () => { + expect(parseKeyCombo('a').key).toBe('a'); + expect(parseKeyCombo('7').key).toBe('7'); + }); + + it('preserves the case of a single character', () => { + expect(parseKeyCombo('A').key).toBe('A'); + }); + + it('takes a lone separator as the plus key', () => { + expect(parseKeyCombo('+')).toEqual({ modifiers: [], key: '+' }); + }); + }); + + describe('rejected input', () => { + it('rejects more than one non-modifier key', () => { + expect(() => parseKeyCombo('ctrl+k+l')).toThrow(/more than one non-modifier key/); + expect(() => parseKeyCombo('a+b')).toThrow(/more than one non-modifier key/); + }); + + it('rejects an unknown key name rather than typing it', () => { + expect(() => parseKeyCombo('foobar')).toThrow(/unknown key "foobar"/); + expect(() => parseKeyCombo('ctrl+hello')).toThrow(/unknown key "hello"/); + }); + + it('rejects a combination with no key', () => { + expect(() => parseKeyCombo('ctrl+shift')).toThrow(/no key specified/); + expect(() => parseKeyCombo('')).toThrow(/no key specified/); + expect(() => parseKeyCombo(' ')).toThrow(/no key specified/); + }); + }); +}); diff --git a/tests/tools/input.test.ts b/tests/tools/input.test.ts index af6ec86d..8bf0f1eb 100644 --- a/tests/tools/input.test.ts +++ b/tests/tools/input.test.ts @@ -2,7 +2,7 @@ * Unit tests for input tools */ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { clickByUidTool, hoverByUidTool, @@ -10,8 +10,13 @@ import { dragByUidToUidTool, fillFormByUidTool, uploadFileByUidTool, + pressKeyTool, } from '../../src/tools/input.js'; +function textOf(result: { content: unknown[] }): string { + return (result.content[0] as { type: 'text'; text: string }).text; +} + describe('Input Tools', () => { describe('Tool Definitions', () => { it('should have correct tool names', () => { @@ -21,6 +26,7 @@ describe('Input Tools', () => { expect(dragByUidToUidTool.name).toBe('drag_by_uid_to_uid'); expect(fillFormByUidTool.name).toBe('fill_form_by_uid'); expect(uploadFileByUidTool.name).toBe('upload_file_by_uid'); + expect(pressKeyTool.name).toBe('press_key'); }); it('should have valid descriptions', () => { @@ -30,6 +36,12 @@ describe('Input Tools', () => { expect(dragByUidToUidTool.description).toContain('drag'); expect(fillFormByUidTool.description).toContain('form'); expect(uploadFileByUidTool.description).toContain('Upload'); + expect(pressKeyTool.description).toContain('Press'); + }); + + it('should steer press_key away from entering text', () => { + expect(pressKeyTool.description).toMatch(/single key/i); + expect(pressKeyTool.description).toContain('fill_by_uid'); }); it('should have valid input schemas', () => { @@ -39,6 +51,7 @@ describe('Input Tools', () => { expect(dragByUidToUidTool.inputSchema.type).toBe('object'); expect(fillFormByUidTool.inputSchema.type).toBe('object'); expect(uploadFileByUidTool.inputSchema.type).toBe('object'); + expect(pressKeyTool.inputSchema.type).toBe('object'); }); }); @@ -92,5 +105,78 @@ describe('Input Tools', () => { expect(required).toContain('uid'); expect(required).toContain('filePath'); }); + + it('pressKeyTool should require key and accept an optional uid', () => { + const { properties, required } = pressKeyTool.inputSchema; + expect(properties).toBeDefined(); + expect(properties?.key.type).toBe('string'); + expect(properties?.uid.type).toBe('string'); + expect(required).toEqual(['key']); + }); + }); + + describe('handlePressKey', () => { + let pressKey: ReturnType; + + beforeEach(() => { + vi.resetModules(); + pressKey = vi.fn().mockResolvedValue(undefined); + vi.doMock('../../src/index.js', () => ({ + args: {}, + getFirefox: vi.fn().mockResolvedValue({ pressKey }), + })); + }); + + afterEach(() => { + vi.doUnmock('../../src/index.js'); + vi.restoreAllMocks(); + }); + + it('should send the key to the focused element when no uid is given', async () => { + const { handlePressKey } = await import('../../src/tools/input.js'); + const result = await handlePressKey({ key: 'ctrl+shift+t' }); + + expect(result.isError).toBeUndefined(); + expect(pressKey).toHaveBeenCalledWith('ctrl+shift+t', undefined); + expect(textOf(result)).toContain('ctrl+shift+t'); + }); + + it('should send the key to the given uid', async () => { + const { handlePressKey } = await import('../../src/tools/input.js'); + const result = await handlePressKey({ key: 'Escape', uid: 'uid-3' }); + + expect(result.isError).toBeUndefined(); + expect(pressKey).toHaveBeenCalledWith('Escape', 'uid-3'); + expect(textOf(result)).toContain('uid-3'); + }); + + it('should reject a missing or non-string key', async () => { + const { handlePressKey } = await import('../../src/tools/input.js'); + + for (const args of [{}, { key: '' }, { key: 42 }]) { + const result = await handlePressKey(args); + expect(result.isError).toBe(true); + expect(textOf(result)).toContain('key parameter is required'); + } + expect(pressKey).not.toHaveBeenCalled(); + }); + + it('should report a stale uid as such', async () => { + pressKey.mockRejectedValue(new Error('UID uid-9 not found in snapshot')); + const { handlePressKey } = await import('../../src/tools/input.js'); + const result = await handlePressKey({ key: 'Enter', uid: 'uid-9' }); + + expect(result.isError).toBe(true); + expect(textOf(result)).toContain('take_snapshot'); + }); + + it('should surface a parse error unchanged', async () => { + pressKey.mockRejectedValue(new Error('press_key: unknown key "foobar" in "foobar"')); + const { handlePressKey } = await import('../../src/tools/input.js'); + const result = await handlePressKey({ key: 'foobar', uid: 'uid-1' }); + + expect(result.isError).toBe(true); + expect(textOf(result)).toContain('unknown key "foobar"'); + }); }); });