Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ Both flags are required because the MCP uses both WebDriver Classic (`--marionet

- Pages: list/new/navigate/select/close/get_page_text (get_page_text supports optional `saveTo`)
- Snapshot/UID: take/resolve/clear (take supports optional `saveTo`)
- Input: click/hover/fill/drag/upload/form fill
- Input: click/hover/fill/drag/upload/form fill, press_key (key combinations on the focused element), type_text (typing with an optional submit key)
- Network: list/get (ID‑first, filters, always‑on capture; both support optional `saveTo`)
- Downloads: list_downloads/clear_downloads (always‑on capture), set_download_behavior (allow/deny/default)
- Console: list/clear (list supports optional `saveTo`)
Expand Down
85 changes: 85 additions & 0 deletions src/firefox/dom.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,22 @@
*/

import { By, Key, WebDriver, WebElement } from 'selenium-webdriver';
import type { Actions } from 'selenium-webdriver/lib/input.js';
import { parseKeyCombination, type ParsedKeyCombination } from '../utils/keyboard.js';

/**
* Append a key press to an action sequence: hold the modifiers, press the key,
* then release the modifiers in reverse order.
*/
function appendKeyCombination(actions: Actions, { modifiers, key }: ParsedKeyCombination): void {
for (const modifier of modifiers) {
actions.keyDown(modifier);
}
actions.sendKeys(key);
for (const modifier of [...modifiers].reverse()) {
actions.keyUp(modifier);
}
}

export class DomInteractions {
constructor(
Expand Down Expand Up @@ -288,6 +304,75 @@ export class DomInteractions {
await this.waitForEventsAfterAction();
}

// ============================================================================
// Keyboard input
// ============================================================================

/**
* Press a key or key combination on the focused element, e.g. "Enter" or
* "Control+Shift+R". Modifiers are held for the duration of the key press
* and released afterwards.
*/
async pressKey(combination: string): Promise<void> {
const parsed = parseKeyCombination(combination);

const actions = this.driver.actions();
appendKeyCombination(actions, parsed);
await this.performKeyActions(actions);

await this.waitForEventsAfterAction();
}

/**
* Type text into the focused element, optionally followed by a single key
* such as "Enter" or "Tab".
*/
async typeText(text: string, submitKey?: string): Promise<void> {
// Parse before typing so an invalid submitKey fails without leaving the
// page half-filled.
const parsed = submitKey === undefined ? null : parseKeyCombination(submitKey);

// Text and submit key go into a single sequence: performed separately, a
// concurrent tool call could move focus in between and the submit key
// would land on a different element.
const actions = this.driver.actions();
if (text.length > 0) {
actions.sendKeys(text);
}
if (parsed) {
appendKeyCombination(actions, parsed);
}
await this.performKeyActions(actions);

await this.waitForEventsAfterAction();
}

/**
* Perform a keyboard action sequence, releasing anything left held down if
* the sequence fails part way through. Without this a failed combination
* would leave its modifiers logically pressed for the rest of the session.
*/
private async performKeyActions(actions: Actions): Promise<void> {
try {
await actions.perform();
} catch (error) {
await this.releaseHeldKeys();
throw error;
}
}

/**
* Release every key and button the session is currently holding down.
* Best effort: this runs while recovering from a failed action sequence.
*/
private async releaseHeldKeys(): Promise<void> {
try {
await this.driver.actions().clear();
} catch {
// Session may already be gone; the original error is the useful one.
}
}

/**
* Wait for events to propagate after user action
* Gives the page time to respond to interactions
Expand Down
16 changes: 16 additions & 0 deletions src/firefox/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,22 @@ export class FirefoxClient {
return await this.dom.uploadFileByUid(uid, filePath);
}

// Keyboard methods, targeting whatever element currently has focus

async pressKey(combination: string): Promise<void> {
if (!this.dom) {
throw new Error('Not connected');
}
return await this.dom.pressKey(combination);
}

async typeText(text: string, submitKey?: string): Promise<void> {
if (!this.dom) {
throw new Error('Not connected');
}
return await this.dom.typeText(text, submitKey);
}

// ============================================================================
// Console
// ============================================================================
Expand Down
99 changes: 99 additions & 0 deletions src/tools/input.ts
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,50 @@ export const uploadFileByUidTool = {
},
};

export const pressKeyTool = {
name: 'press_key',
description:
'Press a key or key combination on the focused element, e.g. "Enter", "Escape" or "Control+Shift+R". Use click_by_uid or fill_by_uid first to move focus.',
annotations: {
readOnlyHint: false,
},
inputSchema: {
type: 'object',
properties: {
key: {
type: 'string',
description:
'Key or combination. Single characters ("a", "/"), named keys ("Enter", "Escape", "Tab", "Backspace", "Delete", "Home", "End", "PageUp", "PageDown", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "F1"-"F12", "Space"), optionally prefixed with "Control+", "Shift+", "Alt+" or "Meta+".',
},
},
required: ['key'],
},
};

export const typeTextTool = {
name: 'type_text',
description:
'Type text into the focused element, optionally followed by a key such as Enter. Use fill_by_uid to replace the value of a known input; use this for elements that only react to real typing, such as autocomplete and rich text editors.',
annotations: {
readOnlyHint: false,
},
inputSchema: {
type: 'object',
properties: {
text: {
type: 'string',
description: 'Text to type',
},
submitKey: {
type: 'string',
minLength: 1,
description: 'Key to press after typing, e.g. "Enter" or "Tab"',
},
},
required: ['text'],
},
};

// Handlers
export async function handleClickByUid(args: unknown): Promise<McpToolResponse> {
try {
Expand Down Expand Up @@ -327,6 +371,59 @@ export async function handleUploadFileByUid(args: unknown): Promise<McpToolRespo
}
}

export async function handlePressKey(args: unknown): Promise<McpToolResponse> {
try {
const { key } = (args as { key: string }) || {};

if (!key || typeof key !== 'string') {
throw new Error('key parameter is required and must be a string');
}

const { getFirefox } = await import('../index.js');
const firefox = await getFirefox();

await firefox.pressKey(key);

return successResponse(`press ${key}`);
} catch (error) {
return errorResponse(error as Error);
}
}

export async function handleTypeText(args: unknown): Promise<McpToolResponse> {
try {
const { text, submitKey } = (args as { text: string; submitKey?: string }) || {};

if (typeof text !== 'string') {
throw new Error('text parameter is required and must be a string');
}

if (submitKey !== undefined) {
if (typeof submitKey !== 'string') {
throw new Error('submitKey parameter must be a string');
}
// An empty string would otherwise be silently dropped and reported as a
// success that never pressed anything.
if (submitKey.length === 0) {
throw new Error('submitKey parameter must not be empty');
}
}

const { getFirefox } = await import('../index.js');
const firefox = await getFirefox();

await firefox.typeText(text, submitKey);

return successResponse(
submitKey !== undefined
? `typed ${text.length} chars, then ${submitKey}`
: `typed ${text.length} chars`
);
} catch (error) {
return errorResponse(error as Error);
}
}

export const module = defineModule({
name: 'input',
description: 'Interact with the page via UID-based clicks, typing, drag, and uploads.',
Expand All @@ -337,5 +434,7 @@ export const module = defineModule({
[dragByUidToUidTool, handleDragByUidToUid],
[fillFormByUidTool, handleFillFormByUid],
[uploadFileByUidTool, handleUploadFileByUid],
[pressKeyTool, handlePressKey],
[typeTextTool, handleTypeText],
],
});
130 changes: 130 additions & 0 deletions src/utils/keyboard.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
/**
* Parsing of key combinations for the keyboard input tools.
*
* Key names follow the DOM KeyboardEvent.key vocabulary ("Enter", "ArrowDown",
* "a") so that a model can name keys the same way it would in page code, and
* are mapped onto the Selenium Key constants used by the actions API.
*/

import { Key } from 'selenium-webdriver';

/** Modifier names, lowercased, mapped to their Selenium constant. */
const MODIFIERS: Record<string, string> = {
control: Key.CONTROL,
ctrl: Key.CONTROL,
shift: Key.SHIFT,
alt: Key.ALT,
option: Key.ALT,
meta: Key.META,
command: Key.COMMAND,
cmd: Key.COMMAND,
};

/** Non-printable key names, lowercased, mapped to their Selenium constant. */
const NAMED_KEYS: Record<string, string> = {
enter: Key.ENTER,
return: Key.RETURN,
escape: Key.ESCAPE,
esc: Key.ESCAPE,
tab: Key.TAB,
backspace: Key.BACK_SPACE,
delete: Key.DELETE,
del: Key.DELETE,
insert: Key.INSERT,
space: Key.SPACE,
home: Key.HOME,
end: Key.END,
pageup: Key.PAGE_UP,
pagedown: Key.PAGE_DOWN,
arrowup: Key.ARROW_UP,
arrowdown: Key.ARROW_DOWN,
arrowleft: Key.ARROW_LEFT,
arrowright: Key.ARROW_RIGHT,
f1: Key.F1,
f2: Key.F2,
f3: Key.F3,
f4: Key.F4,
f5: Key.F5,
f6: Key.F6,
f7: Key.F7,
f8: Key.F8,
f9: Key.F9,
f10: Key.F10,
f11: Key.F11,
f12: Key.F12,
};

export interface ParsedKeyCombination {
/** Selenium constants for the modifiers to hold down, in the given order. */
modifiers: string[];
/** Selenium constant or single character for the key to press. */
key: string;
}

function supportedKeyNames(): string {
return Object.keys(NAMED_KEYS).join(', ');
}

/**
* Split a combination into its modifier names and the final key name.
* A trailing "+" is treated as the literal plus key, so "Control++" and "+"
* both work.
*/
function splitCombination(combination: string): { modifierNames: string[]; keyName: string } {
if (combination === '+') {
return { modifierNames: [], keyName: '+' };
}
if (combination.endsWith('++')) {
return {
modifierNames: combination.slice(0, -2).split('+'),
keyName: '+',
};
}
const parts = combination.split('+');
const keyName = parts.pop() ?? '';
return { modifierNames: parts, keyName };
}

/**
* Parse a combination such as "Enter", "Control+a" or "Control+Shift+R" into
* the modifiers to hold and the key to press. Throws with a descriptive error
* when a name is not recognised.
*/
export function parseKeyCombination(combination: string): ParsedKeyCombination {
if (!combination || typeof combination !== 'string') {
throw new Error('key parameter is required and must be a string');
}

const { modifierNames, keyName } = splitCombination(combination.trim());

const modifiers: string[] = [];
for (const name of modifierNames) {
const modifier = MODIFIERS[name.trim().toLowerCase()];
if (!modifier) {
throw new Error(
`Unknown modifier "${name}" in key combination "${combination}". ` +
`Supported modifiers: ${Object.keys(MODIFIERS).join(', ')}.`
);
}
modifiers.push(modifier);
}

if (keyName.length === 0) {
throw new Error(`Missing key in combination "${combination}".`);
}

const named = NAMED_KEYS[keyName.toLowerCase()];
if (named) {
return { modifiers, key: named };
}

// Anything else must be a single printable character, e.g. "a" or "/".
if ([...keyName].length === 1) {
return { modifiers, key: keyName };
}

throw new Error(
`Unknown key "${keyName}" in key combination "${combination}". ` +
`Use a single character or one of: ${supportedKeyNames()}.`
);
}
Loading