diff --git a/lib/src/cli/serve.dart b/lib/src/cli/serve.dart index 6dfb1841..413167e5 100644 --- a/lib/src/cli/serve.dart +++ b/lib/src/cli/serve.dart @@ -1423,17 +1423,3 @@ Future> _handleQrLoginWaitServe( 'QR code may have expired. Call qr_login_start again for a fresh code.', }; } - -/// Check if Chrome is already running on the specified port -Future _checkForExistingChrome(int port) async { - try { - final uri = Uri.parse('http://127.0.0.1:$port/json'); - final client = HttpClient(); - final request = await client.getUrl(uri); - request.headers.set('Accept', 'application/json'); - final response = await request.close(); - return response.statusCode == 200; - } catch (e) { - return false; - } -} diff --git a/packaging/npm/dart/bin/server.dart b/packaging/npm/dart/bin/server.dart index 11469ef1..65dd8b6d 100644 --- a/packaging/npm/dart/bin/server.dart +++ b/packaging/npm/dart/bin/server.dart @@ -1,3 +1,3 @@ -import 'package:flutter_skill_npm/src/cli/server.dart'; +import 'package:flutter_skill/src/cli/server.dart'; void main(List args) => runServer(args); diff --git a/packaging/npm/dart/lib/flutter_skill.dart b/packaging/npm/dart/lib/flutter_skill.dart index cbb2a219..2e8355cf 100644 --- a/packaging/npm/dart/lib/flutter_skill.dart +++ b/packaging/npm/dart/lib/flutter_skill.dart @@ -1,26 +1,216 @@ import 'dart:async'; import 'dart:convert'; import 'dart:developer' as developer; +import 'dart:math'; import 'dart:ui' as ui; +import 'package:flutter/foundation.dart'; +import 'package:flutter/gestures.dart' show PointerAddedEvent; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; +import 'package:flutter/scheduler.dart'; import 'package:flutter/services.dart'; +import 'flutter_skill_semantic_refs.dart'; + +// Conditional web import for JS interop +import 'flutter_skill_web_stub.dart' + if (dart.library.js_interop) 'flutter_skill_web_interop.dart' + as web_interop; + +// ==================== ERROR CODES ==================== + +class ErrorCode { + // Element errors + static const String elementNotFound = 'E001'; + static const String elementNotVisible = 'E002'; + static const String elementNotEnabled = 'E003'; + static const String multipleElements = 'E004'; + + // Action errors + static const String tapFailed = 'E101'; + static const String swipeFailed = 'E102'; + static const String inputFailed = 'E103'; + + // Connection errors + static const String appDisconnected = 'E201'; + static const String vmServiceError = 'E202'; + + // Timeout errors + static const String operationTimeout = 'E301'; + static const String elementWaitTimeout = 'E302'; +} /// The Binding that enables Flutter Skill automation. class FlutterSkillBinding { - static void ensureInitialized() { + static void ensureInitialized({bool autoEnableIndicators = true}) { if (_registered) return; _registered = true; _registerExtensions(); + + // On web, expose Dart-side element lookup to JavaScript + if (kIsWeb) { + web_interop.registerWebBridge(_handleWebBridgeCall); + } + + // Auto-enable test indicators after a short delay + if (autoEnableIndicators) { + Future.delayed(const Duration(milliseconds: 500), () { + try { + _indicatorsEnabled = true; + _indicatorOverlay = TestIndicatorOverlay(); + _indicatorOverlay!.enable(); + _indicatorOverlay!.setStyle(IndicatorStyle.detailed); + print('🎭 Test Indicators Auto-Enabled (detailed mode)'); + } catch (e) { + print('⚠️ Failed to auto-enable indicators: $e'); + } + }); + } + print('Flutter Skill Binding Initialized 🚀'); } static bool _registered = false; static final List _logs = []; static final List> _errors = []; + static final List> _httpRequests = []; static int _pointerCounter = 1; + // Element cache for semantic ref system + static Map>? _lastInspectResult; + + // Test indicators + static TestIndicatorOverlay? _indicatorOverlay; + static bool _indicatorsEnabled = false; + static Offset? + _lastCharacterPosition; // Track character position for walking effect + + /// Handle a bridge call from the web JS bridge. + /// Returns JSON string result. + static String _handleWebBridgeCall(String method, String paramsJson) { + try { + final params = jsonDecode(paramsJson) as Map; + switch (method) { + case 'find_element': + final key = params['key'] as String?; + final text = params['text'] as String?; + final element = _findElement(key: key, text: text); + if (element == null) return jsonEncode({'found': false}); + final renderObj = element.renderObject; + if (renderObj is RenderBox) { + final offset = renderObj.localToGlobal(Offset.zero); + final size = renderObj.size; + return jsonEncode({ + 'found': true, + 'element': { + 'type': element.widget.runtimeType.toString(), + 'key': key, + 'text': _extractTextFrom(element), + 'bounds': { + 'x': offset.dx.round(), + 'y': offset.dy.round(), + 'width': size.width.round(), + 'height': size.height.round(), + }, + }, + }); + } + return jsonEncode({ + 'found': true, + 'element': { + 'type': element.widget.runtimeType.toString(), + 'key': key + } + }); + + case 'get_text': + final key = params['key'] as String?; + final text = params['text'] as String?; + final element = _findElement(key: key, text: text); + if (element == null) return jsonEncode({'text': null}); + // For text fields, get the current value + if (key != null) { + final value = _getTextFieldValue(key); + if (value != null) return jsonEncode({'text': value}); + } + final extracted = _extractTextFrom(element); + return jsonEncode({'text': extracted}); + + case 'tap': + final key = params['key'] as String?; + final text = params['text'] as String?; + final element = _findElement(key: key, text: text); + if (element == null) + return jsonEncode( + {'success': false, 'message': 'Element not found'}); + final renderObj = element.renderObject; + if (renderObj is RenderBox) { + final center = + renderObj.localToGlobal(renderObj.size.center(Offset.zero)); + // Schedule the tap + WidgetsBinding.instance.addPostFrameCallback((_) { + _performTapAt(center.dx, center.dy); + }); + return jsonEncode({'success': true, 'message': 'Tap scheduled'}); + } + return jsonEncode({'success': false, 'message': 'No render object'}); + + case 'enter_text': + final key = params['key'] as String?; + final textVal = params['text'] as String? ?? ''; + final element = key != null ? _findElementByKey(key) : null; + if (element == null) + return jsonEncode( + {'success': false, 'message': 'Element not found'}); + final renderObj = element.renderObject; + if (renderObj is RenderBox) { + final center = + renderObj.localToGlobal(renderObj.size.center(Offset.zero)); + // Tap to focus, then enter text + WidgetsBinding.instance.addPostFrameCallback((_) async { + await _performTapAt(center.dx, center.dy); + await Future.delayed(const Duration(milliseconds: 100)); + // Use test text input to set the value + final controller = _findTextEditingController(element); + if (controller != null) { + controller.text = textVal; + } + }); + return jsonEncode( + {'success': true, 'message': 'Text entry scheduled'}); + } + return jsonEncode({'success': false, 'message': 'No render object'}); + + case 'wait_for_element': + final key = params['key'] as String?; + final text = params['text'] as String?; + final element = _findElement(key: key, text: text); + return jsonEncode({'found': element != null}); + + default: + return jsonEncode({'error': 'Unknown web bridge method: $method'}); + } + } catch (e) { + return jsonEncode({'error': e.toString()}); + } + } + + static TextEditingController? _findTextEditingController(Element element) { + TextEditingController? controller; + void visit(Element el) { + if (controller != null) return; + final widget = el.widget; + if (widget is EditableText) { + controller = widget.controller; + return; + } + el.visitChildren(visit); + } + + visit(element); + return controller; + } + static void _registerExtensions() { // ==================== EXISTING EXTENSIONS ==================== @@ -37,19 +227,43 @@ class FlutterSkillBinding { } }); + // 1b. Interactive Elements Structured (Enhanced inspect) + developer + .registerExtension('ext.flutter.flutter_skill.interactiveStructured', + (method, parameters) async { + try { + final elements = _findInteractiveElementsStructured(); + return developer.ServiceExtensionResponse.result( + jsonEncode({'type': 'Success', 'data': elements}), + ); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // 1c. Interactive Elements with Ref IDs (New inspect_interactive method) + developer.registerExtension('ext.flutter.flutter_skill.inspectInteractive', + (method, parameters) async { + try { + final elements = _findInteractiveElementsStructured(); + return developer.ServiceExtensionResponse.result( + jsonEncode({'type': 'Success', 'data': elements}), + ); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + // 2. Tap developer.registerExtension('ext.flutter.flutter_skill.tap', (method, parameters) async { try { final key = parameters['key']; final text = parameters['text']; - final success = await _performTap(key: key, text: text); - return developer.ServiceExtensionResponse.result( - jsonEncode({ - 'success': success, - 'message': success ? 'Tap successful' : 'Element not found' - }), - ); + final refId = parameters['ref']; + final result = + await _performTapWithDetails(key: key, text: text, refId: refId); + return developer.ServiceExtensionResponse.result(jsonEncode(result)); } catch (e, stack) { return _errorResponse(e, stack); } @@ -61,19 +275,19 @@ class FlutterSkillBinding { try { final key = parameters['key']; final text = parameters['text']; + final refId = parameters['ref']; if (text == null) { - return developer.ServiceExtensionResponse.error( - developer.ServiceExtensionResponse.invalidParams, - 'Missing text', - ); + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': { + 'code': ErrorCode.inputFailed, + 'message': 'Missing text parameter', + }, + })); } - final success = await _performEnterText(key: key, text: text); - return developer.ServiceExtensionResponse.result( - jsonEncode({ - 'success': success, - 'message': success ? 'Text entered' : 'TextField not found' - }), - ); + final result = await _performEnterTextWithDetails( + key: key, text: text, refId: refId); + return developer.ServiceExtensionResponse.result(jsonEncode(result)); } catch (e, stack) { return _errorResponse(e, stack); } @@ -85,11 +299,13 @@ class FlutterSkillBinding { try { final key = parameters['key']; final text = parameters['text']; - final success = await _performScroll(key: key, text: text); + final result = await _performScroll(key: key, text: text); return developer.ServiceExtensionResponse.result( jsonEncode({ - 'success': success, - 'message': success ? 'Scroll successful' : 'Element not found' + 'success': result['success'], + 'message': result['success'] == true + ? 'Scroll successful' + : (result['error'] ?? 'Scroll failed'), }), ); } catch (e, stack) { @@ -195,10 +411,10 @@ class FlutterSkillBinding { final success = await _performSwipe( direction: direction, distance: distance, key: key); return developer.ServiceExtensionResponse.result( - jsonEncode({ + jsonEncode(_withFrameStatus({ 'success': success, 'message': success ? 'Swipe successful' : 'Swipe failed' - }), + })), ); } catch (e, stack) { return _errorResponse(e, stack); @@ -211,12 +427,14 @@ class FlutterSkillBinding { try { final fromKey = parameters['fromKey']; final toKey = parameters['toKey']; - final success = await _performDrag(fromKey: fromKey, toKey: toKey); + final holdMs = int.tryParse(parameters['hold'] ?? '0') ?? 0; + final success = + await _performDrag(fromKey: fromKey, toKey: toKey, holdMs: holdMs); return developer.ServiceExtensionResponse.result( - jsonEncode({ + jsonEncode(_withFrameStatus({ 'success': success, 'message': success ? 'Drag successful' : 'Drag failed' - }), + })), ); } catch (e, stack) { return _errorResponse(e, stack); @@ -341,11 +559,40 @@ class FlutterSkillBinding { // ==================== SCREENSHOT ==================== - // 18. Screenshot + // 18. Screenshot (with quality and maxWidth support) developer.registerExtension('ext.flutter.flutter_skill.screenshot', (method, parameters) async { try { - final base64Image = await _takeScreenshot(); + final quality = double.tryParse(parameters['quality'] ?? '1.0') ?? 1.0; + final maxWidth = int.tryParse(parameters['maxWidth'] ?? ''); + final shot = + await _takeScreenshot(quality: quality, maxWidth: maxWidth); + // Logical size + DPR let callers map image pixels back to the + // logical coordinates that tap_at/swipe expect. + final view = WidgetsBinding.instance.platformDispatcher.views.first; + final logical = view.physicalSize / view.devicePixelRatio; + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'image': shot?.base64, + 'imageWidth': shot?.width, + 'imageHeight': shot?.height, + 'logicalWidth': logical.width, + 'logicalHeight': logical.height, + 'devicePixelRatio': view.devicePixelRatio, + })); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // 18b. Screenshot Region + developer.registerExtension('ext.flutter.flutter_skill.screenshotRegion', + (method, parameters) async { + try { + final x = double.tryParse(parameters['x'] ?? '0') ?? 0; + final y = double.tryParse(parameters['y'] ?? '0') ?? 0; + final width = double.tryParse(parameters['width'] ?? '100') ?? 100; + final height = double.tryParse(parameters['height'] ?? '100') ?? 100; + final base64Image = await _takeRegionScreenshot(x, y, width, height); return developer.ServiceExtensionResponse.result( jsonEncode({'image': base64Image})); } catch (e, stack) { @@ -380,7 +627,7 @@ class FlutterSkillBinding { try { final route = _getCurrentRoute(); return developer.ServiceExtensionResponse.result( - jsonEncode({'route': route})); + jsonEncode({'route': route, 'routes': _routeEntries()})); } catch (e, stack) { return _errorResponse(e, stack); } @@ -414,6 +661,285 @@ class FlutterSkillBinding { } }); + // ==================== TEST INDICATORS ==================== + + // Enable Test Indicators + developer.registerExtension('ext.flutter.flutter_skill.enableIndicators', + (method, parameters) async { + try { + _indicatorsEnabled = true; + _indicatorOverlay ??= TestIndicatorOverlay(); + _indicatorOverlay!.enable(); + + // Optional: set style + final styleParam = parameters['style']; + if (styleParam != null) { + final style = IndicatorStyle.values.firstWhere( + (s) => s.name == styleParam, + orElse: () => IndicatorStyle.standard, + ); + _indicatorOverlay!.setStyle(style); + } + + return developer.ServiceExtensionResponse.result( + jsonEncode({ + 'success': true, + 'enabled': true, + 'style': _indicatorOverlay!._style.name, + }), + ); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // Disable Test Indicators + developer.registerExtension('ext.flutter.flutter_skill.disableIndicators', + (method, parameters) async { + try { + _indicatorsEnabled = false; + _indicatorOverlay?.disable(); + _indicatorOverlay = null; + + return developer.ServiceExtensionResponse.result( + jsonEncode({ + 'success': true, + 'enabled': false, + }), + ); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // Get Indicator Status + developer.registerExtension('ext.flutter.flutter_skill.getIndicatorStatus', + (method, parameters) async { + try { + return developer.ServiceExtensionResponse.result( + jsonEncode({ + 'enabled': _indicatorsEnabled, + 'style': _indicatorOverlay?._style.name ?? 'standard', + }), + ); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // ==================== FOCUS / TYPING / TOGGLES / HOVER ==================== + + // type_text: insert into the focused field the way an IME commits text + // (replaces the current selection), so onChanged fires as for a user. + developer.registerExtension('ext.flutter.flutter_skill.typeText', + (method, parameters) async { + try { + final text = parameters['text'] ?? ''; + final field = _findFocusedTextField(); + if (field == null) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'No focused text field. Use focus(key) or tap it first.', + })); + } + final value = field.textEditingValue; + final sel = value.selection.isValid + ? value.selection + : TextSelection.collapsed(offset: value.text.length); + final newText = value.text.replaceRange(sel.start, sel.end, text); + field.updateEditingValue(TextEditingValue( + text: newText, + selection: TextSelection.collapsed(offset: sel.start + text.length), + )); + await _settle(); + return developer.ServiceExtensionResponse.result(jsonEncode( + {'success': true, 'message': 'Typed "$text"', 'value': newText})); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // focus / blur: drive the field's own FocusNode. + developer.registerExtension('ext.flutter.flutter_skill.focus', + (method, parameters) async { + try { + final key = parameters['key']; + final field = key == null ? null : _findEditableText(key); + if (field == null) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'No text field with key "$key"', + })); + } + field.widget.focusNode.requestFocus(); + await _settle(); + return developer.ServiceExtensionResponse.result( + jsonEncode({'success': true, 'message': 'Focused "$key"'})); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + developer.registerExtension('ext.flutter.flutter_skill.blur', + (method, parameters) async { + try { + final key = parameters['key']; + final field = + key == null ? _findFocusedTextField() : _findEditableText(key); + if (field == null) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'No focused text field to blur', + })); + } + field.widget.focusNode.unfocus(); + await _settle(); + return developer.ServiceExtensionResponse.result( + jsonEncode({'success': true, 'message': 'Blurred'})); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // set_checkbox: tap the toggle until it shows the requested state. + developer.registerExtension('ext.flutter.flutter_skill.setCheckbox', + (method, parameters) async { + try { + final key = parameters['key'] ?? ''; + final wanted = parameters['checked'] == 'true'; + final current = _getCheckboxState(key); + if (current == null) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'No Checkbox/Switch (or *ListTile) with key "$key"', + })); + } + if (current != wanted) { + await _performTapWithDetails(key: key); + } + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': _getCheckboxState(key) == wanted, + 'checked': _getCheckboxState(key), + })); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // hover: move a mouse pointer over the element (tooltips, hover styles). + developer.registerExtension('ext.flutter.flutter_skill.hover', + (method, parameters) async { + try { + final element = + _findElement(key: parameters['key'], text: parameters['text']); + final box = element?.renderObject; + if (element == null || box is! RenderBox || !box.hasSize) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'Element not found or not laid out', + })); + } + final position = box.localToGlobal(box.size.center(Offset.zero)); + final binding = WidgetsBinding.instance; + // One synthetic mouse device, added once: MouseTracker asserts on a + // repeated PointerAddedEvent for a device that was never removed. + if (!_syntheticMouseAdded) { + binding.handlePointerEvent(PointerAddedEvent( + position: position, + kind: ui.PointerDeviceKind.mouse, + device: _syntheticMouseDevice, + timeStamp: _pointerClock.elapsed)); + _syntheticMouseAdded = true; + } + binding.handlePointerEvent(PointerHoverEvent( + position: position, + kind: ui.PointerDeviceKind.mouse, + device: _syntheticMouseDevice, + timeStamp: _pointerClock.elapsed)); + await _settle(); + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': true, + 'position': {'x': position.dx.round(), 'y': position.dy.round()}, + })); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // ==================== PRESS KEY ==================== + + developer.registerExtension('ext.flutter.flutter_skill.pressKey', + (method, parameters) async { + try { + final key = parameters['key']; + if (key == null || key.isEmpty) { + return developer.ServiceExtensionResponse.result( + jsonEncode({'success': false, 'error': 'Missing key parameter'}), + ); + } + final modifiers = (parameters['modifiers'] ?? '') + .split(',') + .where((s) => s.isNotEmpty) + .toList(); + + final keyMap = { + 'enter': LogicalKeyboardKey.enter, + 'tab': LogicalKeyboardKey.tab, + 'escape': LogicalKeyboardKey.escape, + 'backspace': LogicalKeyboardKey.backspace, + 'delete': LogicalKeyboardKey.delete, + 'space': LogicalKeyboardKey.space, + 'up': LogicalKeyboardKey.arrowUp, + 'down': LogicalKeyboardKey.arrowDown, + 'left': LogicalKeyboardKey.arrowLeft, + 'right': LogicalKeyboardKey.arrowRight, + 'home': LogicalKeyboardKey.home, + 'end': LogicalKeyboardKey.end, + 'pageup': LogicalKeyboardKey.pageUp, + 'pagedown': LogicalKeyboardKey.pageDown, + }; + + final logicalKey = keyMap[key.toLowerCase()] ?? + LogicalKeyboardKey.findKeyByKeyId(key.codeUnitAt(0)); + + if (logicalKey == null) { + return developer.ServiceExtensionResponse.result( + jsonEncode({'success': false, 'error': 'Unknown key: $key'}), + ); + } + + // Build modifier key list (reserved for future modifier support) + // ignore: unused_local_variable + final isShift = modifiers.contains('shift'); + // ignore: unused_local_variable + final isCtrl = modifiers.contains('ctrl'); + // ignore: unused_local_variable + final isAlt = modifiers.contains('alt'); + // ignore: unused_local_variable + final isMeta = modifiers.contains('meta'); + + // Dispatch as a real hardware key: down (with modifiers held), up. + // Enter on a focused text field also performs the platform's "done" + // action, which is what fires onSubmitted on desktop. + final handled = await _pressHardwareKey(logicalKey, modifiers, + character: key.length == 1 ? key : null); + if (logicalKey == LogicalKeyboardKey.enter) { + _findFocusedTextField()?.performAction(TextInputAction.done); + } + await _settle(); + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': true, + 'message': 'Key pressed: $key', + 'handled': handled, + })); + // ignore: dead_code + return developer.ServiceExtensionResponse.result( + jsonEncode({'success': true, 'message': 'Key pressed: $key'}), + ); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + // ==================== DEBUG & LOGS ==================== // 23. Get Logs @@ -462,6 +988,154 @@ class FlutterSkillBinding { } }); + // ==================== COORDINATE-BASED ACTIONS ==================== + + // 27. Tap At Coordinates + developer.registerExtension('ext.flutter.flutter_skill.tapAt', + (method, parameters) async { + try { + final x = double.tryParse(parameters['x'] ?? '0') ?? 0; + final y = double.tryParse(parameters['y'] ?? '0') ?? 0; + await _performTapAt(x, y); + return developer.ServiceExtensionResponse.result(jsonEncode( + _withFrameStatus( + {'success': true, 'message': 'Tapped at ($x, $y)'}))); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // 28. Long Press At Coordinates + developer.registerExtension('ext.flutter.flutter_skill.longPressAt', + (method, parameters) async { + try { + final x = double.tryParse(parameters['x'] ?? '0') ?? 0; + final y = double.tryParse(parameters['y'] ?? '0') ?? 0; + final duration = int.tryParse(parameters['duration'] ?? '500') ?? 500; + await _performLongPressAt(x, y, duration: duration); + return developer.ServiceExtensionResponse.result( + jsonEncode({'success': true, 'message': 'Long pressed at ($x, $y)'}), + ); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // 29. Swipe Coordinates + developer.registerExtension('ext.flutter.flutter_skill.swipeCoordinates', + (method, parameters) async { + try { + final startX = double.tryParse(parameters['startX'] ?? '0') ?? 0; + final startY = double.tryParse(parameters['startY'] ?? '0') ?? 0; + final endX = double.tryParse(parameters['endX'] ?? '0') ?? 0; + final endY = double.tryParse(parameters['endY'] ?? '0') ?? 0; + final duration = int.tryParse(parameters['duration'] ?? '300') ?? 300; + final holdMs = int.tryParse(parameters['hold'] ?? '0') ?? 0; + await _performSwipeCoordinates(startX, startY, endX, endY, + duration: duration, holdMs: holdMs); + return developer.ServiceExtensionResponse.result( + jsonEncode(_withFrameStatus({ + 'success': true, + 'message': 'Swiped from ($startX, $startY) to ($endX, $endY)' + })), + ); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // 30. Edge Swipe (for drawer menus, back gestures) + developer.registerExtension('ext.flutter.flutter_skill.edgeSwipe', + (method, parameters) async { + try { + final edge = parameters['edge'] ?? 'left'; + final direction = parameters['direction'] ?? 'right'; + final distance = + double.tryParse(parameters['distance'] ?? '200') ?? 200; + final success = await _performEdgeSwipe( + edge: edge, direction: direction, distance: distance); + return developer.ServiceExtensionResponse.result( + jsonEncode({ + 'success': success, + 'message': success ? 'Edge swipe successful' : 'Edge swipe failed' + }), + ); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // ==================== PERFORMANCE & MEMORY ==================== + + // 30. Get Frame Stats + developer.registerExtension('ext.flutter.flutter_skill.getFrameStats', + (method, parameters) async { + try { + final stats = _getFrameStats(); + return developer.ServiceExtensionResponse.result(jsonEncode(stats)); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // 25. Log HTTP Request (manual API for apps to call) + developer.registerExtension('ext.flutter.flutter_skill.logHttpRequest', + (method, parameters) async { + try { + final entry = { + 'method': parameters['method'] ?? 'GET', + 'url': parameters['url'] ?? '', + 'status_code': int.tryParse(parameters['status_code'] ?? ''), + 'duration_ms': int.tryParse(parameters['duration_ms'] ?? ''), + 'response_body': parameters['response_body'], + 'error': parameters['error'], + 'timestamp': DateTime.now().toIso8601String(), + }; + // Remove null values + entry.removeWhere((_, v) => v == null); + _httpRequests.add(entry); + if (_httpRequests.length > 500) { + _httpRequests.removeAt(0); + } + return developer.ServiceExtensionResponse.result( + jsonEncode({'logged': true, 'total': _httpRequests.length})); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // 26. Get HTTP Requests (manually logged) + developer.registerExtension('ext.flutter.flutter_skill.getHttpRequests', + (method, parameters) async { + try { + final limit = int.tryParse(parameters['limit'] ?? '50') ?? 50; + final offset = int.tryParse(parameters['offset'] ?? '0') ?? 0; + final paged = _httpRequests.skip(offset).take(limit).toList(); + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'requests': paged, + 'total': _httpRequests.length, + 'returned': paged.length, + 'offset': offset, + 'limit': limit, + })); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // 27. Clear HTTP Requests + developer.registerExtension('ext.flutter.flutter_skill.clearHttpRequests', + (method, parameters) async { + try { + final count = _httpRequests.length; + _httpRequests.clear(); + return developer.ServiceExtensionResponse.result( + jsonEncode({'cleared': count})); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + // Setup error handler FlutterError.onError = (FlutterErrorDetails details) { _errors.add({ @@ -547,55 +1221,242 @@ class FlutterSkillBinding { return null; } - // ==================== ACTIONS ==================== - - static Future _performTap({String? key, String? text}) async { - final element = _findElement(key: key, text: text); - if (element == null) { - _log('Element not found for tap (key: $key, text: $text)'); - return false; + /// Find element by ref ID using caching and semantic ref system + static Element? _findElementByRefId(String refId) { + // Check if this is a legacy ref format (btn_0, tf_1, etc.) + if (SemanticRefGenerator.isLegacyRef(refId)) { + return _findElementByLegacyRef(refId); } - final renderObject = element.renderObject; - if (renderObject is! RenderBox || !renderObject.hasSize) { - _log('RenderObject is not a valid RenderBox'); - return false; + // First, try cached element + final cachedElement = SemanticRefGenerator.getCachedElement(refId); + if (cachedElement != null) { + final weakRef = cachedElement['element'] as WeakReference?; + final element = weakRef?.target; + if (element != null) { + return element; + } } - final center = - renderObject.localToGlobal(renderObject.size.center(Offset.zero)); - _log('Tapping at $center (key: $key, text: $text)'); + // Check last inspect result cache + if (_lastInspectResult != null && _lastInspectResult!.containsKey(refId)) { + final targetElementData = _lastInspectResult![refId]!; + + // Extract bounds to find the element at center position + final bounds = targetElementData['bounds'] as Map?; + if (bounds != null) { + final x = bounds['x'] as int? ?? 0; + final y = bounds['y'] as int? ?? 0; + final w = bounds['w'] as int? ?? 0; + final h = bounds['h'] as int? ?? 0; + + if (w > 0 && h > 0) { + final centerX = x + w / 2; + final centerY = y + h / 2; + return _findElementAtPosition(Offset(centerX, centerY)); + } + } - await _dispatchTap(center); - _log('Tap completed on (key: $key, text: $text)'); - return true; + // Fallback: try to find by text, label, or key if available + final text = targetElementData['text'] as String?; + final label = targetElementData['label'] as String?; + final key = targetElementData['key'] as String?; + + if (key != null) { + final found = _findElementByKey(key); + if (found != null) return found; + } + + if (text != null) { + final found = _findElementByText(text); + if (found != null) return found; + } + + if (label != null) { + final found = _findElementByText(label); + if (found != null) return found; + } + } + + // Handle elem_NNN numeric IDs from get_interactable_elements + if (refId.startsWith('elem_')) { + final freshElements = _findInteractiveElements(); + final match = freshElements.firstWhere( + (e) => e['id'] == refId, + orElse: () => {}, + ); + if (match.isNotEmpty) { + final bounds = match['bounds'] as Map?; + if (bounds != null) { + final x = (bounds['x'] as num?)?.toDouble() ?? 0; + final y = (bounds['y'] as num?)?.toDouble() ?? 0; + final w = (bounds['width'] as num?)?.toDouble() ?? 0; + final h = (bounds['height'] as num?)?.toDouble() ?? 0; + if (w > 0 && h > 0) { + return _findElementAtPosition(Offset(x + w / 2, y + h / 2)); + } + } + } + return null; + } + + // Cache miss - do fresh inspect to rebuild cache + final structured = _findInteractiveElementsStructured(); + final elements = structured['elements'] as List? ?? []; + + // Find the element with matching ref ID + Map? targetElementData; + for (final elementData in elements) { + if (elementData is Map && elementData['ref'] == refId) { + targetElementData = elementData; + break; + } + } + + if (targetElementData == null) return null; + + // Extract bounds to find the element at center position + final bounds = targetElementData['bounds'] as Map?; + if (bounds != null) { + final x = bounds['x'] as int? ?? 0; + final y = bounds['y'] as int? ?? 0; + final w = bounds['w'] as int? ?? 0; + final h = bounds['h'] as int? ?? 0; + + if (w > 0 && h > 0) { + final centerX = x + w / 2; + final centerY = y + h / 2; + return _findElementAtPosition(Offset(centerX, centerY)); + } + } + + return null; } - static Future _dispatchTap(Offset position) async { - final binding = WidgetsBinding.instance; - final pointer = _pointerCounter++; + /// Handle legacy ref format for backward compatibility + static Element? _findElementByLegacyRef(String refId) { + final legacyData = SemanticRefGenerator.parseLegacyRef(refId); + if (legacyData == null) return null; - binding.handlePointerEvent( - PointerDownEvent(position: position, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 50)); - binding.handlePointerEvent( - PointerUpEvent(position: position, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + // Re-run inspect to get current elements + final structured = _findInteractiveElementsStructured(); + final elements = structured['elements'] as List? ?? []; + + final role = legacyData['role'] as String?; + final index = legacyData['index'] as int? ?? 0; + + if (role == null) return null; + + // Find elements of matching semantic role and get by index + final matchingElements = >[]; + for (final elementData in elements) { + if (elementData is Map) { + final refId = elementData['ref'] as String?; + if (refId != null && refId.startsWith('$role:')) { + matchingElements.add(elementData); + } + } + } + + if (matchingElements.isEmpty || index >= matchingElements.length) { + return null; + } + + // Get element at legacy index + final targetElementData = matchingElements[index]; + + // Find by ref ID using the new system + final newRefId = targetElementData['ref'] as String?; + if (newRefId != null) { + return _findElementByRefId(newRefId); + } + + return null; } - static Future _performEnterText( - {String? key, required String text}) async { - final element = _findElement(key: key); - if (element == null) { - _log('TextField not found (key: $key)'); - return false; + /// Find element at a specific position by traversing the widget tree + static Element? _findElementAtPosition(Offset position) { + Element? found; + + void visit(Element element) { + if (found != null) return; + + final renderObject = element.renderObject; + if (renderObject is RenderBox && renderObject.hasSize) { + final offset = renderObject.localToGlobal(Offset.zero); + final size = renderObject.size; + + final bounds = + Rect.fromLTWH(offset.dx, offset.dy, size.width, size.height); + if (bounds.contains(position)) { + // Check if this is an interactive element + final widget = element.widget; + if (_isInteractiveWidget(widget)) { + found = element; + return; + } + } + } + + element.visitChildren(visit); + } + + final binding = WidgetsBinding.instance; + // ignore: invalid_use_of_protected_member + if (binding.rootElement != null) { + visit(binding.rootElement!); } + return found; + } + + /// Check if a widget is interactive + static bool _isInteractiveWidget(Widget widget) { + return widget is ElevatedButton || + widget is TextButton || + widget is OutlinedButton || + widget is IconButton || + widget is FloatingActionButton || + widget is TextField || + widget is TextFormField || + widget is Checkbox || + widget is Switch || + widget is Slider || + widget is DropdownButton || + (widget is InkWell && widget.onTap != null) || + (widget is GestureDetector && widget.onTap != null) || + widget is ListTile; + } + /// Helper method to enter text into a specific element + static Future> _enterTextIntoElement( + Element element, String text, + {required String method}) async { final renderObject = element.renderObject; - if (renderObject is! RenderBox) return false; + if (renderObject is! RenderBox) { + return { + 'success': false, + 'error': { + 'code': ErrorCode.elementNotVisible, + 'message': 'TextField has no valid render box', + }, + 'method': method, + }; + } final center = renderObject.localToGlobal(renderObject.size.center(Offset.zero)); + + // Show indicator if enabled + if (_indicatorsEnabled && _indicatorOverlay != null) { + final bounds = Rect.fromLTWH( + renderObject.localToGlobal(Offset.zero).dx, + renderObject.localToGlobal(Offset.zero).dy, + renderObject.size.width, + renderObject.size.height, + ); + _indicatorOverlay!.showTextInput(bounds, hint: "Entering text: '$text'"); + } + await _dispatchTap(center); await Future.delayed(const Duration(milliseconds: 200)); @@ -609,15 +1470,24 @@ class FlutterSkillBinding { e.visitChildren(findEditable); } - findEditable(element); + try { + findEditable(element); + } catch (_) { + // Element tree may be partially unmounted + } if (editableTextState != null) { editableTextState!.updateEditingValue(TextEditingValue( text: text, selection: TextSelection.collapsed(offset: text.length), )); - _log('Entered text "$text" (key: $key)'); - return true; + _log('Entered text "$text" using $method'); + return { + 'success': true, + 'message': 'Text entered successfully', + 'method': method, + 'enteredText': text, + }; } SystemChannels.textInput.invokeMethod('TextInput.setEditingState', { @@ -628,328 +1498,558 @@ class FlutterSkillBinding { 'composingExtent': -1, }); _log('Text input sent via channel'); - return true; + + return { + 'success': true, + 'message': 'Text entered via system channel', + 'method': method, + 'enteredText': text, + }; } - static Future _performScroll({String? key, String? text}) async { + // ==================== ACTIONS ==================== + + // ignore: unused_element + static Future _performTap({String? key, String? text}) async { final element = _findElement(key: key, text: text); if (element == null) { - _log('Element not found for scroll (key: $key, text: $text)'); + _log('Element not found for tap (key: $key, text: $text)'); return false; } - try { - await Scrollable.ensureVisible(element, - duration: const Duration(milliseconds: 300), curve: Curves.easeInOut); - _log('Scrolled to element (key: $key, text: $text)'); - return true; - } catch (e) { - _log('Scroll failed: $e'); + final renderObject = element.renderObject; + if (renderObject is! RenderBox || !renderObject.hasSize) { + _log('RenderObject is not a valid RenderBox'); return false; } - } - - static Future _performLongPress( - {String? key, String? text, int duration = 500}) async { - final element = _findElement(key: key, text: text); - if (element == null) return false; - - final renderObject = element.renderObject; - if (renderObject is! RenderBox || !renderObject.hasSize) return false; final center = renderObject.localToGlobal(renderObject.size.center(Offset.zero)); - final binding = WidgetsBinding.instance; - final pointer = _pointerCounter++; - - binding.handlePointerEvent( - PointerDownEvent(position: center, pointer: pointer)); - await Future.delayed(Duration(milliseconds: duration)); - binding - .handlePointerEvent(PointerUpEvent(position: center, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + _log('Tapping at $center (key: $key, text: $text)'); - _log('Long press completed (key: $key, text: $text)'); + await _dispatchTap(center); + _log('Tap completed on (key: $key, text: $text)'); return true; } - static Future _performSwipe( - {required String direction, double distance = 300, String? key}) async { - final binding = WidgetsBinding.instance; - Offset start; + /// Enhanced tap with detailed error information and suggestions + static Future> _performTapWithDetails( + {String? key, String? text, String? refId}) async { + Element? element; - if (key != null) { - final element = _findElementByKey(key); - if (element == null) return false; - final renderObject = element.renderObject; - if (renderObject is! RenderBox) return false; - start = renderObject.localToGlobal(renderObject.size.center(Offset.zero)); - } else { - // Use window size for global swipe - final window = binding.platformDispatcher.views.first; - final size = window.physicalSize / window.devicePixelRatio; - start = Offset(size.width / 2, size.height / 2); + // If refId is provided, find element by ref ID first + if (refId != null) { + element = _findElementByRefId(refId); } - Offset delta; - switch (direction.toLowerCase()) { - case 'up': - delta = Offset(0, -distance); - break; - case 'down': - delta = Offset(0, distance); - break; - case 'left': - delta = Offset(-distance, 0); - break; - case 'right': - delta = Offset(distance, 0); - break; - default: - return false; + // Fallback to key/text search if ref not found or not provided + if (element == null) { + element = _findElement(key: key, text: text); } - final pointer = _pointerCounter++; - final end = start + delta; + if (element == null) { + final suggestions = []; + + // Find similar keys if key was provided + if (key != null) { + final similarKeys = _findSimilarKeys(key); + if (similarKeys.isNotEmpty) { + suggestions + .add('Similar keys found: ${similarKeys.take(5).toList()}'); + } + } - binding.handlePointerEvent( - PointerDownEvent(position: start, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 16)); + // Find similar text if text was provided + if (text != null) { + final similarTexts = _findSimilarTexts(text); + if (similarTexts.isNotEmpty) { + suggestions + .add('Similar texts found: ${similarTexts.take(5).toList()}'); + } + } - const steps = 10; - for (int i = 1; i <= steps; i++) { - final current = Offset.lerp(start, end, i / steps)!; - binding.handlePointerEvent(PointerMoveEvent( - position: current, - pointer: pointer, - delta: delta / steps.toDouble())); - await Future.delayed(const Duration(milliseconds: 16)); + suggestions.addAll([ + 'Use inspect() to see available interactive elements', + 'Use get_widget_tree() to explore the widget hierarchy', + ]); + + _log('Element not found for tap (key: $key, text: $text, ref: $refId)'); + final target = refId != null + ? "ref '$refId'" + : key != null + ? "key '$key'" + : "text '$text'"; + return { + 'success': false, + 'error': { + 'code': ErrorCode.elementNotFound, + 'message': 'No element matching $target found in widget tree', + }, + 'target': {'key': key, 'text': text, if (refId != null) 'ref': refId}, + 'suggestions': suggestions, + }; } - binding.handlePointerEvent(PointerUpEvent(position: end, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + final renderObject = element.renderObject; + if (renderObject is! RenderBox || !renderObject.hasSize) { + return { + 'success': false, + 'error': { + 'code': ErrorCode.elementNotVisible, + 'message': 'Element found but has no valid render box', + }, + 'target': {'key': key, 'text': text}, + 'suggestions': [ + 'Element may be offscreen or not yet laid out', + 'Try waiting for the element to be visible' + ], + }; + } - _log('Swipe $direction completed'); - return true; - } + final center = + renderObject.localToGlobal(renderObject.size.center(Offset.zero)); - static Future _performDrag({String? fromKey, String? toKey}) async { - if (fromKey == null || toKey == null) return false; + // Reject off-screen coordinates — element may be from a non-active route + final view = WidgetsBinding.instance.platformDispatcher.views.first; + final sw = view.physicalSize.width / view.devicePixelRatio; + final sh = view.physicalSize.height / view.devicePixelRatio; + if (center.dx < -50 || + center.dx > sw + 50 || + center.dy < -50 || + center.dy > sh + 50) { + return { + 'success': false, + 'error': { + 'code': ErrorCode.elementNotVisible, + 'message': 'Element "${text ?? key ?? refId}" is off-screen ' + '(coords: ${center.dx.round()}, ${center.dy.round()}). ' + 'It may belong to a different route. Navigate to the correct page first.', + }, + 'target': {'key': key, 'text': text}, + 'position': {'x': center.dx.round(), 'y': center.dy.round()}, + }; + } - final fromElement = _findElementByKey(fromKey); - final toElement = _findElementByKey(toKey); - if (fromElement == null || toElement == null) return false; + _log('Tapping at $center (key: $key, text: $text)'); - final fromRender = fromElement.renderObject; - final toRender = toElement.renderObject; - if (fromRender is! RenderBox || toRender is! RenderBox) return false; + // Show indicator if enabled + if (_indicatorsEnabled && _indicatorOverlay != null) { + final targetText = text ?? _extractTextFrom(element) ?? key ?? 'element'; + _indicatorOverlay!.showTap(center, hint: "Tapping '$targetText'"); + } - final start = fromRender.localToGlobal(fromRender.size.center(Offset.zero)); - final end = toRender.localToGlobal(toRender.size.center(Offset.zero)); + // Exactly one activation: drive the tap through the pointer pipeline + // when the element is really hittable at its center; otherwise (covered + // by an overlay, pointer ignored) invoke the widget's callback directly. + if (_isHittableAt(element, center)) { + await _dispatchTap(center); + } else { + _tryInvokeCallback(element); + await _settle(); + } - final binding = WidgetsBinding.instance; - final pointer = _pointerCounter++; + _log('Tap completed on (key: $key, text: $text)'); - binding.handlePointerEvent( - PointerDownEvent(position: start, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + return _withFrameStatus({ + 'success': true, + 'message': 'Tap successful', + 'target': {'key': key, 'text': text}, + 'position': {'x': center.dx.round(), 'y': center.dy.round()}, + }); + } - const steps = 20; - for (int i = 1; i <= steps; i++) { - final current = Offset.lerp(start, end, i / steps)!; - binding.handlePointerEvent( - PointerMoveEvent(position: current, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 16)); + /// Directly invoke the onPressed/onTap callback of a widget — used only + /// when the element is not reachable by a pointer (see [_isHittableAt]). + static void _tryInvokeCallback(Element element) { + if (!element.mounted) return; + final widget = element.widget; + + if (widget is ElevatedButton && widget.onPressed != null) { + widget.onPressed!(); + } else if (widget is TextButton && widget.onPressed != null) { + widget.onPressed!(); + } else if (widget is OutlinedButton && widget.onPressed != null) { + widget.onPressed!(); + } else if (widget is IconButton && widget.onPressed != null) { + widget.onPressed!(); + } else if (widget is FloatingActionButton && widget.onPressed != null) { + widget.onPressed!(); + } else if (widget is InkWell && widget.onTap != null) { + widget.onTap!(); + } else if (widget is GestureDetector && widget.onTap != null) { + widget.onTap!(); } + // For other widget types, the pointer dispatch should handle it + } - binding.handlePointerEvent(PointerUpEvent(position: end, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + /// Get all keys in the widget tree + static List _getAllKeys() { + final keys = []; - _log('Drag from $fromKey to $toKey completed'); - return true; + void visit(Element element) { + final widget = element.widget; + if (widget.key is ValueKey) { + keys.add((widget.key as ValueKey).value); + } + element.visitChildren(visit); + } + + final binding = WidgetsBinding.instance; + // ignore: invalid_use_of_protected_member + if (binding.rootElement != null) { + visit(binding.rootElement!); + } + return keys; } - static Future _performDoubleTap({String? key, String? text}) async { - final element = _findElement(key: key, text: text); - if (element == null) return false; + /// Find keys similar to the given key using simple substring matching + static List _findSimilarKeys(String targetKey) { + final allKeys = _getAllKeys(); + final targetLower = targetKey.toLowerCase(); - final renderObject = element.renderObject; - if (renderObject is! RenderBox || !renderObject.hasSize) return false; + // Score and sort by similarity + final scored = >[]; + for (final key in allKeys) { + final keyLower = key.toLowerCase(); + int score = 0; - final center = - renderObject.localToGlobal(renderObject.size.center(Offset.zero)); + // Exact substring match + if (keyLower.contains(targetLower) || targetLower.contains(keyLower)) { + score += 100; + } - await _dispatchTap(center); - await Future.delayed(const Duration(milliseconds: 50)); - await _dispatchTap(center); + // Common prefix + int prefixLen = 0; + for (int i = 0; i < targetLower.length && i < keyLower.length; i++) { + if (targetLower[i] == keyLower[i]) + prefixLen++; + else + break; + } + score += prefixLen * 10; - _log('Double tap completed (key: $key, text: $text)'); - return true; - } + // Character overlap + for (final char in targetLower.split('')) { + if (keyLower.contains(char)) score += 1; + } - // ==================== UI INSPECTION ==================== + if (score > 5) { + scored.add(MapEntry(key, score)); + } + } - static List> _findInteractiveElements() { - final results = >[]; + scored.sort((a, b) => b.value.compareTo(a.value)); + return scored.map((e) => e.key).toList(); + } - void visit(Element element) { - final widget = element.widget; - String? type; - String? text; - String? key; + /// Find texts similar to the given text + static List _findSimilarTexts(String targetText) { + final allTexts = _getTextContent() + .map((t) => t['text'] as String?) + .whereType() + .toList(); + final targetLower = targetText.toLowerCase(); + + final scored = >[]; + for (final text in allTexts) { + final textLower = text.toLowerCase(); + int score = 0; + + if (textLower.contains(targetLower) || targetLower.contains(textLower)) { + score += 100; + } - if (widget.key is ValueKey) { - key = (widget.key as ValueKey).value; + // Common words + final targetWords = targetLower.split(RegExp(r'\s+')); + final textWords = textLower.split(RegExp(r'\s+')); + for (final word in targetWords) { + if (textWords.contains(word)) score += 20; } - if (widget is ElevatedButton || - widget is TextButton || - widget is OutlinedButton || - widget is IconButton || - widget is FloatingActionButton) { - type = 'Button'; - text = _extractTextFrom(element); - } else if (widget is TextField || widget is TextFormField) { - type = 'TextField'; - } else if (widget is Checkbox) { - type = 'Checkbox'; - } else if (widget is Switch) { - type = 'Switch'; - } else if (widget is Slider) { - type = 'Slider'; - } else if (widget is DropdownButton) { - type = 'Dropdown'; - } else if (widget is InkWell && widget.onTap != null) { - type = 'Tappable'; - } else if (widget is GestureDetector && widget.onTap != null) { - type = 'Tappable'; + if (score > 10 && text != targetText) { + scored.add(MapEntry(text, score)); } + } - if (type != null) { - results.add({ - if (key != null) 'key': key, - if (text != null) 'text': text, - 'type': type, - }); - } + scored.sort((a, b) => b.value.compareTo(a.value)); + return scored.map((e) => e.key).toList(); + } - element.visitChildren(visit); + // ==================== SYNTHETIC KEYBOARD ==================== + + /// USB HID usage codes for the physical keys we can synthesize; the + /// framework requires physical/logical pairs to be consistent. + static const Map _physicalUsageByLogicalId = { + 0x100000008: 0x0007002a, // backspace + 0x100000009: 0x0007002b, // tab + 0x10000000d: 0x00070028, // enter + 0x10000001b: 0x00070029, // escape + 0x10000007f: 0x0007004c, // delete + 0x100000301: 0x00070051, // arrowDown + 0x100000302: 0x00070050, // arrowLeft + 0x100000303: 0x0007004f, // arrowRight + 0x100000304: 0x00070052, // arrowUp + 0x100000305: 0x0007004d, // end + 0x100000306: 0x0007004a, // home + 0x100000307: 0x0007004e, // pageDown + 0x100000308: 0x0007004b, // pageUp + 0x20: 0x0007002c, // space + }; + + /// Device id of the synthetic mouse used by `hover` (added on first use). + static const int _syntheticMouseDevice = 0x5ca1; + static bool _syntheticMouseAdded = false; + + static PhysicalKeyboardKey _physicalFor(LogicalKeyboardKey logical) { + final id = logical.keyId; + final mapped = _physicalUsageByLogicalId[id]; + if (mapped != null) return PhysicalKeyboardKey(mapped); + // Unicode letters/digits: keyA..keyZ = 0x70004.., digit1..9,0 = 0x7001e.. + if (id >= 0x61 && id <= 0x7a) + return PhysicalKeyboardKey(0x00070004 + id - 0x61); + if (id >= 0x31 && id <= 0x39) + return PhysicalKeyboardKey(0x0007001e + id - 0x31); + if (id == 0x30) return const PhysicalKeyboardKey(0x00070027); + return PhysicalKeyboardKey.findKeyByCode(id) ?? PhysicalKeyboardKey.enter; + } + + static const Map + _modifierKeys = { + 'shift': (LogicalKeyboardKey.shiftLeft, PhysicalKeyboardKey.shiftLeft), + 'ctrl': (LogicalKeyboardKey.controlLeft, PhysicalKeyboardKey.controlLeft), + 'control': ( + LogicalKeyboardKey.controlLeft, + PhysicalKeyboardKey.controlLeft + ), + 'alt': (LogicalKeyboardKey.altLeft, PhysicalKeyboardKey.altLeft), + 'meta': (LogicalKeyboardKey.metaLeft, PhysicalKeyboardKey.metaLeft), + }; + + /// Delivers one synthetic [KeyEvent] the way the engine's key path does + /// once a platform message is decoded: [HardwareKeyboard] records the + /// pressed state, then the [KeyMessage] goes to the global handler + /// (FocusManager → Shortcuts/Actions → text-editing shortcuts). + /// Returns whether a handler claimed it. + static bool _deliverKey(KeyEvent event) { + HardwareKeyboard.instance.handleKeyEvent(event); + final handler = ServicesBinding.instance.keyEventManager.keyMessageHandler; + return handler?.call(KeyMessage([event], null)) ?? false; + } + + /// Sends a physical key press (down, up) with [modifiers] held around it. + static Future _pressHardwareKey( + LogicalKeyboardKey logical, List modifiers, + {String? character}) async { + Duration now() => _pointerClock.elapsed; + final held = <(LogicalKeyboardKey, PhysicalKeyboardKey)>[]; + for (final m in modifiers) { + final pair = _modifierKeys[m.toLowerCase()]; + if (pair == null) continue; + _deliverKey(KeyDownEvent( + physicalKey: pair.$2, logicalKey: pair.$1, timeStamp: now())); + held.add(pair); } + final physical = _physicalFor(logical); + // Platforms report no character while a modifier is held (Ctrl+A is a + // shortcut, not text input). + final handled = _deliverKey(KeyDownEvent( + physicalKey: physical, + logicalKey: logical, + character: held.isEmpty ? character : null, + timeStamp: now(), + )); + await Future.delayed(const Duration(milliseconds: 40)); + _deliverKey(KeyUpEvent( + physicalKey: physical, logicalKey: logical, timeStamp: now())); + for (final pair in held.reversed) { + _deliverKey(KeyUpEvent( + physicalKey: pair.$2, logicalKey: pair.$1, timeStamp: now())); + } + return handled; + } + // ==================== SYNTHETIC POINTER ENGINE ==================== + // + // Every emulated input goes through [_emitGesture]. Flutter's gesture + // recognizers and scroll physics derive velocities from event + // timestamps, so each event carries a monotonically increasing + // [PointerEvent.timeStamp]; the pointer is always released, even if a + // handler throws; and the call returns only after the framework has + // rendered the frame that reflects the input, so a state query issued + // right afterwards observes the result. + + static final Stopwatch _pointerClock = Stopwatch()..start(); + + static Future _emitGesture({ + required Offset from, + Offset? to, + Duration hold = Duration.zero, + Duration travel = const Duration(milliseconds: 300), + }) async { final binding = WidgetsBinding.instance; - // ignore: invalid_use_of_protected_member - if (binding.rootElement != null) { - visit(binding.rootElement!); + final pointer = _pointerCounter++; + final end = to ?? from; + Duration now() => _pointerClock.elapsed; + + binding.handlePointerEvent( + PointerDownEvent(position: from, pointer: pointer, timeStamp: now())); + try { + if (hold > Duration.zero) await Future.delayed(hold); + if (end != from) { + // About one move per frame; enough steps to clear the touch slop + // early and give the velocity tracker a smooth sample history. + final steps = (travel.inMilliseconds / 16).round().clamp(4, 60); + final stepDelay = travel ~/ steps; + var last = from; + for (var i = 1; i <= steps; i++) { + await Future.delayed(stepDelay); + final position = Offset.lerp(from, end, i / steps)!; + binding.handlePointerEvent(PointerMoveEvent( + position: position, + delta: position - last, + pointer: pointer, + timeStamp: now(), + )); + last = position; + } + } + } finally { + binding.handlePointerEvent( + PointerUpEvent(position: end, pointer: pointer, timeStamp: now())); } + await _settle(); + } - return results; + /// Completes once two frames have rendered: the one that runs the input's + /// callbacks and the one that paints the resulting state. Each wait is + /// bounded: a minimized/occluded window produces no frames on some + /// platforms (Windows), and an action must never hang on that. + static Future _settle() async { + _framesStalled = !await _nextFrame() && !await _nextFrame(); } - static Map _getWidgetTree({int maxDepth = 10}) { - Map buildNode(Element element, int depth) { - final widget = element.widget; - final node = { - 'type': widget.runtimeType.toString(), - }; + /// True when the last action saw no frame at all — the app is most likely + /// minimized or hidden, so nothing it did is observable yet. + static bool _framesStalled = false; - if (widget.key is ValueKey) { - node['key'] = (widget.key as ValueKey).value; - } + static const _stalledWarning = + 'No frame was rendered after this action; the app window is probably ' + 'minimized or hidden. Restore it before reading state or taking a ' + 'screenshot.'; - if (widget is Text) { - node['text'] = widget.data; - } + /// Adds the stalled-frames warning to an action response when relevant. + static Map _withFrameStatus(Map response) => + _framesStalled ? {...response, 'warning': _stalledWarning} : response; - final renderObject = element.renderObject; - if (renderObject is RenderBox && renderObject.hasSize) { - node['size'] = { - 'width': renderObject.size.width, - 'height': renderObject.size.height - }; - final offset = renderObject.localToGlobal(Offset.zero); - node['position'] = {'x': offset.dx, 'y': offset.dy}; - } + static Future _nextFrame() => SchedulerBinding.instance.endOfFrame + .then((_) => true) + .timeout(const Duration(milliseconds: 500), onTimeout: () => false); - if (depth < maxDepth) { - final children = >[]; - element.visitChildren((child) { - children.add(buildNode(child, depth + 1)); - }); - if (children.isNotEmpty) { - node['children'] = children; - } + /// Whether a pointer at [position] would reach [element] (or a descendant), + /// i.e. nothing opaque covers it and it takes part in hit testing. + static bool _isHittableAt(Element element, Offset position) { + final target = element.renderObject; + if (target == null) return false; + final binding = WidgetsBinding.instance; + final result = HitTestResult(); + binding.hitTestInView( + result, position, binding.platformDispatcher.views.first.viewId); + for (final entry in result.path) { + RenderObject? node = + entry.target is RenderObject ? entry.target as RenderObject : null; + while (node != null) { + if (identical(node, target)) return true; + node = node.parent; } + } + return false; + } - return node; + static Future _dispatchTap(Offset position) async { + // Show animated character walking to position and tapping + if (_indicatorsEnabled && _indicatorOverlay != null) { + // Use last position as starting point for walking animation + final startPos = _lastCharacterPosition ?? position; + _indicatorOverlay!.showCharacter( + startPos, + CharacterAction.tapping, + hint: "Tapping", + endPosition: position, // Character walks from startPos to position + ); + _lastCharacterPosition = position; // Remember position for next action + await Future.delayed(const Duration(milliseconds: 200)); } - final binding = WidgetsBinding.instance; - // ignore: invalid_use_of_protected_member - if (binding.rootElement != null) { - return buildNode(binding.rootElement!, 0); + await _emitGesture(from: position, hold: const Duration(milliseconds: 50)); + + // Show tap ripple after character + if (_indicatorsEnabled && _indicatorOverlay != null) { + _indicatorOverlay!.showTap(position); } - return {}; } - static Map? _getWidgetProperties(String key) { - final element = _findElementByKey(key); - if (element == null) return null; - - final widget = element.widget; - final props = { - 'type': widget.runtimeType.toString(), - 'key': key, - }; + // ignore: unused_element + static Future _performEnterText( + {String? key, required String text}) async { + final element = _findElement(key: key); + if (element == null) { + _log('TextField not found (key: $key)'); + return false; + } final renderObject = element.renderObject; - if (renderObject is RenderBox && renderObject.hasSize) { - props['size'] = { - 'width': renderObject.size.width, - 'height': renderObject.size.height - }; - final offset = renderObject.localToGlobal(Offset.zero); - props['position'] = {'x': offset.dx, 'y': offset.dy}; - props['visible'] = renderObject.attached && - renderObject.size.width > 0 && - renderObject.size.height > 0; + if (renderObject is! RenderBox) return false; + + final center = + renderObject.localToGlobal(renderObject.size.center(Offset.zero)); + await _dispatchTap(center); + await Future.delayed(const Duration(milliseconds: 200)); + + EditableTextState? editableTextState; + void findEditable(Element e) { + if (editableTextState != null) return; + if (e is StatefulElement && e.state is EditableTextState) { + editableTextState = e.state as EditableTextState; + return; + } + e.visitChildren(findEditable); } - if (widget is Text) { - props['text'] = widget.data; - props['style'] = widget.style?.toString(); - } else if (widget is Container) { - props['color'] = widget.color?.toString(); - props['padding'] = widget.padding?.toString(); - props['margin'] = widget.margin?.toString(); + findEditable(element); + + if (editableTextState != null) { + editableTextState!.updateEditingValue(TextEditingValue( + text: text, + selection: TextSelection.collapsed(offset: text.length), + )); + _log('Entered text "$text" (key: $key)'); + return true; } - return props; + SystemChannels.textInput.invokeMethod('TextInput.setEditingState', { + 'text': text, + 'selectionBase': text.length, + 'selectionExtent': text.length, + 'composingBase': -1, + 'composingExtent': -1, + }); + _log('Text input sent via channel'); + return true; } - static List> _getTextContent() { - final results = >[]; + /// Find the currently focused TextField's EditableTextState + static EditableTextState? _findFocusedTextField() { + EditableTextState? focused; void visit(Element element) { - final widget = element.widget; - String? key; - if (widget.key is ValueKey) { - key = (widget.key as ValueKey).value; - } - - if (widget is Text && widget.data != null) { - results.add({ - 'text': widget.data, - if (key != null) 'key': key, - 'type': 'Text', - }); - } else if (widget is RichText) { - results.add({ - 'text': widget.text.toPlainText(), - if (key != null) 'key': key, - 'type': 'RichText', - }); + if (focused != null) return; + if (element is StatefulElement && element.state is EditableTextState) { + final state = element.state as EditableTextState; + if (state.widget.focusNode.hasFocus) { + focused = state; + return; + } } - element.visitChildren(visit); } @@ -958,338 +2058,3212 @@ class FlutterSkillBinding { if (binding.rootElement != null) { visit(binding.rootElement!); } - - return results; + return focused; } - static List> _findByType(String typeName) { - final results = >[]; + /// Enhanced enter text with detailed error information + static Future> _performEnterTextWithDetails( + {String? key, required String text, String? refId}) async { + Element? element; + + // If refId is provided, find element by ref ID first + if (refId != null) { + element = _findElementByRefId(refId); + if (element != null) { + // Found by ref ID, proceed with text input + return await _enterTextIntoElement(element, text, method: 'ref_id'); + } + } - void visit(Element element) { - final widget = element.widget; - final type = widget.runtimeType.toString(); + // If no key or refId provided (null or empty), try the currently focused TextField + if ((key == null || key.isEmpty) && refId == null) { + final focusedField = _findFocusedTextField(); + if (focusedField != null) { + focusedField.updateEditingValue(TextEditingValue( + text: text, + selection: TextSelection.collapsed(offset: text.length), + )); + _log('Entered text "$text" into focused TextField'); + return { + 'success': true, + 'message': 'Text entered into focused TextField', + 'method': 'focused_field', + 'enteredText': text, + }; + } - if (type.toLowerCase().contains(typeName.toLowerCase())) { - String? key; - if (widget.key is ValueKey) { - key = (widget.key as ValueKey).value; - } + // No focused field found, try system channel as last resort + try { + await SystemChannels.textInput + .invokeMethod('TextInput.setEditingState', { + 'text': text, + 'selectionBase': text.length, + 'selectionExtent': text.length, + 'composingBase': -1, + 'composingExtent': -1, + }); + _log('Text input sent via system channel (no key, no focus)'); + return { + 'success': true, + 'message': + 'Text entered via system channel (no focused TextField found)', + 'method': 'system_channel', + 'enteredText': text, + }; + } catch (e) { + return { + 'success': false, + 'error': { + 'code': ErrorCode.elementNotFound, + 'message': 'No focused TextField found and system channel failed', + }, + 'suggestions': [ + 'Tap on a TextField first to focus it, then call enter_text(text: "...")', + 'Or provide a key: enter_text(key: "field_key", text: "...")', + 'Or use ref: enter_text(ref: "tf_0", text: "...")', + 'Use inspect_interactive() to find TextField elements with ref IDs', + ], + }; + } + } - final node = {'type': type}; - if (key != null) node['key'] = key; + // Try to find element by key if provided + if (key != null) { + element = _findElement(key: key); + } - final renderObject = element.renderObject; - if (renderObject is RenderBox && renderObject.hasSize) { - final offset = renderObject.localToGlobal(Offset.zero); - node['position'] = {'x': offset.dx, 'y': offset.dy}; - node['size'] = { - 'width': renderObject.size.width, - 'height': renderObject.size.height - }; + if (element == null) { + final suggestions = []; + + if (key != null) { + final similarKeys = _findSimilarKeys(key); + if (similarKeys.isNotEmpty) { + suggestions + .add('Similar keys found: ${similarKeys.take(5).toList()}'); } - results.add(node); + // Find TextField keys specifically + final textFieldKeys = _getAllKeys() + .where((k) => + k.toLowerCase().contains('field') || + k.toLowerCase().contains('input') || + k.toLowerCase().contains('text')) + .toList(); + if (textFieldKeys.isNotEmpty) { + suggestions.add( + 'TextField keys available: ${textFieldKeys.take(5).toList()}'); + } } - element.visitChildren(visit); + suggestions.add( + 'Use inspect_interactive() to find TextField elements with ref IDs'); + suggestions.add( + 'Or omit key/ref to enter text into the currently focused TextField'); + + return { + 'success': false, + 'error': { + 'code': ErrorCode.elementNotFound, + 'message': + 'No TextField matching ${key != null ? "key '$key'" : (refId != null ? "ref '$refId'" : "criteria")} found', + }, + 'target': {'key': key, 'ref': refId}, + 'suggestions': suggestions, + }; } - final binding = WidgetsBinding.instance; - // ignore: invalid_use_of_protected_member - if (binding.rootElement != null) { - visit(binding.rootElement!); + // Use the helper method to enter text into the found element + return await _enterTextIntoElement(element, text, + method: key != null ? 'key' : 'fallback'); + } + + static Future> _performScroll( + {String? key, String? text}) async { + var element = _findElement(key: key, text: text); + // Lazily built lists (ListView.builder, slivers) only mount the rows in + // view, so an off-screen target has no element yet. Page through every + // scrollable until it appears, then let ensureVisible do the fine work. + element ??= await _revealByPaging(() => _findElement(key: key, text: text)); + if (element == null) { + return { + 'success': false, + 'error': 'Element not found (key: $key, text: $text), ' + 'also after paging through all scrollables', + }; } - return results; + // Guard: context must still be mounted before calling ensureVisible. + // An unmounted context throws StateError and can crash the VM Service + // connection if the exception escapes the isolate's error zone. + if (!element.mounted) { + return { + 'success': false, + 'error': + 'Element context is no longer mounted (key: $key, text: $text)', + }; + } + + try { + await Scrollable.ensureVisible( + element, + duration: const Duration(milliseconds: 300), + curve: Curves.easeInOut, + ).timeout( + const Duration(seconds: 5), + onTimeout: () { + _log('scroll_to timed out after 5s (key: $key, text: $text)'); + }, + ); + _log('Scrolled to element (key: $key, text: $text)'); + return {'success': true}; + } catch (e) { + _log('Scroll failed: $e'); + return {'success': false, 'error': e.toString()}; + } } - // ==================== STATE & VALIDATION ==================== + static Future _performLongPress( + {String? key, String? text, int duration = 500}) async { + final element = _findElement(key: key, text: text); + if (element == null) return false; - static String? _getTextFieldValue(String key) { - final element = _findElementByKey(key); - if (element == null) return null; + final renderObject = element.renderObject; + if (renderObject is! RenderBox || !renderObject.hasSize) return false; - EditableTextState? editableTextState; - void findEditable(Element e) { - if (editableTextState != null) return; - if (e is StatefulElement && e.state is EditableTextState) { - editableTextState = e.state as EditableTextState; - return; - } - e.visitChildren(findEditable); - } + final center = + renderObject.localToGlobal(renderObject.size.center(Offset.zero)); + await _emitGesture(from: center, hold: Duration(milliseconds: duration)); + + _log('Long press completed (key: $key, text: $text)'); + return true; + } + + static Future _performSwipe( + {required String direction, double distance = 300, String? key}) async { + final binding = WidgetsBinding.instance; + Offset start; + + if (key != null) { + final element = _findElementByKey(key); + if (element == null) return false; + final renderObject = element.renderObject; + if (renderObject is! RenderBox) return false; + start = renderObject.localToGlobal(renderObject.size.center(Offset.zero)); + } else { + // Use window size for global swipe + final window = binding.platformDispatcher.views.first; + final size = window.physicalSize / window.devicePixelRatio; + start = Offset(size.width / 2, size.height / 2); + } + + Offset delta; + switch (direction.toLowerCase()) { + case 'up': + delta = Offset(0, -distance); + break; + case 'down': + delta = Offset(0, distance); + break; + case 'left': + delta = Offset(-distance, 0); + break; + case 'right': + delta = Offset(distance, 0); + break; + default: + return false; + } + + await _emitGesture(from: start, to: start + delta); + + _log('Swipe $direction completed'); + return true; + } + + static Future _performDrag( + {String? fromKey, String? toKey, int holdMs = 0}) async { + if (fromKey == null || toKey == null) return false; + + final fromElement = _findElementByKey(fromKey); + final toElement = _findElementByKey(toKey); + if (fromElement == null || toElement == null) return false; + + final fromRender = fromElement.renderObject; + final toRender = toElement.renderObject; + if (fromRender is! RenderBox || toRender is! RenderBox) return false; + + final start = fromRender.localToGlobal(fromRender.size.center(Offset.zero)); + final end = toRender.localToGlobal(toRender.size.center(Offset.zero)); + + // `hold` lets LongPressDraggable (and long-press-to-select lists) + // recognise the drag before the pointer starts moving. + await _emitGesture( + from: start, + to: end, + hold: Duration(milliseconds: holdMs), + travel: const Duration(milliseconds: 400), + ); + + _log('Drag from $fromKey to $toKey completed'); + return true; + } + + static Future _performDoubleTap({String? key, String? text}) async { + final element = _findElement(key: key, text: text); + if (element == null) return false; + + final renderObject = element.renderObject; + if (renderObject is! RenderBox || !renderObject.hasSize) return false; + + final center = + renderObject.localToGlobal(renderObject.size.center(Offset.zero)); + + await _dispatchTap(center); + await Future.delayed(const Duration(milliseconds: 50)); + await _dispatchTap(center); + + _log('Double tap completed (key: $key, text: $text)'); + return true; + } + + // ==================== COORDINATE-BASED ACTIONS ==================== + + static Future _performTapAt(double x, double y) async { + await _dispatchTap(Offset(x, y)); + _log('Tap at coordinates ($x, $y) completed'); + } + + static Future _performLongPressAt(double x, double y, + {int duration = 500}) async { + final position = Offset(x, y); + + // Show animated character holding + if (_indicatorsEnabled && _indicatorOverlay != null) { + _indicatorOverlay!.showCharacter(position, CharacterAction.holding, + hint: "Long pressing"); + await Future.delayed(const Duration(milliseconds: 200)); + _indicatorOverlay!.showLongPress( + position, + Duration(milliseconds: duration), + ); + } + + await _emitGesture(from: position, hold: Duration(milliseconds: duration)); + + _log('Long press at coordinates ($x, $y) completed'); + } + + static Future _performSwipeCoordinates( + double startX, + double startY, + double endX, + double endY, { + int duration = 300, + int holdMs = 0, + }) async { + final start = Offset(startX, startY); + final end = Offset(endX, endY); + final delta = end - start; + + // Show animated character swiping + if (_indicatorsEnabled && _indicatorOverlay != null) { + final direction = _getSwipeDirection(delta); + _indicatorOverlay!.showCharacter(start, CharacterAction.swiping, + hint: "Swiping $direction", endPosition: end); + await Future.delayed(const Duration(milliseconds: 200)); + _indicatorOverlay!.showSwipe(start, end); + } + + await _emitGesture( + from: start, + to: end, + hold: Duration(milliseconds: holdMs), + travel: Duration(milliseconds: duration), + ); + + _log('Swipe from ($startX, $startY) to ($endX, $endY) completed'); + } + + static Future _performEdgeSwipe({ + required String edge, + required String direction, + double distance = 200, + }) async { + final binding = WidgetsBinding.instance; + final view = binding.platformDispatcher.views.first; + final screenSize = view.physicalSize / view.devicePixelRatio; + + double startX, startY; + + // Calculate start position based on edge + switch (edge) { + case 'left': + startX = 0; + startY = screenSize.height / 2; + break; + case 'right': + startX = screenSize.width; + startY = screenSize.height / 2; + break; + case 'top': + startX = screenSize.width / 2; + startY = 0; + break; + case 'bottom': + startX = screenSize.width / 2; + startY = screenSize.height; + break; + default: + _log('Invalid edge: $edge'); + return false; + } + + double endX = startX, endY = startY; + + // Calculate end position based on direction + switch (direction) { + case 'right': + endX = startX + distance; + break; + case 'left': + endX = startX - distance; + break; + case 'up': + endY = startY - distance; + break; + case 'down': + endY = startY + distance; + break; + default: + _log('Invalid direction: $direction'); + return false; + } + + // Clamp to screen bounds + endX = endX.clamp(0, screenSize.width); + endY = endY.clamp(0, screenSize.height); + + await _performSwipeCoordinates(startX, startY, endX, endY, duration: 300); + _log( + 'Edge swipe from $edge edge, direction: $direction, distance: $distance'); + return true; + } + + // ==================== PERFORMANCE ==================== + + static Map _getFrameStats() { + return { + 'timestamp': DateTime.now().toIso8601String(), + 'logCount': _logs.length, + 'errorCount': _errors.length, + 'message': 'Frame stats available via DevTools timeline', + }; + } + + // ==================== UI INSPECTION ==================== + + static List> _findInteractiveElements() { + final results = >[]; + int elementCounter = 0; + + void visit(Element element, List ancestors) { + final widget = element.widget; + String? type; + String? text; + String? key; + String? semanticsLabel; + String? tooltip; + String? icon; + + if (widget.key is ValueKey) { + key = (widget.key as ValueKey).value; + } + + if (widget is ElevatedButton || + widget is TextButton || + widget is OutlinedButton || + widget is IconButton || + widget is FloatingActionButton) { + type = 'Button'; + text = _extractTextFrom(element); + if (widget is FloatingActionButton) { + // Prefer child text, fall back to tooltip (e.g. FAB with Icon child) + final fabTooltip = widget.tooltip; + if ((text == null || text.isEmpty) && + fabTooltip != null && + fabTooltip.isNotEmpty) { + text = fabTooltip; + } + } else if (widget is IconButton) { + if (widget.icon is Icon) { + icon = (widget.icon as Icon).icon?.toString(); + } + // Use tooltip as label when there is no visible text + final btnTooltip = widget.tooltip; + if ((text == null || text.isEmpty) && + btnTooltip != null && + btnTooltip.isNotEmpty) { + text = btnTooltip; + } + } + } else if (widget is TextField || widget is TextFormField) { + type = 'TextField'; + } else if (widget is Checkbox) { + type = 'Checkbox'; + } else if (widget is Switch) { + type = 'Switch'; + } else if (widget is Slider) { + type = 'Slider'; + } else if (widget is DropdownButton) { + type = 'Dropdown'; + } else if (widget is InkWell && widget.onTap != null) { + type = 'Tappable'; + text = _extractTextFrom(element); + } else if (widget is GestureDetector && widget.onTap != null) { + type = 'Tappable'; + text = _extractTextFrom(element); + } else if (widget is ListTile) { + type = 'ListTile'; + text = _extractTextFrom(element); + } else if (widget is BottomNavigationBarItem) { + type = 'BottomNavItem'; + } + + // Extract semantics label + if (widget is Semantics && widget.properties.label != null) { + semanticsLabel = widget.properties.label; + } + + // Extract tooltip + if (widget is Tooltip) { + tooltip = widget.message; + } + + if (type != null) { + elementCounter++; + final entry = { + 'id': 'elem_${elementCounter.toString().padLeft(3, '0')}', + 'type': type, + 'widgetType': widget.runtimeType.toString(), + }; + + if (key != null) entry['key'] = key; + if (text != null) entry['text'] = text; + if (semanticsLabel != null) entry['semanticsLabel'] = semanticsLabel; + if (tooltip != null) entry['tooltip'] = tooltip; + if (icon != null) entry['icon'] = icon; + + // Add ancestors (last 3 meaningful ancestors) + final meaningfulAncestors = ancestors + .where((a) => + !a.startsWith('_') && a != 'Builder' && a != 'StatefulWidget') + .toList(); + if (meaningfulAncestors.isNotEmpty) { + entry['ancestors'] = meaningfulAncestors.reversed.take(3).toList(); + } + + // Add position and size from render object + final renderObject = element.renderObject; + if (renderObject is RenderBox && renderObject.hasSize) { + final offset = renderObject.localToGlobal(Offset.zero); + + // Filter out elements from non-active routes (e.g. previous page still + // in Navigator stack). Negative x means the page was pushed left. + // Use a generous threshold to still include elements near the edge. + final view = WidgetsBinding.instance.platformDispatcher.views.first; + final sw = view.physicalSize.width / view.devicePixelRatio; + final sh = view.physicalSize.height / view.devicePixelRatio; + if (offset.dx < -sw * 0.5 || + offset.dx > sw * 1.5 || + offset.dy < -sh * 0.5 || + offset.dy > sh * 1.5) { + element.visitChildren((child) => visit(child, ancestors)); + return; + } + + // Helper function to safely convert double to int, handling Infinity/NaN + int safeRound(double value) { + if (!value.isFinite) return 0; + return value.round(); + } + + // Detect if coordinates are reliable (not (0,0) unless widget is actually at origin) + final isAtOrigin = offset.dx == 0 && offset.dy == 0; + final hasValidSize = + renderObject.size.width > 0 && renderObject.size.height > 0; + final isFinite = offset.dx.isFinite && offset.dy.isFinite; + + // For TextFields, check if they're genuinely at origin or just not laid out + bool coordinatesReliable = isFinite && hasValidSize; + if (isAtOrigin && (widget is TextField || widget is TextFormField)) { + // TextField at (0,0) is suspicious - likely not fully laid out + // Check if there are other widgets to compare against + coordinatesReliable = false; + } + + entry['bounds'] = { + 'x': safeRound(offset.dx), + 'y': safeRound(offset.dy), + 'width': safeRound(renderObject.size.width), + 'height': safeRound(renderObject.size.height), + }; + entry['center'] = { + 'x': safeRound(offset.dx + renderObject.size.width / 2), + 'y': safeRound(offset.dy + renderObject.size.height / 2), + }; + entry['visible'] = hasValidSize; + entry['coordinatesReliable'] = coordinatesReliable; + // `visible` only says the box has a size; `hittable` says a pointer + // at its center actually reaches it (not offstage, not covered by + // a bar/overlay) — the flag to trust before tapping/dragging. + entry['hittable'] = coordinatesReliable && + _isHittableAt( + element, offset + renderObject.size.center(Offset.zero)); + + // Add warning for unreliable coordinates + if (!coordinatesReliable && isAtOrigin) { + entry['warning'] = + 'Coordinates may be unreliable - widget might not be fully laid out. Use key or text for targeting.'; + } + } + + results.add(entry); + } + + // Build new ancestors list for children + final widgetName = widget.runtimeType.toString(); + final newAncestors = [...ancestors, widgetName]; + + element.visitChildren((child) => visit(child, newAncestors)); + } + + final binding = WidgetsBinding.instance; + // ignore: invalid_use_of_protected_member + if (binding.rootElement != null) { + visit(binding.rootElement!, []); + } + + return results; + } + + /// Enhanced interactive elements discovery for better automation with semantic ref ID system. + /// Returns structured data with semantic ref IDs, actions, bounds, and state information. + static Map _findInteractiveElementsStructured() { + final elements = >[]; + final refCounts = {}; + + void visit(Element element) { + final widget = element.widget; + String? type; + String? text; + String? key; + String? label; + String? tooltip; + List actions = []; + Map state = {}; + + // Get widget key + if (widget.key is ValueKey) { + key = (widget.key as ValueKey).value; + } + + // Identify widget type and available actions + if (widget is ElevatedButton || + widget is TextButton || + widget is OutlinedButton || + widget is IconButton || + widget is FloatingActionButton) { + type = widget.runtimeType.toString(); + text = _extractTextFrom(element); + actions = ['tap', 'long_press']; + + // Get button state and tooltip fallback + bool enabled = true; + if (widget is FloatingActionButton) { + enabled = widget.onPressed != null; + final fabTooltip = widget.tooltip; + if ((text == null || text.isEmpty) && + fabTooltip != null && + fabTooltip.isNotEmpty) { + text = fabTooltip; + } + } else if (widget is IconButton) { + enabled = widget.onPressed != null; + final btnTooltip = widget.tooltip; + if ((text == null || text.isEmpty) && + btnTooltip != null && + btnTooltip.isNotEmpty) { + text = btnTooltip; + } + } else if (widget is ElevatedButton) { + enabled = widget.onPressed != null; + } else if (widget is TextButton) { + enabled = widget.onPressed != null; + } else if (widget is OutlinedButton) { + enabled = widget.onPressed != null; + } + state['enabled'] = enabled; + } else if (widget is TextField || widget is TextFormField) { + type = widget.runtimeType.toString(); + actions = ['tap', 'enter_text']; + + // Get field label/hint - improved TextFormField label extraction + if (widget is TextField) { + label = widget.decoration?.labelText ?? widget.decoration?.hintText; + } else if (widget is TextFormField) { + // For TextFormField, try to extract the label from the decoration + // by looking at the child TextField + String? extractedLabel; + + void findTextField(Element e) { + if (extractedLabel != null) return; + final w = e.widget; + if (w is TextField) { + extractedLabel = + w.decoration?.labelText ?? w.decoration?.hintText; + return; + } + e.visitChildren(findTextField); + } + + findTextField(element); + label = extractedLabel ?? 'Text Field'; + } + + // Get current text value + final currentValue = _getTextFieldValueByElement(element); + state['value'] = currentValue ?? ''; + state['enabled'] = true; // TextFields are typically enabled + } else if (widget is Checkbox) { + type = 'Checkbox'; + actions = ['tap']; + + state['value'] = widget.value ?? false; + state['enabled'] = widget.onChanged != null; + } else if (widget is Switch) { + type = 'Switch'; + actions = ['tap']; + + state['value'] = widget.value; + state['enabled'] = widget.onChanged != null; + } else if (widget is Slider) { + type = 'Slider'; + actions = ['tap', 'swipe']; // Can tap to set value or swipe to adjust + + state['value'] = widget.value; + state['min'] = widget.min; + state['max'] = widget.max; + state['enabled'] = widget.onChanged != null; + } else if (widget is DropdownButton) { + type = 'DropdownButton'; + actions = ['tap']; + + // Get current value if available + state['value'] = widget.value?.toString() ?? ''; + state['enabled'] = widget.onChanged != null; + } else if (widget is InkWell && widget.onTap != null) { + type = 'InkWell'; + text = _extractTextFrom(element); + actions = ['tap', 'long_press']; + + state['enabled'] = true; + } else if (widget is GestureDetector && widget.onTap != null) { + type = 'GestureDetector'; + text = _extractTextFrom(element); + actions = ['tap', 'long_press']; + + state['enabled'] = true; + } else if (widget is ListTile) { + type = 'ListTile'; + text = _extractTextFrom(element); + actions = ['tap', 'long_press']; + + state['enabled'] = widget.enabled; + } + + // Extract tooltip + if (widget is Tooltip) { + tooltip = widget.message; + } + + // If this is an interactive element, add it to results + if (type != null) { + // Generate semantic ref ID using the new system + final refId = SemanticRefGenerator.generateRefId(element, refCounts); + + final elementEntry = { + 'ref': refId, + 'type': type, + 'actions': actions, + }; + + // Add optional fields + if (key != null && key.isNotEmpty) elementEntry['key'] = key; + if (text != null && text.isNotEmpty) elementEntry['text'] = text; + if (label != null && label.isNotEmpty) elementEntry['label'] = label; + if (tooltip != null) elementEntry['tooltip'] = tooltip; + + // Add bounds information - improved with RenderBox + final renderObject = element.renderObject; + if (renderObject is RenderBox && renderObject.hasSize) { + final offset = renderObject.localToGlobal(Offset.zero); + final size = renderObject.size; + + // Filter out elements from non-active routes + final view = WidgetsBinding.instance.platformDispatcher.views.first; + final sw = view.physicalSize.width / view.devicePixelRatio; + final sh = view.physicalSize.height / view.devicePixelRatio; + if (offset.dx < -sw * 0.5 || + offset.dx > sw * 1.5 || + offset.dy < -sh * 0.5 || + offset.dy > sh * 1.5) { + element.visitChildren(visit); + return; + } + + // Helper function to safely convert double to int + int safeRound(double value) { + if (!value.isFinite) return 0; + return value.round(); + } + + final bounds = { + 'x': safeRound(offset.dx), + 'y': safeRound(offset.dy), + 'w': safeRound(size.width), + 'h': safeRound(size.height), + }; + elementEntry['bounds'] = bounds; + + final isVisible = size.width > 0 && + size.height > 0 && + offset.dx.isFinite && + offset.dy.isFinite; + state['visible'] = isVisible; + state['hittable'] = isVisible && + _isHittableAt(element, offset + size.center(Offset.zero)); + + // Cache element for performance optimization + if (isVisible && refId.isNotEmpty) { + SemanticRefGenerator.cacheElement(refId, element, bounds); + } + } else { + // Element has no render box or size - set default bounds + elementEntry['bounds'] = {'x': 0, 'y': 0, 'w': 0, 'h': 0}; + state['visible'] = false; + state['hittable'] = false; + } + + // Merge state information + elementEntry.addAll(state); + + elements.add(elementEntry); + } + + // Continue traversing children + element.visitChildren(visit); + } + + final binding = WidgetsBinding.instance; + // ignore: invalid_use_of_protected_member + if (binding.rootElement != null) { + visit(binding.rootElement!); + } + + // Generate summary based on ref counts + final summaryParts = []; + refCounts.forEach((prefix, count) { + switch (prefix) { + case 'btn': + summaryParts.add('$count button${count == 1 ? '' : 's'}'); + break; + case 'tf': + summaryParts.add('$count text field${count == 1 ? '' : 's'}'); + break; + case 'sw': + summaryParts.add('$count switch${count == 1 ? '' : 'es'}'); + break; + case 'sl': + summaryParts.add('$count slider${count == 1 ? '' : 's'}'); + break; + case 'dd': + summaryParts.add('$count dropdown${count == 1 ? '' : 's'}'); + break; + case 'item': + summaryParts.add('$count list item${count == 1 ? '' : 's'}'); + break; + case 'lnk': + summaryParts.add('$count link${count == 1 ? '' : 's'}'); + break; + } + }); + + final summary = summaryParts.isEmpty + ? 'No interactive elements found' + : '${elements.length} interactive: ${summaryParts.join(', ')}'; + + // Cache the result for server.dart usage + _lastInspectResult = >{}; + for (final element in elements) { + final refId = element['ref'] as String?; + if (refId != null) { + _lastInspectResult![refId] = element; + } + } + + return { + 'elements': elements, + 'summary': summary, + }; + } + + /// Helper method to get TextField value by Element (used by structured inspect) + static String? _getTextFieldValueByElement(Element element) { + EditableTextState? editableTextState; + + void findEditable(Element e) { + if (editableTextState != null) return; + if (e is StatefulElement && e.state is EditableTextState) { + editableTextState = e.state as EditableTextState; + return; + } + e.visitChildren(findEditable); + } + + findEditable(element); + + if (editableTextState != null) { + return editableTextState!.textEditingValue.text; + } + + return null; + } + + static Map _getWidgetTree({int maxDepth = 10}) { + Map buildNode(Element element, int depth) { + final widget = element.widget; + final node = { + 'type': widget.runtimeType.toString(), + }; + + if (widget.key is ValueKey) { + node['key'] = (widget.key as ValueKey).value; + } + + if (widget is Text) { + node['text'] = widget.data; + } + + final renderObject = element.renderObject; + if (renderObject is RenderBox && renderObject.hasSize) { + node['size'] = { + 'width': renderObject.size.width, + 'height': renderObject.size.height + }; + final offset = renderObject.localToGlobal(Offset.zero); + node['position'] = {'x': offset.dx, 'y': offset.dy}; + } + + if (depth < maxDepth) { + final children = >[]; + element.visitChildren((child) { + children.add(buildNode(child, depth + 1)); + }); + if (children.isNotEmpty) { + node['children'] = children; + } + } + + return node; + } + + final binding = WidgetsBinding.instance; + // ignore: invalid_use_of_protected_member + if (binding.rootElement != null) { + return buildNode(binding.rootElement!, 0); + } + return {}; + } + + static Map? _getWidgetProperties(String key) { + final element = _findElementByKey(key); + if (element == null) return null; + + final widget = element.widget; + final props = { + 'type': widget.runtimeType.toString(), + 'key': key, + }; + + final renderObject = element.renderObject; + if (renderObject is RenderBox && renderObject.hasSize) { + props['size'] = { + 'width': renderObject.size.width, + 'height': renderObject.size.height + }; + final offset = renderObject.localToGlobal(Offset.zero); + props['position'] = {'x': offset.dx, 'y': offset.dy}; + props['visible'] = renderObject.attached && + renderObject.size.width > 0 && + renderObject.size.height > 0; + } + + if (widget is Text) { + props['text'] = widget.data; + props['style'] = widget.style?.toString(); + } else if (widget is Container) { + props['color'] = widget.color?.toString(); + props['padding'] = widget.padding?.toString(); + props['margin'] = widget.margin?.toString(); + } + + return props; + } + + static List> _getTextContent() { + final results = >[]; + + void visit(Element element) { + final widget = element.widget; + String? key; + if (widget.key is ValueKey) { + key = (widget.key as ValueKey).value; + } + + if (widget is Text && widget.data != null) { + results.add({ + 'text': widget.data, + if (key != null) 'key': key, + 'type': 'Text', + }); + } else if (widget is RichText) { + results.add({ + 'text': widget.text.toPlainText(), + if (key != null) 'key': key, + 'type': 'RichText', + }); + } + + element.visitChildren(visit); + } + + final binding = WidgetsBinding.instance; + // ignore: invalid_use_of_protected_member + if (binding.rootElement != null) { + visit(binding.rootElement!); + } + + return results; + } + + static List> _findByType(String typeName) { + final results = >[]; + + void visit(Element element) { + final widget = element.widget; + final type = widget.runtimeType.toString(); + + if (type.toLowerCase().contains(typeName.toLowerCase())) { + String? key; + if (widget.key is ValueKey) { + key = (widget.key as ValueKey).value; + } + + final node = {'type': type}; + if (key != null) node['key'] = key; + + final renderObject = element.renderObject; + if (renderObject is RenderBox && renderObject.hasSize) { + final offset = renderObject.localToGlobal(Offset.zero); + + // Helper function to safely convert double to int, handling Infinity/NaN + int safeRound(double value) { + if (!value.isFinite) return 0; + return value.round(); + } + + // Use safeRound to prevent JSON serialization errors + node['position'] = { + 'x': safeRound(offset.dx), + 'y': safeRound(offset.dy) + }; + node['size'] = { + 'width': safeRound(renderObject.size.width), + 'height': safeRound(renderObject.size.height) + }; + + // Add coordinate reliability flag + final isReliable = offset.dx.isFinite && + offset.dy.isFinite && + (offset.dx != 0 || offset.dy != 0) && + renderObject.size.width > 0 && + renderObject.size.height > 0; + node['coordinatesReliable'] = isReliable; + } + + results.add(node); + } + + element.visitChildren(visit); + } + + final binding = WidgetsBinding.instance; + // ignore: invalid_use_of_protected_member + if (binding.rootElement != null) { + visit(binding.rootElement!); + } + + return results; + } + + // ==================== STATE & VALIDATION ==================== + + static String? _getTextFieldValue(String key) { + final element = _findElementByKey(key); + if (element == null) return null; + + // First try: look for EditableTextState (TextField/TextFormField) + EditableTextState? editableTextState; + void findEditable(Element e) { + if (editableTextState != null) return; + if (e is StatefulElement && e.state is EditableTextState) { + editableTextState = e.state as EditableTextState; + return; + } + e.visitChildren(findEditable); + } + + findEditable(element); + + if (editableTextState != null) { + return editableTextState!.textEditingValue.text; + } + + // Fallback: read child Text widget content (for buttons, labels, etc.) + return _extractTextFrom(element); + } + + static bool? _getCheckboxState(String key) { + final element = _findElementByKey(key); + if (element == null) return null; + + final widget = element.widget; + if (widget is Checkbox) return widget.value; + if (widget is Switch) return widget.value; + if (widget is CheckboxListTile) return widget.value; + if (widget is SwitchListTile) return widget.value; + return null; + } + + /// The EditableText inside the widget with [key] (TextField, TextFormField + /// or a wrapper), or null. + static EditableTextState? _findEditableText(String key) { + final element = _findElementByKey(key); + if (element == null) return null; + EditableTextState? found; + void visit(Element e) { + if (found != null) return; + if (e is StatefulElement && e.state is EditableTextState) { + found = e.state as EditableTextState; + return; + } + e.visitChildren(visit); + } + + visit(element); + return found; + } + + static double? _getSliderValue(String key) { + final element = _findElementByKey(key); + if (element == null) return null; + + final widget = element.widget; + if (widget is Slider) { + return widget.value; + } + return null; + } + + static Future _waitForElement( + {String? key, String? text, int timeout = 5000}) async { + final endTime = DateTime.now().add(Duration(milliseconds: timeout)); + + while (DateTime.now().isBefore(endTime)) { + final element = _findElement(key: key, text: text); + if (element != null) return true; + await Future.delayed(const Duration(milliseconds: 100)); + } + + return false; + } + + static Future _waitForGone( + {String? key, String? text, int timeout = 5000}) async { + final endTime = DateTime.now().add(Duration(milliseconds: timeout)); + + while (DateTime.now().isBefore(endTime)) { + final element = _findElement(key: key, text: text); + if (element == null) return true; + await Future.delayed(const Duration(milliseconds: 100)); + } + + return false; + } + + // ==================== SCREENSHOT ==================== + + /// Captures the full composited scene from the RenderView's layer. + /// This correctly captures the current visible page even after navigation, + /// avoiding the bug where DFS finds a RepaintBoundary from an old route. + /// Returns the image in logical pixel dimensions scaled by [pixelRatio]. + static Future _captureFullScene({double pixelRatio = 1.0}) async { + // Bounded: a hidden window may never deliver a frame (see _nextFrame). + await _nextFrame(); + + final binding = WidgetsBinding.instance; + // ignore: invalid_use_of_protected_member + final renderObject = binding.rootElement?.renderObject; + if (renderObject == null) return null; + + // Walk up to the RenderView (root of the render tree). + // RenderView's layer contains ALL Navigator routes composited together. + RenderObject root = renderObject; + while (root.parent != null) { + root = root.parent!; + } + + // ignore: invalid_use_of_protected_member + final layer = root.layer; + if (layer is OffsetLayer) { + // RenderView's TransformLayer includes a device-pixel-ratio scale. + // Use physical bounds and adjust pixelRatio to produce logical-sized output. + final view = binding.platformDispatcher.views.first; + final physicalSize = view.physicalSize; + final dpr = view.devicePixelRatio; + return layer.toImage( + Offset.zero & physicalSize, + pixelRatio: pixelRatio / dpr, + ); + } + + // Fallback: try a RepaintBoundary (pre-navigation or single-route apps) + if (renderObject is RenderRepaintBoundary) { + return renderObject.toImage(pixelRatio: pixelRatio); + } + + RenderRepaintBoundary? boundary; + void findBoundary(RenderObject obj) { + if (boundary != null) return; + if (obj is RenderRepaintBoundary) { + boundary = obj; + return; + } + obj.visitChildren(findBoundary); + } + + renderObject.visitChildren(findBoundary); + if (boundary == null) { + _log('No RenderRepaintBoundary found'); + return null; + } + return boundary!.toImage(pixelRatio: pixelRatio); + } + + static Future<_Screenshot?> _takeScreenshot( + {double quality = 1.0, int? maxWidth}) async { + try { + // Use quality as pixel ratio (lower = smaller image) + var pixelRatio = quality.clamp(0.1, 1.0); + var image = await _captureFullScene(pixelRatio: pixelRatio); + if (image == null) { + _log('Screenshot capture failed'); + return null; + } + + // Scale down if maxWidth is specified + if (maxWidth != null && image.width > maxWidth) { + final scale = maxWidth / image.width; + final recorder = ui.PictureRecorder(); + final canvas = Canvas(recorder); + final newWidth = (image.width * scale).toInt(); + final newHeight = (image.height * scale).toInt(); + + canvas.drawImageRect( + image, + Rect.fromLTWH(0, 0, image.width.toDouble(), image.height.toDouble()), + Rect.fromLTWH(0, 0, newWidth.toDouble(), newHeight.toDouble()), + Paint()..filterQuality = FilterQuality.medium, + ); + + final picture = recorder.endRecording(); + image = await picture.toImage(newWidth, newHeight); + } + + final byteData = await image.toByteData(format: ui.ImageByteFormat.png); + if (byteData == null) return null; + return _Screenshot(base64Encode(byteData.buffer.asUint8List()), + image.width, image.height); + } catch (e) { + _log('Screenshot failed: $e'); + return null; + } + } + + static Future _takeRegionScreenshot( + double x, double y, double width, double height) async { + try { + final fullImage = await _captureFullScene(pixelRatio: 1.0); + if (fullImage == null) { + _log('Region screenshot capture failed'); + return null; + } + + // Crop to specified region + final recorder = ui.PictureRecorder(); + final canvas = Canvas(recorder); + + canvas.drawImageRect( + fullImage, + Rect.fromLTWH(x, y, width, height), + Rect.fromLTWH(0, 0, width, height), + Paint(), + ); + + final picture = recorder.endRecording(); + final croppedImage = await picture.toImage(width.toInt(), height.toInt()); + final byteData = + await croppedImage.toByteData(format: ui.ImageByteFormat.png); + if (byteData == null) return null; + return base64Encode(byteData.buffer.asUint8List()); + } catch (e) { + _log('Region screenshot failed: $e'); + return null; + } + } + + static Future _takeElementScreenshot(String key) async { + try { + final element = _findElementByKey(key); + if (element == null) { + _log('Element not found: $key'); + return null; + } + + final renderObject = element.renderObject; + if (renderObject == null) { + _log('No render object for element: $key'); + return null; + } + + // If it's a RenderRepaintBoundary, capture directly + if (renderObject is RenderRepaintBoundary) { + final image = await renderObject.toImage(pixelRatio: 1.0); + final byteData = await image.toByteData(format: ui.ImageByteFormat.png); + if (byteData == null) return null; + return base64Encode(byteData.buffer.asUint8List()); + } + + // For other render objects, find the nearest RepaintBoundary ancestor + RenderObject? current = renderObject; + while (current != null) { + if (current is RenderRepaintBoundary) { + // Get the element's bounds relative to the boundary + if (renderObject is RenderBox) { + final box = renderObject; + final boundaryBox = current; + + // Get element position relative to boundary + final offset = + box.localToGlobal(Offset.zero, ancestor: boundaryBox); + final size = box.size; + + // Capture the boundary + final fullImage = await current.toImage(pixelRatio: 1.0); + + // Crop to element bounds + final recorder = ui.PictureRecorder(); + final canvas = Canvas(recorder); + + // Draw the cropped portion + canvas.drawImageRect( + fullImage, + Rect.fromLTWH(offset.dx, offset.dy, size.width, size.height), + Rect.fromLTWH(0, 0, size.width, size.height), + Paint(), + ); + + final picture = recorder.endRecording(); + final croppedImage = + await picture.toImage(size.width.toInt(), size.height.toInt()); + final byteData = + await croppedImage.toByteData(format: ui.ImageByteFormat.png); + if (byteData == null) return null; + return base64Encode(byteData.buffer.asUint8List()); + } + } + current = current.parent; + } + + _log('No suitable RenderRepaintBoundary ancestor found for: $key'); + return null; + } catch (e) { + _log('Element screenshot failed: $e'); + return null; + } + } + + // ==================== NAVIGATION ==================== + + /// Every route currently mounted, in stack order (outer navigator first, + /// each navigator bottom → top, nested navigators after the route that + /// hosts them). Each entry: `name` (RouteSettings.name — go_router sets + /// the route name or path), `key` (Page key), `isCurrent`, `depth` + /// (navigator nesting), `type`. + /// + /// Routes are found through the `_ModalScopeStatus` inherited widget that + /// every ModalRoute installs above its content: walking the element tree + /// meets each route exactly once and does not register dependencies (as + /// `ModalRoute.of` would). Field access is dynamic because the class is + /// private to the framework; its members are public and stable. + static List> _routeEntries() { + final entries = >[]; + void visit(Element element, int depth) { + var next = depth; + final widget = element.widget; + if (widget.runtimeType.toString() == '_ModalScopeStatus') { + try { + final route = (widget as dynamic).route as Route; + final settings = route.settings; + entries.add({ + 'name': settings.name, + 'key': settings is Page ? settings.key?.toString() : null, + 'isCurrent': route.isCurrent, + 'depth': depth, + 'type': route.runtimeType.toString(), + }); + next = depth + 1; + } catch (_) { + // framework internals changed shape; skip silently + } + } + element.visitChildren((child) => visit(child, next)); + } + + // ignore: invalid_use_of_protected_member + final rootElement = WidgetsBinding.instance.rootElement; + if (rootElement != null) visit(rootElement, 0); + return entries; + } + + /// Name of the topmost current route (innermost navigator wins). Falls + /// back to the page key when the route has no name. + static String? _getCurrentRoute() { + Map? current; + for (final entry in _routeEntries()) { + if (entry['isCurrent'] == true) current = entry; + } + if (current == null) return null; + return (current['name'] ?? current['key']) as String?; + } + + static bool _goBack() { + final context = _findNavigatorContext(); + if (context == null) return false; + + final navigator = Navigator.of(context, rootNavigator: false); + if (navigator.canPop()) { + navigator.pop(); + return true; + } + return false; + } + + static List _getNavigationStack() { + final routes = []; + for (final entry in _routeEntries()) { + final label = (entry['name'] ?? entry['key'] ?? entry['type']) as String; + routes.add(label); + } + return routes; + } + + /// Pages every mounted vertical/horizontal scrollable from its start to + /// its end (viewport-sized jumps, one frame each) until [find] returns an + /// element. Returns it, or null when nothing appeared anywhere. + static Future _revealByPaging(Element? Function() find) async { + final positions = []; + void visit(Element e) { + if (e is StatefulElement && e.state is ScrollableState) { + final position = (e.state as ScrollableState).position; + if (position.hasContentDimensions && + position.maxScrollExtent > position.minScrollExtent) { + positions.add(position); + } + } + e.visitChildren(visit); + } + + // ignore: invalid_use_of_protected_member + final rootElement = WidgetsBinding.instance.rootElement; + if (rootElement != null) visit(rootElement); + + for (final position in positions) { + final origin = position.pixels; + final step = position.viewportDimension * 0.8; + var offset = position.minScrollExtent; + while (true) { + position.jumpTo(offset); + await _settle(); + final found = find(); + if (found != null) return found; + if (offset >= position.maxScrollExtent) break; + offset = (offset + step) + .clamp(position.minScrollExtent, position.maxScrollExtent); + } + position.jumpTo(origin); + await _settle(); + } + return null; + } + + static BuildContext? _findNavigatorContext() { + BuildContext? context; + final binding = WidgetsBinding.instance; + + void visit(Element element) { + if (context != null) return; + if (element.widget is Navigator) { + context = element; + return; + } + element.visitChildren(visit); + } + + // ignore: invalid_use_of_protected_member + if (binding.rootElement != null) { + visit(binding.rootElement!); + } + + return context; + } + + // ==================== DEBUG & LOGS ==================== + + static void _log(String message) { + final logEntry = '[${DateTime.now().toIso8601String()}] $message'; + _logs.add(logEntry); + print('Flutter Skill: $message'); + } + + static Map _getPerformanceMetrics() { + return { + 'timestamp': DateTime.now().toIso8601String(), + 'logCount': _logs.length, + 'errorCount': _errors.length, + }; + } + + // ==================== HELPERS ==================== + + static String _getSwipeDirection(Offset delta) { + if (delta.dx.abs() > delta.dy.abs()) { + return delta.dx > 0 ? 'right' : 'left'; + } else { + return delta.dy > 0 ? 'down' : 'up'; + } + } + + static String? _extractTextFrom(Element element) { + String? found; + void visit(Element e) { + if (found != null) return; + if (e.widget is Text) { + final data = (e.widget as Text).data; + // Skip null or empty text — internal Flutter widgets (e.g. inside + // FloatingActionButton, Tooltip) often render empty Text("") nodes. + if (data != null && data.isNotEmpty) { + found = data; + return; + } + } else if (e.widget is RichText) { + final plain = (e.widget as RichText).text.toPlainText(); + if (plain.isNotEmpty) { + found = plain; + return; + } + } + e.visitChildren(visit); + } + + visit(element); + return found; + } +} + +// ==================== TEST INDICATORS ==================== + +/// Visual indicator types for test actions +enum IndicatorType { tap, swipe, longPress, textInput, hint, cursor, character } + +/// Character action types for animated character +enum CharacterAction { + tapping, // Character tapping/clicking + typing, // Character typing + swiping, // Character swiping gesture + holding, // Character holding/pressing + dragging, // Character dragging + pointing, // Character pointing +} + +/// Character state for persistent character +enum CharacterState { + idle, // Standing still, breathing animation + walking, // Moving to target + acting, // Performing action +} + +/// Style configuration for indicators +enum IndicatorStyle { + minimal, // Small, fast, no hints + standard, // Medium, normal speed, 1s hints + detailed, // Large, slow, 2s hints + debug info +} + +/// Data model for an active indicator +class IndicatorData { + final IndicatorType type; + final Offset? position; + final Offset? endPosition; + final Duration? duration; + final Rect? bounds; + final String? message; + final CharacterAction? action; + final DateTime timestamp; + + IndicatorData({ + required this.type, + this.position, + this.endPosition, + this.duration, + this.bounds, + this.message, + this.action, + DateTime? timestamp, + }) : timestamp = timestamp ?? DateTime.now(); +} + +/// Manages the overlay entry and indicator display +class TestIndicatorOverlay { + OverlayEntry? _entry; + final GlobalKey<_TestIndicatorWidgetState> _widgetKey = GlobalKey(); + IndicatorStyle _style = IndicatorStyle.standard; + + /// Enable the indicator overlay + void enable() { + if (_entry != null) return; + + final overlay = _findOverlayState(); + if (overlay == null) { + print('Flutter Skill: Cannot find Overlay to show indicators'); + return; + } + + _entry = OverlayEntry( + builder: (context) => _TestIndicatorWidget( + key: _widgetKey, + style: _style, + ), + ); + + overlay.insert(_entry!); + print('Flutter Skill: Test indicators enabled'); + } + + /// Disable and remove the indicator overlay + void disable() { + _entry?.remove(); + _entry = null; + print('Flutter Skill: Test indicators disabled'); + } + + /// Set indicator style + void setStyle(IndicatorStyle style) { + _style = style; + _widgetKey.currentState?.setStyle(style); + } + + /// Show tap indicator + void showTap(Offset position, {String? hint}) { + _widgetKey.currentState?.addIndicator(IndicatorData( + type: IndicatorType.tap, + position: position, + message: hint, + )); + } + + /// Show swipe indicator + void showSwipe(Offset start, Offset end, {String? hint}) { + _widgetKey.currentState?.addIndicator(IndicatorData( + type: IndicatorType.swipe, + position: start, + endPosition: end, + message: hint, + )); + } + + /// Show long press indicator + void showLongPress(Offset position, Duration duration, {String? hint}) { + _widgetKey.currentState?.addIndicator(IndicatorData( + type: IndicatorType.longPress, + position: position, + duration: duration, + message: hint, + )); + } + + /// Show text input indicator + void showTextInput(Rect bounds, {String? hint}) { + _widgetKey.currentState?.addIndicator(IndicatorData( + type: IndicatorType.textInput, + bounds: bounds, + message: hint, + )); + } + + /// Show action hint only + void showHint(String message) { + _widgetKey.currentState?.addIndicator(IndicatorData( + type: IndicatorType.hint, + message: message, + )); + } + + /// Show cursor/pointer indicator at position + void showCursor(Offset position, {String? hint}) { + _widgetKey.currentState?.addIndicator(IndicatorData( + type: IndicatorType.cursor, + position: position, + message: hint, + )); + } + + /// Show animated character with action + void showCharacter(Offset position, CharacterAction action, + {String? hint, Offset? endPosition}) { + _widgetKey.currentState?.addIndicator(IndicatorData( + type: IndicatorType.character, + position: position, + endPosition: endPosition, + action: action, + message: hint, + )); + } + + /// Find the app's overlay state + OverlayState? _findOverlayState() { + final context = WidgetsBinding.instance.rootElement; + if (context == null) return null; + + OverlayState? overlayState; + + void visit(Element element) { + if (overlayState != null) return; + if (element.widget is Overlay) { + overlayState = (element as StatefulElement).state as OverlayState; + return; + } + element.visitChildren(visit); + } + + visit(context); + return overlayState; + } +} + +/// Widget that renders all test indicators +class _TestIndicatorWidget extends StatefulWidget { + final IndicatorStyle style; + + const _TestIndicatorWidget({super.key, required this.style}); + + @override + State<_TestIndicatorWidget> createState() => _TestIndicatorWidgetState(); +} + +class _TestIndicatorWidgetState extends State<_TestIndicatorWidget> + with TickerProviderStateMixin { + final List _indicators = []; + final Map _controllers = {}; + IndicatorStyle _style = IndicatorStyle.standard; + + @override + void initState() { + super.initState(); + _style = widget.style; + } + + @override + void dispose() { + for (final controller in _controllers.values) { + controller.dispose(); + } + super.dispose(); + } + + void setStyle(IndicatorStyle style) { + setState(() { + _style = style; + }); + } + + void addIndicator(IndicatorData indicator) { + setState(() { + // If adding a character indicator, remove any existing character indicators first + // This ensures only one character is visible at a time + if (indicator.type == IndicatorType.character) { + final existingCharacters = _indicators + .where((i) => i.type == IndicatorType.character) + .toList(); + + for (final existing in existingCharacters) { + _indicators.remove(existing); + _controllers[existing]?.dispose(); + _controllers.remove(existing); + } + } + + _indicators.add(indicator); + + // Create animation controller + final duration = _getAnimationDuration(indicator); + final controller = AnimationController( + vsync: this, + duration: duration, + ); + + _controllers[indicator] = controller; + + // Start animation + controller.forward().then((_) { + // Remove after animation completes + Future.delayed(const Duration(milliseconds: 100), () { + if (mounted) { + setState(() { + _indicators.remove(indicator); + _controllers[indicator]?.dispose(); + _controllers.remove(indicator); + }); + } + }); + }); + }); + } + + Duration _getAnimationDuration(IndicatorData indicator) { + switch (_style) { + case IndicatorStyle.minimal: + return const Duration(milliseconds: 200); + case IndicatorStyle.standard: + return indicator.type == IndicatorType.longPress && + indicator.duration != null + ? indicator.duration! + : const Duration(milliseconds: 500); + case IndicatorStyle.detailed: + return indicator.type == IndicatorType.longPress && + indicator.duration != null + ? indicator.duration! + : indicator.type == IndicatorType.character + ? const Duration( + milliseconds: + 10000) // 10 seconds - persistent walking character + : const Duration(milliseconds: 800); + } + } + + @override + Widget build(BuildContext context) { + return IgnorePointer( + child: Stack( + children: [ + // Render each indicator + for (final indicator in _indicators) + _buildIndicator(context, indicator), + + // Render action hints at the top + if (_style != IndicatorStyle.minimal) ..._buildActionHints(context), + ], + ), + ); + } + + Widget _buildIndicator(BuildContext context, IndicatorData indicator) { + final controller = _controllers[indicator]; + if (controller == null) return const SizedBox.shrink(); + + switch (indicator.type) { + case IndicatorType.tap: + return _TapIndicator( + position: indicator.position!, + animation: controller, + style: _style, + ); + + case IndicatorType.swipe: + return _SwipeIndicator( + start: indicator.position!, + end: indicator.endPosition!, + animation: controller, + style: _style, + ); + + case IndicatorType.longPress: + return _LongPressIndicator( + position: indicator.position!, + animation: controller, + style: _style, + ); + + case IndicatorType.textInput: + return _TextInputIndicator( + bounds: indicator.bounds!, + animation: controller, + style: _style, + ); + + case IndicatorType.cursor: + return _CursorIndicator( + position: indicator.position!, + animation: controller, + style: _style, + ); + + case IndicatorType.character: + return _AnimatedCharacterIndicator( + position: indicator.position!, + endPosition: indicator.endPosition, + action: indicator.action ?? CharacterAction.pointing, + animation: controller, + style: _style, + ); + + case IndicatorType.hint: + return const SizedBox.shrink(); + } + } + + List _buildActionHints(BuildContext context) { + final hints = _indicators + .where((i) => i.message != null && i.message!.isNotEmpty) + .toList(); + + if (hints.isEmpty) return []; + + // Show only the most recent hint + final latestHint = hints.last; + final controller = _controllers[latestHint]; + if (controller == null) return []; + + return [ + Positioned( + top: MediaQuery.of(context).padding.top + 8, + left: 16, + right: 16, + child: _ActionHint( + message: latestHint.message!, + animation: controller, + style: _style, + ), + ), + ]; + } +} + +// ==================== INDICATOR WIDGETS ==================== + +/// Tap indicator: expanding circle +class _TapIndicator extends StatelessWidget { + final Offset position; + final Animation animation; + final IndicatorStyle style; + + const _TapIndicator({ + required this.position, + required this.animation, + required this.style, + }); + + @override + Widget build(BuildContext context) { + return AnimatedBuilder( + animation: animation, + builder: (context, child) { + final size = _getSize(); + final radius = size * animation.value; + final opacity = (1.0 - animation.value) * 0.5; + + return Positioned( + left: position.dx - radius, + top: position.dy - radius, + child: Container( + width: radius * 2, + height: radius * 2, + decoration: BoxDecoration( + shape: BoxShape.circle, + border: Border.all( + color: const Color(0xFF4CAF50).withOpacity(opacity), + width: 3, + ), + color: const Color(0xFF4CAF50).withOpacity(opacity * 0.3), + ), + ), + ); + }, + ); + } + + double _getSize() { + switch (style) { + case IndicatorStyle.minimal: + return 30; + case IndicatorStyle.standard: + return 50; + case IndicatorStyle.detailed: + return 70; + } + } +} + +/// Swipe indicator: arrow with trail +class _SwipeIndicator extends StatelessWidget { + final Offset start; + final Offset end; + final Animation animation; + final IndicatorStyle style; + + const _SwipeIndicator({ + required this.start, + required this.end, + required this.animation, + required this.style, + }); + + @override + Widget build(BuildContext context) { + return AnimatedBuilder( + animation: animation, + builder: (context, child) { + return CustomPaint( + painter: _SwipePainter( + start: start, + end: end, + progress: animation.value, + style: style, + ), + child: Container(), + ); + }, + ); + } +} + +class _SwipePainter extends CustomPainter { + final Offset start; + final Offset end; + final double progress; + final IndicatorStyle style; + + _SwipePainter({ + required this.start, + required this.end, + required this.progress, + required this.style, + }); + + @override + void paint(Canvas canvas, Size size) { + final paint = Paint() + ..color = const Color(0xFF9C27B0).withOpacity((1.0 - progress) * 0.5) + ..strokeWidth = style == IndicatorStyle.minimal ? 2 : 3 + ..style = PaintingStyle.stroke; + + // Draw dashed line + final path = Path(); + path.moveTo(start.dx, start.dy); + path.lineTo(end.dx, end.dy); + + canvas.drawPath( + _createDashedPath(path, dashWidth: 10, dashSpace: 5), + paint, + ); + + // Draw arrow head + final angle = (end - start).direction; + final arrowSize = style == IndicatorStyle.minimal ? 15.0 : 20.0; + + final arrowPath = Path(); + arrowPath.moveTo(end.dx, end.dy); + arrowPath.lineTo( + end.dx - arrowSize * cos(angle - pi / 6), + end.dy - arrowSize * sin(angle - pi / 6), + ); + arrowPath.moveTo(end.dx, end.dy); + arrowPath.lineTo( + end.dx - arrowSize * cos(angle + pi / 6), + end.dy - arrowSize * sin(angle + pi / 6), + ); + + paint.style = PaintingStyle.stroke; + canvas.drawPath(arrowPath, paint); + } + + Path _createDashedPath(Path source, + {required double dashWidth, required double dashSpace}) { + final path = Path(); + for (final metric in source.computeMetrics()) { + double distance = 0; + bool draw = true; + while (distance < metric.length) { + final length = draw ? dashWidth : dashSpace; + if (distance + length > metric.length) { + if (draw) { + path.addPath( + metric.extractPath(distance, metric.length), + Offset.zero, + ); + } + break; + } + if (draw) { + path.addPath( + metric.extractPath(distance, distance + length), + Offset.zero, + ); + } + distance += length; + draw = !draw; + } + } + return path; + } + + @override + bool shouldRepaint(_SwipePainter oldDelegate) => + progress != oldDelegate.progress; +} + +/// Long press indicator: filling circle +class _LongPressIndicator extends StatelessWidget { + final Offset position; + final Animation animation; + final IndicatorStyle style; + + const _LongPressIndicator({ + required this.position, + required this.animation, + required this.style, + }); + + @override + Widget build(BuildContext context) { + return AnimatedBuilder( + animation: animation, + builder: (context, child) { + final size = _getSize(); + final opacity = 0.5; + + return Positioned( + left: position.dx - size, + top: position.dy - size, + child: SizedBox( + width: size * 2, + height: size * 2, + child: CustomPaint( + painter: _LongPressPainter( + progress: animation.value, + opacity: opacity, + ), + ), + ), + ); + }, + ); + } + + double _getSize() { + switch (style) { + case IndicatorStyle.minimal: + return 25; + case IndicatorStyle.standard: + return 35; + case IndicatorStyle.detailed: + return 45; + } + } +} + +class _LongPressPainter extends CustomPainter { + final double progress; + final double opacity; + + _LongPressPainter({required this.progress, required this.opacity}); + + @override + void paint(Canvas canvas, Size size) { + final center = Offset(size.width / 2, size.height / 2); + final radius = size.width / 2; + + // Background circle + final bgPaint = Paint() + ..color = const Color(0xFFFF9800).withOpacity(opacity * 0.2) + ..style = PaintingStyle.fill; + canvas.drawCircle(center, radius, bgPaint); + + // Progress arc + final progressPaint = Paint() + ..color = const Color(0xFFFF9800).withOpacity(opacity) + ..strokeWidth = 4 + ..style = PaintingStyle.stroke + ..strokeCap = StrokeCap.round; + + canvas.drawArc( + Rect.fromCircle(center: center, radius: radius - 2), + -pi / 2, + 2 * pi * progress, + false, + progressPaint, + ); + } + + @override + bool shouldRepaint(_LongPressPainter oldDelegate) => + progress != oldDelegate.progress; +} + +/// Text input indicator: glowing border +class _TextInputIndicator extends StatelessWidget { + final Rect bounds; + final Animation animation; + final IndicatorStyle style; + + const _TextInputIndicator({ + required this.bounds, + required this.animation, + required this.style, + }); + + @override + Widget build(BuildContext context) { + return AnimatedBuilder( + animation: animation, + builder: (context, child) { + final opacity = (sin(animation.value * pi * 4) + 1) / 2 * 0.5; + + return Positioned( + left: bounds.left, + top: bounds.top, + child: Container( + width: bounds.width, + height: bounds.height, + decoration: BoxDecoration( + border: Border.all( + color: const Color(0xFF4CAF50).withOpacity(opacity), + width: 2, + ), + borderRadius: BorderRadius.circular(4), + ), + ), + ); + }, + ); + } +} - findEditable(element); +/// Cursor indicator: mouse pointer/hand icon +class _CursorIndicator extends StatelessWidget { + final Offset position; + final Animation animation; + final IndicatorStyle style; + + const _CursorIndicator({ + required this.position, + required this.animation, + required this.style, + }); + + @override + Widget build(BuildContext context) { + return AnimatedBuilder( + animation: animation, + builder: (context, child) { + // Fade in quickly, stay visible, then fade out + final opacity = animation.value < 0.1 + ? animation.value / 0.1 + : animation.value > 0.9 + ? (1.0 - animation.value) / 0.1 + : 1.0; + + // Slight bounce effect + final scale = 1.0 + (sin(animation.value * pi * 2) * 0.1); + + return Positioned( + left: position.dx, + top: position.dy, + child: Transform.scale( + scale: scale, + child: Opacity( + opacity: opacity, + child: CustomPaint( + size: Size( + style == IndicatorStyle.minimal + ? 24 + : style == IndicatorStyle.standard + ? 32 + : 40, + style == IndicatorStyle.minimal + ? 24 + : style == IndicatorStyle.standard + ? 32 + : 40, + ), + painter: _CursorPainter(style: style), + ), + ), + ), + ); + }, + ); + } +} - return editableTextState?.textEditingValue.text; +/// Custom painter for cursor/pointer icon +class _CursorPainter extends CustomPainter { + final IndicatorStyle style; + + _CursorPainter({required this.style}); + + @override + void paint(Canvas canvas, Size size) { + final paint = Paint() + ..color = const Color(0xFF2196F3) + ..style = PaintingStyle.fill; + + final strokePaint = Paint() + ..color = Colors.white + ..style = PaintingStyle.stroke + ..strokeWidth = 2; + + // Draw hand/pointer cursor shape + final path = Path(); + + // Pointer arrow shape + path.moveTo(size.width * 0.2, size.height * 0.2); + path.lineTo(size.width * 0.2, size.height * 0.7); + path.lineTo(size.width * 0.35, size.height * 0.6); + path.lineTo(size.width * 0.5, size.height * 0.8); + path.lineTo(size.width * 0.6, size.height * 0.75); + path.lineTo(size.width * 0.45, size.height * 0.55); + path.lineTo(size.width * 0.7, size.height * 0.5); + path.close(); + + // Draw shadow/outline + canvas.drawPath(path, strokePaint); + // Draw main cursor + canvas.drawPath(path, paint); + + // Add small circle at tip for emphasis + final tipPaint = Paint() + ..color = const Color(0xFFFFEB3B) + ..style = PaintingStyle.fill; + canvas.drawCircle( + Offset(size.width * 0.2, size.height * 0.2), + size.width * 0.08, + tipPaint, + ); } - static bool? _getCheckboxState(String key) { - final element = _findElementByKey(key); - if (element == null) return null; + @override + bool shouldRepaint(_CursorPainter oldDelegate) => false; +} - final widget = element.widget; - if (widget is Checkbox) { - return widget.value; +/// Animated character indicator: game-like character with particle effects +class _AnimatedCharacterIndicator extends StatelessWidget { + final Offset position; + final Offset? endPosition; + final CharacterAction action; + final Animation animation; + final IndicatorStyle style; + + const _AnimatedCharacterIndicator({ + required this.position, + this.endPosition, + required this.action, + required this.animation, + required this.style, + }); + + @override + Widget build(BuildContext context) { + return AnimatedBuilder( + animation: animation, + builder: (context, child) { + // Smooth movement using game-like easing curves + // Phase 1 (0-0.3): Walking to target with elastic easing + // Phase 2 (0.3-0.6): Performing action with anticipation + // Phase 3 (0.6-1.0): Idle with gentle breathing + + double walkProgress; + if (animation.value < 0.3) { + // Walking phase - use elastic easing for bouncy movement + walkProgress = Curves.easeOutCubic.transform(animation.value / 0.3); + } else { + walkProgress = 1.0; + } + + final currentPos = endPosition != null + ? Offset.lerp(position, endPosition, walkProgress)! + : position; + + // Enhanced fade with glow effect + final opacity = animation.value < 0.03 + ? animation.value / 0.03 + : animation.value > 0.97 + ? (1.0 - animation.value) / 0.03 + : 1.0; + + // Determine current state based on animation progress + CharacterState currentState; + double actionProgress; + + if (animation.value < 0.3) { + // Walking phase + currentState = CharacterState.walking; + actionProgress = animation.value / 0.3; + } else if (animation.value < 0.6) { + // Action phase - with anticipation and impact + currentState = CharacterState.acting; + actionProgress = + Curves.easeOutBack.transform((animation.value - 0.3) / 0.3); + } else { + // Idle phase - gentle breathing animation + currentState = CharacterState.idle; + actionProgress = (animation.value - 0.6) / 0.4; + } + + return Positioned( + left: currentPos.dx - 40, + top: currentPos.dy - 80, + child: Opacity( + opacity: opacity, + child: Stack( + children: [ + // Particle effects layer + CustomPaint( + size: const Size(80, 80), + painter: _ParticleEffectPainter( + action: action, + progress: actionProgress, + state: currentState, + globalProgress: animation.value, + ), + ), + // Character layer with shadow + CustomPaint( + size: const Size(80, 80), + painter: _GameCharacterPainter( + action: action, + progress: actionProgress, + state: currentState, + style: style, + globalProgress: animation.value, + ), + ), + ], + ), + ), + ); + }, + ); + } +} + +/// Particle effect painter for game-like visual effects +class _ParticleEffectPainter extends CustomPainter { + final CharacterAction action; + final double progress; + final CharacterState state; + final double globalProgress; + + _ParticleEffectPainter({ + required this.action, + required this.progress, + required this.state, + required this.globalProgress, + }); + + @override + void paint(Canvas canvas, Size size) { + final cx = size.width / 2; + final cy = size.height / 2; + + // Draw particles based on state and action + if (state == CharacterState.walking) { + _drawWalkingDust(canvas, cx, cy); + } else if (state == CharacterState.acting) { + _drawActionParticles(canvas, cx, cy); } - if (widget is Switch) { - return widget.value; + } + + void _drawWalkingDust(Canvas canvas, double cx, double cy) { + final dustPaint = Paint() + ..color = Colors.grey.withOpacity(0.3 * (1 - progress)) + ..style = PaintingStyle.fill; + + // Dust clouds behind character + for (var i = 0; i < 3; i++) { + final offset = i * 8.0; + final size = 4.0 - i; + canvas.drawCircle( + Offset(cx - offset, cy + 30 + i * 2), + size * (1 - progress), + dustPaint, + ); } - return null; } - static double? _getSliderValue(String key) { - final element = _findElementByKey(key); - if (element == null) return null; + void _drawActionParticles(Canvas canvas, double cx, double cy) { + final particlePaint = Paint() + ..style = PaintingStyle.fill + ..maskFilter = const MaskFilter.blur(BlurStyle.normal, 2); + + // Sparkle particles during action + for (var i = 0; i < 6; i++) { + final angle = (i / 6) * 2 * pi + progress * pi; + final distance = 20 + progress * 15; + final x = cx + cos(angle) * distance; + final y = cy + sin(angle) * distance; + + // Color gradient from yellow to orange + final color = Color.lerp( + const Color(0xFFFFEB3B), + const Color(0xFFFF5722), + i / 6, + )! + .withValues(alpha: (0.8 * (1 - progress)).clamp(0.0, 1.0)); + + particlePaint.color = color; + canvas.drawCircle(Offset(x, y), 3 * (1 - progress), particlePaint); + } - final widget = element.widget; - if (widget is Slider) { - return widget.value; + // Impact effect for tapping + if (action == CharacterAction.tapping && progress > 0.5) { + final impactProgress = (progress - 0.5) * 2; + final impactPaint = Paint() + ..color = + const Color(0xFFFF5722).withOpacity(0.5 * (1 - impactProgress)) + ..style = PaintingStyle.stroke + ..strokeWidth = 3; + + canvas.drawCircle( + Offset(cx + 20, cy), + 10 + impactProgress * 15, + impactPaint, + ); } - return null; } - static Future _waitForElement( - {String? key, String? text, int timeout = 5000}) async { - final endTime = DateTime.now().add(Duration(milliseconds: timeout)); + @override + bool shouldRepaint(_ParticleEffectPainter oldDelegate) => true; +} - while (DateTime.now().isBefore(endTime)) { - final element = _findElement(key: key, text: text); - if (element != null) return true; - await Future.delayed(const Duration(milliseconds: 100)); +/// Game-like character painter with shadows and smooth animations +class _GameCharacterPainter extends CustomPainter { + final CharacterAction action; + final double progress; + final IndicatorStyle style; + final CharacterState state; + final double globalProgress; + + _GameCharacterPainter({ + required this.action, + required this.progress, + required this.style, + required this.state, + required this.globalProgress, + }); + + @override + void paint(Canvas canvas, Size size) { + final centerX = size.width / 2; + final centerY = size.height / 2; + + // Draw shadow first (for depth) + final shadowPaint = Paint() + ..color = Colors.black.withOpacity(0.3) + ..maskFilter = const MaskFilter.blur(BlurStyle.normal, 4); + + // Elliptical shadow on ground + canvas.drawOval( + Rect.fromCenter( + center: Offset(centerX, centerY + 35), + width: 30, + height: 8, + ), + shadowPaint, + ); + + // Game-like character colors with gradients + final bodyPaint = Paint() + ..color = const Color(0xFF4CAF50) + ..style = PaintingStyle.stroke + ..strokeWidth = style == IndicatorStyle.minimal ? 4 : 5 + ..strokeCap = StrokeCap.round; + + final headPaint = Paint() + ..color = const Color(0xFFFFEB3B) + ..style = PaintingStyle.fill; + + // Head glow effect + final glowPaint = Paint() + ..color = const Color(0xFFFFEB3B).withOpacity(0.3) + ..maskFilter = const MaskFilter.blur(BlurStyle.normal, 6); + + final accentPaint = Paint() + ..color = const Color(0xFFFF5722) + ..style = PaintingStyle.fill; + + // Breathing animation in idle state + final breatheOffset = + state == CharacterState.idle ? sin(globalProgress * pi * 3) * 1.5 : 0.0; + + // Draw glow around head for added depth + if (style != IndicatorStyle.minimal) { + canvas.drawCircle( + Offset(centerX, centerY - 15 + breatheOffset), + 12, + glowPaint, + ); } - return false; + // Draw based on current state + switch (state) { + case CharacterState.walking: + _drawWalkingCharacter( + canvas, size, centerX, centerY, bodyPaint, headPaint, accentPaint); + break; + case CharacterState.acting: + // Draw action-specific animation + switch (action) { + case CharacterAction.tapping: + _drawTappingCharacter(canvas, size, centerX, centerY, bodyPaint, + headPaint, accentPaint); + break; + case CharacterAction.typing: + _drawTypingCharacter(canvas, size, centerX, centerY, bodyPaint, + headPaint, accentPaint); + break; + case CharacterAction.swiping: + _drawSwipingCharacter(canvas, size, centerX, centerY, bodyPaint, + headPaint, accentPaint); + break; + case CharacterAction.holding: + _drawHoldingCharacter(canvas, size, centerX, centerY, bodyPaint, + headPaint, accentPaint); + break; + case CharacterAction.dragging: + _drawDraggingCharacter(canvas, size, centerX, centerY, bodyPaint, + headPaint, accentPaint); + break; + case CharacterAction.pointing: + _drawPointingCharacter(canvas, size, centerX, centerY, bodyPaint, + headPaint, accentPaint); + break; + } + break; + case CharacterState.idle: + _drawIdleCharacter( + canvas, size, centerX, centerY, bodyPaint, headPaint, accentPaint); + break; + } } - static Future _waitForGone( - {String? key, String? text, int timeout = 5000}) async { - final endTime = DateTime.now().add(Duration(milliseconds: timeout)); + void _drawTappingCharacter(Canvas canvas, Size size, double cx, double cy, + Paint body, Paint head, Paint accent) { + // Animated bounce effect + final bounce = sin(progress * pi * 4) * 3; + + // Head + canvas.drawCircle(Offset(cx, cy - 15 + bounce), 8, head); + + // Eyes + final eyePaint = Paint() + ..color = Colors.black + ..style = PaintingStyle.fill; + canvas.drawCircle(Offset(cx - 3, cy - 16 + bounce), 1.5, eyePaint); + canvas.drawCircle(Offset(cx + 3, cy - 16 + bounce), 1.5, eyePaint); + + // Body + canvas.drawLine( + Offset(cx, cy - 7 + bounce), + Offset(cx, cy + 10 + bounce), + body, + ); - while (DateTime.now().isBefore(endTime)) { - final element = _findElement(key: key, text: text); - if (element == null) return true; - await Future.delayed(const Duration(milliseconds: 100)); - } + // Arms - tapping motion + final armAngle = sin(progress * pi * 8) * 0.3; + canvas.drawLine( + Offset(cx, cy), + Offset(cx - 12, cy + 8 + bounce + armAngle * 5), + body, + ); + canvas.drawLine( + Offset(cx, cy), + Offset(cx + 12, cy + 8 + bounce - armAngle * 5), + body, + ); - return false; - } + // Tapping indicator (finger point) + canvas.drawCircle( + Offset(cx + 12, cy + 8 + bounce - armAngle * 5), 3, accent); - // ==================== SCREENSHOT ==================== + // Legs + canvas.drawLine( + Offset(cx, cy + 10 + bounce), + Offset(cx - 8, cy + 22 + bounce), + body, + ); + canvas.drawLine( + Offset(cx, cy + 10 + bounce), + Offset(cx + 8, cy + 22 + bounce), + body, + ); + } - static Future _takeScreenshot() async { - try { - final binding = WidgetsBinding.instance; - // ignore: invalid_use_of_protected_member - final renderObject = binding.rootElement?.renderObject; - if (renderObject is! RenderRepaintBoundary) { - // Try to find a RenderRepaintBoundary - RenderRepaintBoundary? boundary; - void findBoundary(RenderObject obj) { - if (boundary != null) return; - if (obj is RenderRepaintBoundary) { - boundary = obj; - return; - } - obj.visitChildren(findBoundary); - } + void _drawTypingCharacter(Canvas canvas, Size size, double cx, double cy, + Paint body, Paint head, Paint accent) { + // Head + canvas.drawCircle(Offset(cx, cy - 12), 8, head); + + // Eyes looking down + final eyePaint = Paint() + ..color = Colors.black + ..style = PaintingStyle.fill; + canvas.drawCircle(Offset(cx - 3, cy - 10), 1.5, eyePaint); + canvas.drawCircle(Offset(cx + 3, cy - 10), 1.5, eyePaint); + + // Body + canvas.drawLine( + Offset(cx, cy - 4), + Offset(cx, cy + 12), + body, + ); - renderObject?.visitChildren(findBoundary); + // Arms - typing motion + final armWave = sin(progress * pi * 12) * 3; + canvas.drawLine( + Offset(cx, cy + 2), + Offset(cx - 10, cy + 14 + armWave), + body, + ); + canvas.drawLine( + Offset(cx, cy + 2), + Offset(cx + 10, cy + 14 - armWave), + body, + ); - if (boundary == null) { - _log('No RenderRepaintBoundary found'); - return null; - } + // Keyboard representation + final keyboardPaint = Paint() + ..color = const Color(0xFF9E9E9E) + ..style = PaintingStyle.stroke + ..strokeWidth = 2; + canvas.drawRect( + Rect.fromCenter(center: Offset(cx, cy + 22), width: 24, height: 4), + keyboardPaint, + ); - final image = await boundary!.toImage(pixelRatio: 1.0); - final byteData = await image.toByteData(format: ui.ImageByteFormat.png); - if (byteData == null) return null; - return base64Encode(byteData.buffer.asUint8List()); - } + // Typing indicators (small dots) + canvas.drawCircle(Offset(cx - 6 + armWave, cy + 14 + armWave), 2, accent); + canvas.drawCircle(Offset(cx + 6 - armWave, cy + 14 - armWave), 2, accent); - final image = await renderObject.toImage(pixelRatio: 1.0); - final byteData = await image.toByteData(format: ui.ImageByteFormat.png); - if (byteData == null) return null; - return base64Encode(byteData.buffer.asUint8List()); - } catch (e) { - _log('Screenshot failed: $e'); - return null; - } + // Legs + canvas.drawLine( + Offset(cx, cy + 12), + Offset(cx - 6, cy + 20), + body, + ); + canvas.drawLine( + Offset(cx, cy + 12), + Offset(cx + 6, cy + 20), + body, + ); } - static Future _takeElementScreenshot(String key) async { - try { - final element = _findElementByKey(key); - if (element == null) { - _log('Element not found: $key'); - return null; - } + void _drawSwipingCharacter(Canvas canvas, Size size, double cx, double cy, + Paint body, Paint head, Paint accent) { + // Leaning forward with swipe motion + final swipeAngle = progress * pi * 2; + final armX = cos(swipeAngle) * 15; + final armY = sin(swipeAngle) * 8; + + // Head + canvas.drawCircle(Offset(cx + armX * 0.3, cy - 12), 8, head); + + // Eyes + final eyePaint = Paint() + ..color = Colors.black + ..style = PaintingStyle.fill; + canvas.drawCircle(Offset(cx + armX * 0.3 - 3, cy - 13), 1.5, eyePaint); + canvas.drawCircle(Offset(cx + armX * 0.3 + 3, cy - 13), 1.5, eyePaint); + + // Body leaning + canvas.drawLine( + Offset(cx + armX * 0.3, cy - 4), + Offset(cx, cy + 10), + body, + ); - final renderObject = element.renderObject; - if (renderObject == null) { - _log('No render object for element: $key'); - return null; - } + // Swiping arm + canvas.drawLine( + Offset(cx, cy), + Offset(cx + armX, cy + armY), + body, + ); - // If it's a RenderRepaintBoundary, capture directly - if (renderObject is RenderRepaintBoundary) { - final image = await renderObject.toImage(pixelRatio: 1.0); - final byteData = await image.toByteData(format: ui.ImageByteFormat.png); - if (byteData == null) return null; - return base64Encode(byteData.buffer.asUint8List()); - } + // Swipe trail + final trailPaint = Paint() + ..color = const Color(0xFF9C27B0).withOpacity(0.5) + ..style = PaintingStyle.stroke + ..strokeWidth = 3; + canvas.drawLine( + Offset(cx + armX - 10, cy + armY), + Offset(cx + armX + 10, cy + armY), + trailPaint, + ); - // For other render objects, find the nearest RepaintBoundary ancestor - RenderObject? current = renderObject; - while (current != null) { - if (current is RenderRepaintBoundary) { - // Get the element's bounds relative to the boundary - if (renderObject is RenderBox) { - final box = renderObject; - final boundaryBox = current; + // Hand indicator + canvas.drawCircle(Offset(cx + armX, cy + armY), 3, accent); - // Get element position relative to boundary - final offset = - box.localToGlobal(Offset.zero, ancestor: boundaryBox); - final size = box.size; + // Other arm + canvas.drawLine( + Offset(cx, cy), + Offset(cx - 10, cy + 8), + body, + ); - // Capture the boundary - final fullImage = await current.toImage(pixelRatio: 1.0); + // Legs + canvas.drawLine( + Offset(cx, cy + 10), + Offset(cx - 8, cy + 22), + body, + ); + canvas.drawLine( + Offset(cx, cy + 10), + Offset(cx + 8, cy + 22), + body, + ); + } - // Crop to element bounds - final recorder = ui.PictureRecorder(); - final canvas = Canvas(recorder); + void _drawHoldingCharacter(Canvas canvas, Size size, double cx, double cy, + Paint body, Paint head, Paint accent) { + // Concentrated holding pose + final pulse = 1.0 + sin(progress * pi * 6) * 0.1; + + // Head + canvas.drawCircle(Offset(cx, cy - 12), 8 * pulse, head); + + // Focused eyes + final eyePaint = Paint() + ..color = Colors.black + ..style = PaintingStyle.fill; + canvas.drawCircle(Offset(cx - 3, cy - 13), 2, eyePaint); + canvas.drawCircle(Offset(cx + 3, cy - 13), 2, eyePaint); + + // Body + canvas.drawLine( + Offset(cx, cy - 4), + Offset(cx, cy + 10), + body, + ); - // Draw the cropped portion - canvas.drawImageRect( - fullImage, - Rect.fromLTWH(offset.dx, offset.dy, size.width, size.height), - Rect.fromLTWH(0, 0, size.width, size.height), - Paint(), - ); + // Arms holding/pressing + canvas.drawLine( + Offset(cx, cy), + Offset(cx - 10, cy + 10), + body, + ); + canvas.drawLine( + Offset(cx, cy), + Offset(cx + 10, cy + 10), + body, + ); - final picture = recorder.endRecording(); - final croppedImage = - await picture.toImage(size.width.toInt(), size.height.toInt()); - final byteData = - await croppedImage.toByteData(format: ui.ImageByteFormat.png); - if (byteData == null) return null; - return base64Encode(byteData.buffer.asUint8List()); - } - } - current = current.parent; - } + // Pressure indicator + canvas.drawCircle(Offset(cx, cy + 14), 6 * pulse, accent); - _log('No suitable RenderRepaintBoundary ancestor found for: $key'); - return null; - } catch (e) { - _log('Element screenshot failed: $e'); - return null; - } + // Legs + canvas.drawLine( + Offset(cx, cy + 10), + Offset(cx - 6, cy + 20), + body, + ); + canvas.drawLine( + Offset(cx, cy + 10), + Offset(cx + 6, cy + 20), + body, + ); } - // ==================== NAVIGATION ==================== + void _drawDraggingCharacter(Canvas canvas, Size size, double cx, double cy, + Paint body, Paint head, Paint accent) { + // Pulling/dragging motion + final dragX = sin(progress * pi) * 5; + + // Head + canvas.drawCircle(Offset(cx - dragX, cy - 12), 8, head); + + // Determined eyes + final eyePaint = Paint() + ..color = Colors.black + ..style = PaintingStyle.fill; + canvas.drawCircle(Offset(cx - dragX - 3, cy - 13), 1.5, eyePaint); + canvas.drawCircle(Offset(cx - dragX + 3, cy - 13), 1.5, eyePaint); + + // Body leaning back + canvas.drawLine( + Offset(cx - dragX, cy - 4), + Offset(cx, cy + 10), + body, + ); - static String? _getCurrentRoute() { - String? currentRoute; - final binding = WidgetsBinding.instance; + // Arms pulling + canvas.drawLine( + Offset(cx, cy), + Offset(cx + 12 + dragX, cy + 2), + body, + ); + canvas.drawLine( + Offset(cx, cy + 4), + Offset(cx + 12 + dragX, cy + 6), + body, + ); - void visit(Element element) { - if (element.widget is ModalRoute) { - final route = ModalRoute.of(element); - if (route != null) { - currentRoute = route.settings.name; - } - } - element.visitChildren(visit); - } + // Object being dragged + final objectPaint = Paint() + ..color = const Color(0xFF2196F3) + ..style = PaintingStyle.fill; + canvas.drawRect( + Rect.fromCenter( + center: Offset(cx + 16 + dragX, cy + 4), + width: 8, + height: 8, + ), + objectPaint, + ); - // ignore: invalid_use_of_protected_member - if (binding.rootElement != null) { - visit(binding.rootElement!); - } + // Drag trail + final trailPaint = Paint() + ..color = const Color(0xFF9C27B0).withOpacity(0.3) + ..style = PaintingStyle.stroke + ..strokeWidth = 2 + ..strokeCap = StrokeCap.round; + canvas.drawLine( + Offset(cx + 20 + dragX, cy + 4), + Offset(cx + 28, cy + 4), + trailPaint, + ); - return currentRoute; + // Legs bracing + canvas.drawLine( + Offset(cx, cy + 10), + Offset(cx - 10, cy + 22), + body, + ); + canvas.drawLine( + Offset(cx, cy + 10), + Offset(cx + 6, cy + 22), + body, + ); } - static bool _goBack() { - final context = _findNavigatorContext(); - if (context == null) return false; + void _drawPointingCharacter(Canvas canvas, Size size, double cx, double cy, + Paint body, Paint head, Paint accent) { + // Simple pointing pose + // Head + canvas.drawCircle(Offset(cx, cy - 12), 8, head); + + // Eyes + final eyePaint = Paint() + ..color = Colors.black + ..style = PaintingStyle.fill; + canvas.drawCircle(Offset(cx - 3, cy - 13), 1.5, eyePaint); + canvas.drawCircle(Offset(cx + 3, cy - 13), 1.5, eyePaint); + + // Smile + final smilePaint = Paint() + ..color = Colors.black + ..style = PaintingStyle.stroke + ..strokeWidth = 1; + final smilePath = Path(); + smilePath.moveTo(cx - 3, cy - 10); + smilePath.quadraticBezierTo(cx, cy - 8, cx + 3, cy - 10); + canvas.drawPath(smilePath, smilePaint); + + // Body + canvas.drawLine( + Offset(cx, cy - 4), + Offset(cx, cy + 10), + body, + ); - final navigator = Navigator.of(context, rootNavigator: false); - if (navigator.canPop()) { - navigator.pop(); - return true; - } - return false; - } + // Pointing arm + canvas.drawLine( + Offset(cx, cy), + Offset(cx + 14, cy + 8), + body, + ); - static List _getNavigationStack() { - final routes = []; - final context = _findNavigatorContext(); - if (context == null) return routes; + // Pointing finger + canvas.drawCircle(Offset(cx + 14, cy + 8), 3, accent); - // This is a simplified version - full implementation would need NavigatorState access - final currentRoute = _getCurrentRoute(); - if (currentRoute != null) { - routes.add(currentRoute); - } + // Other arm + canvas.drawLine( + Offset(cx, cy), + Offset(cx - 10, cy + 6), + body, + ); - return routes; + // Legs + canvas.drawLine( + Offset(cx, cy + 10), + Offset(cx - 6, cy + 22), + body, + ); + canvas.drawLine( + Offset(cx, cy + 10), + Offset(cx + 6, cy + 22), + body, + ); } - static BuildContext? _findNavigatorContext() { - BuildContext? context; - final binding = WidgetsBinding.instance; + void _drawWalkingCharacter(Canvas canvas, Size size, double cx, double cy, + Paint body, Paint head, Paint accent) { + // Walking animation with alternating legs + final walkCycle = + progress * pi * 4; // Multiple walking cycles during the walk + final bobbing = sin(walkCycle) * 2; // Vertical bobbing motion + final leftLegSwing = sin(walkCycle) * 12; + final rightLegSwing = sin(walkCycle + pi) * 12; + final armSwing = sin(walkCycle) * 8; + + // Head with bobbing + canvas.drawCircle(Offset(cx, cy - 12 + bobbing), 8, head); + + // Eyes (determined look) + final eyePaint = Paint() + ..color = Colors.black + ..style = PaintingStyle.fill; + canvas.drawCircle(Offset(cx - 3, cy - 13 + bobbing), 1.5, eyePaint); + canvas.drawCircle(Offset(cx + 3, cy - 13 + bobbing), 1.5, eyePaint); + + // Body with bobbing + canvas.drawLine( + Offset(cx, cy - 4 + bobbing), + Offset(cx, cy + 10 + bobbing), + body, + ); - void visit(Element element) { - if (context != null) return; - if (element.widget is Navigator) { - context = element; - return; - } - element.visitChildren(visit); - } + // Arms swinging + canvas.drawLine( + Offset(cx, cy + bobbing), + Offset(cx - 10 + armSwing, cy + 8 + bobbing), + body, + ); + canvas.drawLine( + Offset(cx, cy + bobbing), + Offset(cx + 10 - armSwing, cy + 8 + bobbing), + body, + ); - // ignore: invalid_use_of_protected_member - if (binding.rootElement != null) { - visit(binding.rootElement!); - } + // Legs alternating (walking motion) + canvas.drawLine( + Offset(cx, cy + 10 + bobbing), + Offset(cx - 6 + leftLegSwing * 0.5, cy + 22), + body, + ); + canvas.drawLine( + Offset(cx, cy + 10 + bobbing), + Offset(cx + 6 + rightLegSwing * 0.5, cy + 22), + body, + ); - return context; + // Walking dust/motion lines for extra effect + if (style == IndicatorStyle.detailed) { + final dustPaint = Paint() + ..color = const Color(0xFF9E9E9E).withOpacity(0.4) + ..style = PaintingStyle.stroke + ..strokeWidth = 2 + ..strokeCap = StrokeCap.round; + + // Small motion lines behind character + canvas.drawLine( + Offset(cx - 8, cy + 20), + Offset(cx - 12, cy + 22), + dustPaint, + ); + canvas.drawLine( + Offset(cx - 6, cy + 18), + Offset(cx - 10, cy + 20), + dustPaint, + ); + } } - // ==================== DEBUG & LOGS ==================== + void _drawIdleCharacter(Canvas canvas, Size size, double cx, double cy, + Paint body, Paint head, Paint accent) { + // Idle animation with subtle breathing + final breathing = sin(progress * pi * 2) * 1.5; // Slow breathing motion + final blink = + (progress * 20) % 1.0 > 0.9 ? 0.5 : 1.5; // Occasional blinking + + // Head with subtle breathing + canvas.drawCircle(Offset(cx, cy - 12 + breathing * 0.3), 8, head); + + // Eyes (blinking) + final eyePaint = Paint() + ..color = Colors.black + ..style = PaintingStyle.fill; + canvas.drawCircle( + Offset(cx - 3, cy - 13 + breathing * 0.3), blink, eyePaint); + canvas.drawCircle( + Offset(cx + 3, cy - 13 + breathing * 0.3), blink, eyePaint); + + // Smile + final smilePaint = Paint() + ..color = Colors.black + ..style = PaintingStyle.stroke + ..strokeWidth = 1; + final smilePath = Path(); + smilePath.moveTo(cx - 3, cy - 10 + breathing * 0.3); + smilePath.quadraticBezierTo( + cx, cy - 8 + breathing * 0.3, cx + 3, cy - 10 + breathing * 0.3); + canvas.drawPath(smilePath, smilePaint); + + // Body with breathing + canvas.drawLine( + Offset(cx, cy - 4 + breathing * 0.3), + Offset(cx, cy + 10 + breathing), + body, + ); - static void _log(String message) { - final logEntry = '[${DateTime.now().toIso8601String()}] $message'; - _logs.add(logEntry); - print('Flutter Skill: $message'); - } + // Arms relaxed at sides + canvas.drawLine( + Offset(cx, cy + breathing * 0.5), + Offset(cx - 8, cy + 10 + breathing), + body, + ); + canvas.drawLine( + Offset(cx, cy + breathing * 0.5), + Offset(cx + 8, cy + 10 + breathing), + body, + ); - static Map _getPerformanceMetrics() { - return { - 'timestamp': DateTime.now().toIso8601String(), - 'logCount': _logs.length, - 'errorCount': _errors.length, - }; + // Legs standing still + canvas.drawLine( + Offset(cx, cy + 10 + breathing), + Offset(cx - 6, cy + 22), + body, + ); + canvas.drawLine( + Offset(cx, cy + 10 + breathing), + Offset(cx + 6, cy + 22), + body, + ); } - // ==================== HELPERS ==================== - - static String? _extractTextFrom(Element element) { - String? found; - void visit(Element e) { - if (found != null) return; - if (e.widget is Text) { - found = (e.widget as Text).data; - } else if (e.widget is RichText) { - found = (e.widget as RichText).text.toPlainText(); - } - e.visitChildren(visit); - } + @override + bool shouldRepaint(_GameCharacterPainter oldDelegate) => + progress != oldDelegate.progress || + action != oldDelegate.action || + state != oldDelegate.state || + globalProgress != oldDelegate.globalProgress; +} - visit(element); - return found; +/// Action hint banner +class _ActionHint extends StatelessWidget { + final String message; + final Animation animation; + final IndicatorStyle style; + + const _ActionHint({ + required this.message, + required this.animation, + required this.style, + }); + + @override + Widget build(BuildContext context) { + return AnimatedBuilder( + animation: animation, + builder: (context, child) { + // Slide in from top, then fade out + final slideProgress = + animation.value < 0.2 ? animation.value / 0.2 : 1.0; + final fadeProgress = + animation.value > 0.8 ? (1.0 - animation.value) / 0.2 : 1.0; + final opacity = fadeProgress * 0.95; + + return Transform.translate( + offset: Offset(0, -20 * (1 - slideProgress)), + child: Opacity( + opacity: opacity, + child: Material( + color: Colors.transparent, + child: Container( + padding: + const EdgeInsets.symmetric(horizontal: 16, vertical: 8), + decoration: BoxDecoration( + color: Colors.black.withOpacity(0.8), + borderRadius: BorderRadius.circular(8), + border: Border.all( + color: const Color(0xFF4CAF50).withOpacity(0.5), + width: 1, + ), + ), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + const Icon( + Icons.touch_app, + color: Color(0xFF4CAF50), + size: 16, + ), + const SizedBox(width: 8), + Flexible( + child: Text( + message, + style: const TextStyle( + color: Colors.white, + fontSize: 14, + fontWeight: FontWeight.w500, + ), + overflow: TextOverflow.ellipsis, + ), + ), + ], + ), + ), + ), + ), + ); + }, + ); } } + +/// PNG screenshot payload with its pixel dimensions. +class _Screenshot { + const _Screenshot(this.base64, this.width, this.height); + final String base64; + final int width; + final int height; +} diff --git a/packaging/npm/dart/lib/flutter_skill_semantic_refs.dart b/packaging/npm/dart/lib/flutter_skill_semantic_refs.dart new file mode 100644 index 00000000..77859979 --- /dev/null +++ b/packaging/npm/dart/lib/flutter_skill_semantic_refs.dart @@ -0,0 +1,370 @@ +import 'package:flutter/material.dart'; +import 'package:flutter/cupertino.dart'; + +/// Semantic fingerprint system for reliable element referencing. +/// Generates human-readable ref IDs in the format {role}:{content}[{index}] +class SemanticRefGenerator { + // Element cache for performance optimization + static final Map> _elementCache = {}; + + /// Clear the element cache + static void clearCache() { + _elementCache.clear(); + } + + /// Generate semantic ref ID for an element + static String generateRefId(Element element, Map refCounts) { + final widget = element.widget; + + // Determine semantic role + final role = _getSemanticRole(widget); + + // Extract content with priority: key > text/label/hint > tooltip > fallback + String? content = _extractContent(element, widget); + + // Clean and format content + if (content != null && content.isNotEmpty) { + content = _sanitizeContent(content); + + final baseRef = '$role:$content'; + + // Check for duplicates and add index if needed + final count = refCounts[baseRef] ?? 0; + refCounts[baseRef] = count + 1; + + if (count == 0) { + return baseRef; + } else { + return '${baseRef}[$count]'; + } + } else { + // No content - use role + index fallback + final count = refCounts[role] ?? 0; + refCounts[role] = count + 1; + return '${role}[$count]'; + } + } + + /// Determine semantic role for a widget + static String _getSemanticRole(Widget widget) { + // Button family - all button-like widgets + if (widget is ElevatedButton || + widget is TextButton || + widget is OutlinedButton || + widget is IconButton || + widget is FloatingActionButton || + widget is CupertinoButton) { + return 'button'; + } + + // Input family - all text input widgets + if (widget is TextField || + widget is TextFormField || + widget is CupertinoTextField || + widget is EditableText) { + return 'input'; + } + + // Toggle family - switches and checkboxes + if (widget is Switch || + widget is Checkbox || + widget is Radio || + widget is CupertinoSwitch || + (widget is ToggleButtons)) { + return 'toggle'; + } + + // Slider family + if (widget is Slider || + widget is RangeSlider || + widget is CupertinoSlider) { + return 'slider'; + } + + // Select family - dropdowns and pickers + if (widget is DropdownButton || + widget is PopupMenuButton || + widget is CupertinoActionSheet || + widget is CupertinoPicker) { + return 'select'; + } + + // Link family - tappable text or links + if ((widget is InkWell && _hasText(widget)) || + (widget is GestureDetector && + widget.onTap != null && + _hasTextInChildren(widget)) || + (widget is TextButton && widget.child is Text)) { + return 'link'; + } + + // Item family - list items + if (widget is ListTile || widget is Card || widget is ExpansionTile) { + return 'item'; + } + + // Fallback for other interactive widgets + return 'element'; + } + + /// Extract content from element with priority + static String? _extractContent(Element element, Widget widget) { + // Priority 1: ValueKey - most stable + if (widget.key is ValueKey) { + return (widget.key as ValueKey).value; + } + + // Priority 2: Widget-specific text/label/hint + String? content = _getWidgetSpecificContent(widget); + if (content != null && content.isNotEmpty) { + return content; + } + + // Priority 3: Extract text from child widgets + content = _extractTextFromChildren(element); + if (content != null && content.isNotEmpty) { + return content; + } + + // Priority 4: Tooltip + if (widget is Tooltip) { + return widget.message; + } + + // Priority 5: Semantic label + content = _extractSemanticLabel(element); + if (content != null && content.isNotEmpty) { + return content; + } + + return null; + } + + /// Get widget-specific content (text, label, hint, etc.) + static String? _getWidgetSpecificContent(Widget widget) { + // TextField family + if (widget is TextField) { + return widget.decoration?.labelText ?? + widget.decoration?.hintText ?? + widget.decoration?.helperText; + } + + if (widget is TextFormField) { + // TextFormField wraps a TextField, so we need to access it differently + // This is a simplified approach - in practice we'd need to traverse the element tree + return 'Text Field'; // Fallback for TextFormField + } + + // Button family + if (widget is ElevatedButton && widget.child is Text) { + return (widget.child as Text).data; + } + + if (widget is TextButton && widget.child is Text) { + return (widget.child as Text).data; + } + + if (widget is OutlinedButton && widget.child is Text) { + return (widget.child as Text).data; + } + + // Direct text widgets + if (widget is Text) { + return widget.data; + } + + if (widget is RichText) { + return widget.text.toPlainText(); + } + + // ListTile + if (widget is ListTile) { + if (widget.title is Text) { + return (widget.title as Text).data; + } + } + + return null; + } + + /// Extract text from child elements + static String? _extractTextFromChildren(Element element) { + String? foundText; + + void visit(Element child) { + if (foundText != null) return; + + final widget = child.widget; + if (widget is Text && widget.data != null) { + foundText = widget.data; + return; + } + + if (widget is RichText) { + foundText = widget.text.toPlainText(); + return; + } + + child.visitChildren(visit); + } + + element.visitChildren(visit); + return foundText; + } + + /// Extract semantic label from element + static String? _extractSemanticLabel(Element element) { + final widget = element.widget; + if (widget is Semantics && widget.properties.label != null) { + return widget.properties.label; + } + return null; + } + + /// Sanitize content for use in ref IDs + static String _sanitizeContent(String content) { + // Replace spaces with underscores + String sanitized = content.replaceAll(' ', '_'); + + // Remove special characters, keep only alphanumeric and underscores + sanitized = sanitized.replaceAll(RegExp(r'[^\w]'), ''); + + // Truncate if too long + if (sanitized.length > 30) { + sanitized = '${sanitized.substring(0, 27)}...'; + } + + return sanitized; + } + + /// Check if widget has text property + static bool _hasText(Widget widget) { + if (widget is InkWell) { + return widget.child is Text; + } + return false; + } + + /// Check if widget has text in its children (for GestureDetector) + static bool _hasTextInChildren(Widget widget) { + // This is a simplified check - in practice, we'd need to traverse children + return true; // Conservative approach + } + + /// Cache element data for performance + static void cacheElement( + String refId, Element element, Map bounds) { + final weakRef = WeakReference(element); + _elementCache[refId] = { + 'bounds': bounds, + 'element': weakRef, + 'timestamp': DateTime.now().millisecondsSinceEpoch, + }; + + // Clean up old cache entries + _cleanupCache(); + } + + /// Get cached element data + static Map? getCachedElement(String refId) { + final cached = _elementCache[refId]; + if (cached == null) return null; + + // Check if element is still valid + final weakRef = cached['element'] as WeakReference?; + if (weakRef?.target == null) { + _elementCache.remove(refId); + return null; + } + + return cached; + } + + /// Clean up stale cache entries + static void _cleanupCache() { + const maxCacheSize = 100; + const maxCacheAge = 30000; // 30 seconds + + final now = DateTime.now().millisecondsSinceEpoch; + + // Remove stale entries + _elementCache.removeWhere((key, value) { + final timestamp = value['timestamp'] as int? ?? 0; + final isStale = now - timestamp > maxCacheAge; + + final weakRef = value['element'] as WeakReference?; + final isValid = weakRef?.target != null; + + return isStale || !isValid; + }); + + // If still too many entries, remove oldest + if (_elementCache.length > maxCacheSize) { + final entries = _elementCache.entries.toList(); + entries.sort((a, b) { + final aTime = a.value['timestamp'] as int? ?? 0; + final bTime = b.value['timestamp'] as int? ?? 0; + return aTime.compareTo(bTime); + }); + + // Remove oldest 20% + final toRemove = (entries.length * 0.2).round(); + for (int i = 0; i < toRemove; i++) { + _elementCache.remove(entries[i].key); + } + } + } + + /// Backward compatibility: check if ref looks like old format + static bool isLegacyRef(String ref) { + // Old format: btn_0, tf_1, sw_2, etc. + // NOTE: elem_NNN is NOT a legacy ref — it's a numeric element ID from + // get_interactable_elements and must NOT be handled by legacy parsing. + if (ref.startsWith('elem_')) return false; + return RegExp(r'^[a-z]+_\d+$').hasMatch(ref); + } + + /// Convert legacy ref to search parameters for backward compatibility + static Map? parseLegacyRef(String ref) { + if (!isLegacyRef(ref)) return null; + + final parts = ref.split('_'); + if (parts.length != 2) return null; + + final prefix = parts[0]; + final index = int.tryParse(parts[1]); + if (index == null) return null; + + // Map old prefixes to new roles for search fallback + String? role; + switch (prefix) { + case 'btn': + role = 'button'; + break; + case 'tf': + role = 'input'; + break; + case 'sw': + role = 'toggle'; + break; + case 'sl': + role = 'slider'; + break; + case 'dd': + role = 'select'; + break; + case 'lnk': + role = 'link'; + break; + case 'item': + role = 'item'; + break; + } + + return { + 'role': role, + 'index': index, + 'legacy': true, + }; + } +} diff --git a/packaging/npm/dart/lib/flutter_skill_web_interop.dart b/packaging/npm/dart/lib/flutter_skill_web_interop.dart new file mode 100644 index 00000000..bb0635a3 --- /dev/null +++ b/packaging/npm/dart/lib/flutter_skill_web_interop.dart @@ -0,0 +1,25 @@ +/// Web implementation: exposes Dart-side element lookup to JavaScript. +/// Registers window.__FLUTTER_SKILL_DART_CALL__(method, paramsJson) +/// so the JS bridge (flutter-skill.js) can call into Dart for key-based lookups. + +import 'dart:js_interop'; + +@JS('window.__FLUTTER_SKILL_DART_CALL__') +external set _dartCallSetter(JSFunction? fn); + +String Function(String, String)? _handler; + +@JS('window.__FLUTTER_SKILL_DART_BRIDGE_READY__') +external set _dartBridgeReady(JSBoolean value); + +String _jsDartCall(JSString method, JSString paramsJson) { + if (_handler == null) return '{"error":"No handler registered"}'; + return _handler!(method.toDart, paramsJson.toDart); +} + +void registerWebBridge( + String Function(String method, String paramsJson) handler) { + _handler = handler; + _dartCallSetter = _jsDartCall.toJS; + _dartBridgeReady = true.toJS; +} diff --git a/packaging/npm/dart/lib/flutter_skill_web_stub.dart b/packaging/npm/dart/lib/flutter_skill_web_stub.dart new file mode 100644 index 00000000..bf3c5fa3 --- /dev/null +++ b/packaging/npm/dart/lib/flutter_skill_web_stub.dart @@ -0,0 +1,7 @@ +/// Stub implementation for non-web platforms. +/// This is a no-op; the real implementation is in flutter_skill_web_interop.dart. + +void registerWebBridge( + String Function(String method, String paramsJson) handler) { + // No-op on non-web platforms +} diff --git a/packaging/npm/dart/lib/src/bridge/bridge.dart b/packaging/npm/dart/lib/src/bridge/bridge.dart new file mode 100644 index 00000000..171020b6 --- /dev/null +++ b/packaging/npm/dart/lib/src/bridge/bridge.dart @@ -0,0 +1,5 @@ +/// Bridge module — cross-framework bridge protocol and proxy. +library; + +export 'bridge_protocol.dart'; +export 'web_bridge_proxy.dart'; diff --git a/packaging/npm/dart/lib/src/bridge/bridge_protocol.dart b/packaging/npm/dart/lib/src/bridge/bridge_protocol.dart new file mode 100644 index 00000000..21ecef8b --- /dev/null +++ b/packaging/npm/dart/lib/src/bridge/bridge_protocol.dart @@ -0,0 +1,174 @@ +import 'dart:convert'; + +/// Bridge Protocol — enables communication with any framework's in-app SDK. +/// +/// Each target app (React Native, web, native iOS/Android, etc.) installs a +/// lightweight SDK that starts a bridge service. The testing tool connects +/// over the network using JSON-RPC 2.0 over WebSocket. + +/// Protocol version. SDKs must return this in their health-check response. +const String bridgeProtocolVersion = '1.0'; + +/// Default port for bridge services. +const int bridgeDefaultPort = 18118; + +/// Port range scanned when looking for bridge-enabled apps. +const int bridgePortRangeStart = 18118; +const int bridgePortRangeEnd = 18128; + +/// Health-check endpoint path. A GET request here returns [BridgeServiceInfo]. +const String bridgeHealthPath = '/.flutter-skill'; + +/// Core methods every bridge SDK MUST implement. +const List bridgeCoreMethods = [ + 'initialize', + 'screenshot', + 'inspect', + 'tap', + 'enter_text', + 'swipe', + 'scroll', + 'find_element', + 'get_text', + 'wait_for_element', +]; + +/// Extended methods that SDKs MAY implement (advertised via capabilities). +const List bridgeExtendedMethods = [ + 'long_press', + 'double_tap', + 'drag', + 'get_state', + 'get_logs', + 'clear_logs', + 'hot_reload', + 'get_route', + 'go_back', +]; + +/// Information about a discovered bridge service. +class BridgeServiceInfo { + /// Framework name (e.g. "react-native", "web", "swiftui"). + final String framework; + + /// Application name as reported by the SDK. + final String appName; + + /// Platform (e.g. "ios", "android", "web", "macos"). + final String platform; + + /// Set of capabilities the SDK advertises (method names). + final Set capabilities; + + /// SDK version string. + final String sdkVersion; + + /// Port the bridge service is listening on. + final int port; + + /// Full WebSocket URI for JSON-RPC communication. + final String wsUri; + + BridgeServiceInfo({ + required this.framework, + required this.appName, + required this.platform, + required this.capabilities, + required this.sdkVersion, + required this.port, + required this.wsUri, + }); + + /// Parse from the JSON returned by the health-check endpoint. + factory BridgeServiceInfo.fromHealthCheck( + Map json, int port) { + final caps = (json['capabilities'] as List?) + ?.map((e) => e.toString()) + .toSet() ?? + {}; + return BridgeServiceInfo( + framework: json['framework'] as String? ?? 'unknown', + appName: json['app_name'] as String? ?? 'unknown', + platform: json['platform'] as String? ?? 'unknown', + capabilities: caps, + sdkVersion: json['sdk_version'] as String? ?? '0.0.0', + port: port, + wsUri: json['ws_uri'] as String? ?? + 'ws://127.0.0.1:${json['ws_port'] ?? port}/ws', + ); + } + + Map toJson() => { + 'framework': framework, + 'app_name': appName, + 'platform': platform, + 'capabilities': capabilities.toList(), + 'sdk_version': sdkVersion, + 'port': port, + 'ws_uri': wsUri, + }; + + @override + String toString() => '$framework app "$appName" on $platform (port $port)'; +} + +// --------------------------------------------------------------------------- +// JSON-RPC 2.0 message helpers +// --------------------------------------------------------------------------- + +int _nextId = 1; + +/// Build a JSON-RPC 2.0 request string. +String buildRpcRequest(String method, [Map? params]) { + final id = _nextId++; + return jsonEncode({ + 'jsonrpc': '2.0', + 'id': id, + 'method': method, + if (params != null) 'params': params, + }); +} + +/// Build a JSON-RPC 2.0 request with a specific id. +String buildRpcRequestWithId(int id, String method, + [Map? params]) { + return jsonEncode({ + 'jsonrpc': '2.0', + 'id': id, + 'method': method, + if (params != null) 'params': params, + }); +} + +/// Parse a JSON-RPC 2.0 response. Returns the `result` field on success +/// or throws on error. +Map parseRpcResponse(String raw) { + final json = jsonDecode(raw) as Map; + if (json.containsKey('error')) { + final err = json['error'] as Map; + throw RpcError( + code: err['code'] as int? ?? -1, + message: err['message'] as String? ?? 'Unknown error', + data: err['data'], + ); + } + return (json['result'] as Map?) ?? {}; +} + +/// Extract the `id` from a JSON-RPC message. +int? parseRpcId(String raw) { + final json = jsonDecode(raw) as Map; + return json['id'] as int?; +} + +/// JSON-RPC error. +class RpcError implements Exception { + final int code; + final String message; + final dynamic data; + + RpcError({required this.code, required this.message, this.data}); + + @override + String toString() => 'RpcError($code): $message'; +} diff --git a/packaging/npm/dart/lib/src/bridge/cdp_appmcp_methods.dart b/packaging/npm/dart/lib/src/bridge/cdp_appmcp_methods.dart new file mode 100644 index 00000000..96711f8b --- /dev/null +++ b/packaging/npm/dart/lib/src/bridge/cdp_appmcp_methods.dart @@ -0,0 +1,322 @@ +part of 'cdp_driver.dart'; + +/// AppMCP methods: tool discovery, tool calling, form discovery, +/// element highlighting, response mocking. +extension CdpAppmcpMethods on CdpDriver { + /// Highlight an element with a colored overlay. + Future> highlightElement(String selector, + {String? color, int duration = 3000}) async { + final c = color ?? 'red'; + final js = ''' +(function() { + var el = document.querySelector('$selector') || document.getElementById('$selector') || document.querySelector('[data-testid="$selector"]'); + if (!el) return JSON.stringify({success: false, error: 'Element not found'}); + var rect = el.getBoundingClientRect(); + var overlay = document.createElement('div'); + overlay.id = '__fs_highlight_' + Date.now(); + overlay.style.cssText = 'position:fixed;top:'+rect.top+'px;left:'+rect.left+'px;width:'+rect.width+'px;height:'+rect.height+'px;border:3px solid $c;background:${c}33;z-index:2147483647;pointer-events:none;transition:opacity 0.3s;'; + document.body.appendChild(overlay); + setTimeout(function(){ overlay.style.opacity='0'; setTimeout(function(){ overlay.remove(); }, 300); }, $duration); + return JSON.stringify({success: true, selector: '$selector', color: '$c', duration: $duration, bounds: {x: rect.x, y: rect.y, w: rect.width, h: rect.height}}); +})() +'''; + final result = await _evalJs(js); + final v = result['result']?['value'] as String?; + if (v != null) return jsonDecode(v) as Map; + return {'success': false, 'error': 'Eval failed'}; + } + + /// Mock a network response for requests matching a URL pattern. + Future> mockResponse( + String urlPattern, int statusCode, String body, + {Map? headers}) async { + return await interceptRequests(urlPattern, + statusCode: statusCode, body: body, headers: headers); + } + + // ── WebMCP: Discover and call structured page tools ── + + /// Discover all tools on the current page from multiple sources: + /// 0. Chrome 146+ native WebMCP (navigator.modelContextTesting) + /// 1. JS-registered tools (window.__flutter_skill_tools__) + /// 2. data-mcp-tool annotated forms + /// 3. manifest + /// 4. /.well-known/mcp.json + /// 5. Auto-converted
elements + Future> discoverTools() async { + final result = await _call('Runtime.evaluate', { + 'expression': ''' + (async () => { + const tools = []; + // 0. Chrome 146+ native WebMCP API (navigator.modelContextTesting) + if (typeof navigator !== 'undefined' && navigator.modelContextTesting) { + try { + const webMcpTools = await navigator.modelContextTesting.getTools(); + webMcpTools.forEach(t => { + tools.push({ + name: t.name, + description: t.description || '', + params: t.inputSchema && t.inputSchema.properties ? t.inputSchema.properties : {}, + inputSchema: t.inputSchema || null, + annotations: t.annotations || {}, + source: 'webmcp-native', + }); + }); + } catch(e) {} + } + // 1. JS-registered tools + if (window.__flutter_skill_tools__ && Array.isArray(window.__flutter_skill_tools__)) { + window.__flutter_skill_tools__.forEach(t => { + tools.push({ name: t.name, description: t.description || '', params: t.params || {}, source: 'js-registered', hasHandler: typeof t.handler === 'function' }); + }); + } + // 2. data-mcp-tool annotated elements + document.querySelectorAll('[data-mcp-tool]').forEach(el => { + const name = el.getAttribute('data-mcp-tool'); + const desc = el.getAttribute('data-mcp-description') || ''; + const params = {}; + el.querySelectorAll('[data-mcp-param]').forEach(input => { + const pName = input.getAttribute('data-mcp-param'); + params[pName] = { type: input.getAttribute('data-mcp-type') || 'string', required: input.hasAttribute('data-mcp-required'), description: input.getAttribute('data-mcp-description') || input.getAttribute('placeholder') || '' }; + }); + el.querySelectorAll('input[name], textarea[name], select[name]').forEach(input => { + if (!input.hasAttribute('data-mcp-param')) { + const pName = input.getAttribute('name'); + if (!params[pName]) params[pName] = { type: input.type === 'number' ? 'number' : input.type === 'checkbox' ? 'boolean' : 'string', required: input.required, description: input.getAttribute('placeholder') || '' }; + } + }); + tools.push({ name: name, description: desc, params: params, source: 'data-mcp-tool' }); + }); + // 3. manifests + for (const link of document.querySelectorAll('link[rel="mcp-tools"]')) { + try { const r = await fetch(link.href); const m = await r.json(); if (m.tools) m.tools.forEach(t => tools.push({ ...t, source: 'link-manifest', manifestUrl: link.href })); } catch(e) {} + } + // 4. /.well-known/mcp.json + try { const r = await fetch('/.well-known/mcp.json'); if (r.ok) { const m = await r.json(); if (m.tools) m.tools.forEach(t => tools.push({ ...t, source: 'well-known' })); } } catch(e) {} + // 5. Auto-discover forms + document.querySelectorAll('form').forEach((form, i) => { + if (form.hasAttribute('data-mcp-tool')) return; + const fid = form.id || form.getAttribute('name') || ''; + const name = 'form_' + (fid || i); + const params = {}; + form.querySelectorAll('input[name], textarea[name], select[name]').forEach(input => { + const pName = input.getAttribute('name'); + const label = (input.labels && input.labels[0]) ? input.labels[0].textContent.trim() : ''; + params[pName] = { type: input.type === 'number' ? 'number' : input.type === 'checkbox' ? 'boolean' : 'string', required: input.required, description: label || input.getAttribute('placeholder') || '' }; + }); + if (Object.keys(params).length > 0) tools.push({ name: name, description: 'Form: ' + (fid || form.getAttribute('action') || 'unnamed'), params: params, source: 'auto-form', formAction: form.getAttribute('action') || '', formMethod: (form.getAttribute('method') || 'GET').toUpperCase() }); + }); + // 6. Auto-discover ALL interactive elements as tools (zero-config) + const seen = new Set(tools.map(t => t.name)); + document.querySelectorAll('button, a[href], input, textarea, select, [role="button"], [role="link"], [role="switch"], [role="slider"], [onclick]').forEach((el, i) => { + const tag = el.tagName.toLowerCase(); + const text = (el.textContent || '').trim().substring(0, 50); + const id = el.id || el.getAttribute('name') || ''; + const label = text || id || el.getAttribute('aria-label') || ''; + if (!label) return; + const safeName = label.toLowerCase().replace(/[^a-z0-9]/g, '_').replace(/_+/g, '_').replace(/^_|_\$/g, ''); + if (!safeName) return; + const role = el.getAttribute('role') || ''; + let toolName, desc, action, params = {}; + if (tag === 'input' || tag === 'textarea') { + toolName = 'fill_' + safeName; + desc = 'Enter text: ' + label; + action = 'enter_text'; + params = { text: { type: 'string', required: true } }; + } else if (tag === 'select') { + toolName = 'select_' + safeName; + desc = 'Select: ' + label; + action = 'select'; + params = { value: { type: 'string', required: true } }; + } else if (role === 'switch' || el.type === 'checkbox') { + toolName = 'toggle_' + safeName; + desc = 'Toggle: ' + label; + action = 'tap'; + } else if (role === 'slider' || el.type === 'range') { + toolName = 'set_' + safeName; + desc = 'Set slider: ' + label; + action = 'set_value'; + params = { value: { type: 'number', required: true } }; + } else { + toolName = 'tap_' + safeName; + desc = 'Tap: ' + label; + action = 'tap'; + } + if (!seen.has(toolName)) { + seen.add(toolName); + const selector = id ? '#' + CSS.escape(id) : null; + tools.push({ name: toolName, description: desc, params, source: 'auto-ui', action, selector, elementIndex: i }); + } + }); + return JSON.stringify({ tools: tools, count: tools.length }); + })() + ''', + 'returnByValue': true, + 'awaitPromise': true, + }); + final v = result['result']?['value'] as String?; + if (v != null) return jsonDecode(v) as Map; + return {'tools': [], 'count': 0}; + } + + /// Call a discovered tool by name with parameters. + /// Routes to the appropriate handler based on tool source. + Future> callTool( + String toolName, Map params) async { + final paramsJson = jsonEncode(params); + final escapedName = jsonEncode(toolName); + final result = await _call('Runtime.evaluate', { + 'expression': ''' + (async () => { + const toolName = $escapedName; + const params = $paramsJson; + // 0. Chrome 146+ native WebMCP API (navigator.modelContextTesting) + if (typeof navigator !== 'undefined' && navigator.modelContextTesting) { + try { + const allTools = await navigator.modelContextTesting.getTools(); + const found = allTools.find(t => t.name === toolName); + if (found) { + const result = await navigator.modelContextTesting.executeTool(toolName, params); + return JSON.stringify({ success: true, result: result, source: 'webmcp-native' }); + } + } catch(e) {} + } + // 1. JS-registered tools + if (window.__flutter_skill_tools__) { + const tool = window.__flutter_skill_tools__.find(t => t.name === toolName); + if (tool && typeof tool.handler === 'function') { + try { + const result = await tool.handler(params); + return JSON.stringify({ success: true, result: result, source: 'js-registered' }); + } catch (e) { + return JSON.stringify({ success: false, error: e.message, source: 'js-registered' }); + } + } + } + // 2. data-mcp-tool forms + const mcpForm = document.querySelector('[data-mcp-tool="' + toolName + '"]'); + if (mcpForm) { + for (const [key, value] of Object.entries(params)) { + const input = mcpForm.querySelector('[name="' + key + '"], [data-mcp-param="' + key + '"]'); + if (input) { + if (input.type === 'checkbox') input.checked = !!value; + else input.value = value; + input.dispatchEvent(new Event('input', { bubbles: true })); + input.dispatchEvent(new Event('change', { bubbles: true })); + } + } + if (typeof mcpForm.requestSubmit === 'function') mcpForm.requestSubmit(); + else mcpForm.submit(); + return JSON.stringify({ success: true, source: 'data-mcp-tool', action: 'form-submitted' }); + } + // 3. Auto-discovered forms + const forms = document.querySelectorAll('form'); + for (let i = 0; i < forms.length; i++) { + const form = forms[i]; + const fid = form.id || form.getAttribute('name') || ''; + const formName = 'form_' + (fid || i); + if (formName === toolName) { + for (const [key, value] of Object.entries(params)) { + const input = form.querySelector('[name="' + key + '"]'); + if (input) { + if (input.type === 'checkbox') input.checked = !!value; + else input.value = value; + input.dispatchEvent(new Event('input', { bubbles: true })); + input.dispatchEvent(new Event('change', { bubbles: true })); + } + } + if (typeof form.requestSubmit === 'function') form.requestSubmit(); + else form.submit(); + return JSON.stringify({ success: true, source: 'auto-form', action: 'form-submitted' }); + } + } + // 4. Auto-UI tools (tap/fill/toggle by selector or text) + if (toolName.startsWith('tap_') || toolName.startsWith('toggle_')) { + const els = document.querySelectorAll('button, a[href], [role="button"], [role="link"], [role="switch"], [onclick], input[type="checkbox"]'); + for (const el of els) { + const text = (el.textContent || '').trim().substring(0, 50); + const id = el.id || el.getAttribute('name') || ''; + const label = text || id || el.getAttribute('aria-label') || ''; + const safeName = label.toLowerCase().replace(/[^a-z0-9]/g, '_').replace(/_+/g, '_').replace(/^_|_\$/g, ''); + const prefix = (el.getAttribute('role') === 'switch' || el.type === 'checkbox') ? 'toggle_' : 'tap_'; + if (prefix + safeName === toolName) { + el.click(); + return JSON.stringify({ success: true, source: 'auto-ui', action: 'clicked', element: label }); + } + } + } + if (toolName.startsWith('fill_')) { + const els = document.querySelectorAll('input, textarea'); + for (const el of els) { + const text = (el.textContent || '').trim().substring(0, 50); + const id = el.id || el.getAttribute('name') || ''; + const label = text || id || el.getAttribute('aria-label') || el.getAttribute('placeholder') || ''; + const safeName = label.toLowerCase().replace(/[^a-z0-9]/g, '_').replace(/_+/g, '_').replace(/^_|_\$/g, ''); + if ('fill_' + safeName === toolName) { + el.focus(); + el.value = params.text || ''; + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + return JSON.stringify({ success: true, source: 'auto-ui', action: 'filled', element: label }); + } + } + } + return JSON.stringify({ success: false, error: 'Tool not found: ' + toolName }); + })() + ''', + 'returnByValue': true, + 'awaitPromise': true, + }); + final v = result['result']?['value'] as String?; + if (v != null) return jsonDecode(v) as Map; + return {'success': false, 'error': 'Evaluation failed'}; + } + + /// Auto-discover ALL form elements on the page and convert them to tools. + Future> autoDiscoverForms() async { + final result = await _evalJs(''' + (() => { + const tools = []; + document.querySelectorAll('form').forEach((form, i) => { + const fid = form.id || form.getAttribute('name') || ''; + const action = form.getAttribute('action') || ''; + const method = (form.getAttribute('method') || 'GET').toUpperCase(); + const name = 'form_' + (fid || i); + const params = {}; + form.querySelectorAll('input, textarea, select').forEach(input => { + const inputName = input.getAttribute('name') || input.id || ''; + if (!inputName || input.type === 'hidden' || input.type === 'submit') return; + const label = (input.labels && input.labels[0]) ? input.labels[0].textContent.trim() : ''; + const placeholder = input.getAttribute('placeholder') || ''; + const ariaLabel = input.getAttribute('aria-label') || ''; + params[inputName] = { + type: input.type === 'number' || input.type === 'range' ? 'number' + : input.type === 'checkbox' ? 'boolean' + : input.type === 'email' ? 'string (email)' + : input.type === 'date' ? 'string (date)' + : 'string', + required: input.required || input.hasAttribute('data-mcp-required'), + description: label || ariaLabel || placeholder || inputName, + inputType: input.type || 'text', + tag: input.tagName.toLowerCase() + }; + }); + tools.push({ + name: name, + description: fid ? ('Form: ' + fid) : ('Form #' + i + (action ? ' → ' + action : '')), + params: params, + fieldCount: Object.keys(params).length, + source: 'auto-form', + formAction: action, + formMethod: method, + hasSubmitButton: !!form.querySelector('[type="submit"], button:not([type="button"])') + }); + }); + return JSON.stringify({ tools: tools, count: tools.length }); + })() + '''); + final v = result['result']?['value'] as String?; + if (v != null) return jsonDecode(v) as Map; + return {'tools': [], 'count': 0}; + } +} diff --git a/packaging/npm/dart/lib/src/bridge/cdp_browser_methods.dart b/packaging/npm/dart/lib/src/bridge/cdp_browser_methods.dart new file mode 100644 index 00000000..cf195240 --- /dev/null +++ b/packaging/npm/dart/lib/src/bridge/cdp_browser_methods.dart @@ -0,0 +1,1030 @@ +part of 'cdp_driver.dart'; + +/// Browser-specific CDP methods: cookies, storage, viewport, +/// tabs, frames, network, emulation, PDF, dialogs, etc. +extension CdpBrowserMethods on CdpDriver { + /// Get/set cookies. + Future> getCookies() async { + final result = await _call('Network.getCookies'); + return result; + } + + Future> setCookie(String name, String value, + {String? domain, String? path}) async { + await _call('Network.setCookie', { + 'name': name, + 'value': value, + if (domain != null) 'domain': domain, + 'path': path ?? '/', + }); + return {"success": true}; + } + + Future> clearCookies() async { + await _call('Network.clearBrowserCookies'); + return {"success": true}; + } + + /// LocalStorage operations. + Future> getLocalStorage() async { + final result = await _evalJs( + 'JSON.stringify(Object.fromEntries(Object.entries(localStorage)))'); + final v = result['result']?['value'] as String?; + if (v == null) return {}; + return jsonDecode(v) as Map; + } + + Future> setLocalStorage(String key, String value) async { + await _evalJs( + "localStorage.setItem('${key.replaceAll("'", "\\'")}', '${value.replaceAll("'", "\\'")}')"); + return {"success": true}; + } + + Future> clearLocalStorage() async { + await _evalJs("localStorage.clear()"); + return {"success": true}; + } + + /// Get console messages (via Runtime.consoleAPICalled events). + Future> getConsoleMessages() async { + // Enable console tracking if not already + try { + await _call('Runtime.enable'); + } catch (_) {} + final result = await _evalJs(''' + JSON.stringify(window.__cdpConsoleLog || []) + '''); + final v = result['result']?['value'] as String?; + return {"messages": v != null ? jsonDecode(v) : [], "source": "runtime"}; + } + + /// Get network requests (requires Network.enable). + Future> getNetworkRequests({int limit = 100}) async { + final result = await _evalJs(''' + JSON.stringify(performance.getEntriesByType('resource').slice(-$limit).map(r => ({ + name: r.name, + type: r.initiatorType, + duration: Math.round(r.duration), + size: r.transferSize || 0, + status: r.responseStatus || 200 + }))) + '''); + final v = result['result']?['value'] as String?; + final requests = v != null ? jsonDecode(v) : []; + return {"requests": requests, "limited_to": limit}; + } + + /// Set viewport size. + Future> setViewport(int width, int height, + {double deviceScaleFactor = 1.0}) async { + await _call('Emulation.setDeviceMetricsOverride', { + 'width': width, + 'height': height, + 'deviceScaleFactor': deviceScaleFactor, + 'mobile': false, + }); + return {"success": true, "width": width, "height": height}; + } + + /// Emulate device (mobile/tablet). + Future> emulateDevice(String device) async { + // Empty/blank device name → list all available devices + if (device.trim().isEmpty) { + final categories = listDevicesByCategory(); + return { + "success": true, + "action": "list", + "total": devicePresets.length, + "devices": categories, + }; + } + + final preset = lookupDevice(device); + if (preset == null) { + // Find close matches for helpful error + final normalized = + device.trim().toLowerCase().replaceAll(RegExp(r'[\s_]+'), '-'); + final suggestions = devicePresets.keys + .where((k) => k.contains(normalized) || normalized.contains(k)) + .take(10) + .toList(); + return { + "success": false, + "message": "Unknown device: $device", + "suggestions": suggestions, + "total_available": devicePresets.length, + "hint": "Pass an empty device name to list all available devices.", + }; + } + + await _call('Emulation.setDeviceMetricsOverride', { + 'width': preset.width, + 'height': preset.height, + 'deviceScaleFactor': preset.deviceScaleFactor, + 'mobile': preset.isMobile, + }); + if (preset.hasTouch) { + await _call('Emulation.setTouchEmulationEnabled', {'enabled': true}); + } + if (preset.userAgent != null) { + await _call( + 'Emulation.setUserAgentOverride', {'userAgent': preset.userAgent}); + } + return { + "success": true, + "device": device, + "viewport": {"width": preset.width, "height": preset.height}, + "deviceScaleFactor": preset.deviceScaleFactor, + "isMobile": preset.isMobile, + "hasTouch": preset.hasTouch, + }; + } + + /// Generate PDF (headless Chrome only). + Future> generatePdf() async { + try { + final result = await _call('Page.printToPDF', { + 'printBackground': true, + 'preferCSSPageSize': true, + }); + final data = result['data'] as String?; + if (data != null) { + final timestamp = DateTime.now().millisecondsSinceEpoch; + final file = File( + '${Directory.systemTemp.path}/flutter_skill_page_$timestamp.pdf'); + await file.writeAsBytes(base64.decode(data)); + return {"success": true, "file_path": file.path}; + } + return {"success": false, "message": "No PDF data"}; + } catch (e) { + return { + "success": false, + "message": "PDF generation requires headless Chrome: $e" + }; + } + } + + /// Wait for navigation (page load). + Future> waitForNavigation( + {int timeoutMs = 10000}) async { + // Simple approach: wait for load event + await Future.delayed( + Duration(milliseconds: timeoutMs > 3000 ? 3000 : timeoutMs)); + final url = await getCurrentRoute(); + return {"success": true, "url": url}; + } + + /// Navigate to URL. + Future> navigate(String url) async { + await _call('Page.navigate', {'url': url}); + await Future.delayed(const Duration(seconds: 2)); + return {"success": true, "url": url}; + } + + /// Go forward in history. + Future goForward() async { + await _evalJs("history.forward()"); + await Future.delayed(const Duration(milliseconds: 300)); + return true; + } + + /// Reload page. + Future> reload() async { + await _call('Page.reload'); + await Future.delayed(const Duration(seconds: 1)); + return {"success": true}; + } + + /// Get element attribute. + Future> getAttribute( + String key, String attribute) async { + final result = await _evalJs(''' + (() => { + const el = document.getElementById('$key') || document.querySelector('[data-testid="$key"]'); + if (!el) return null; + return el.getAttribute('$attribute'); + })() + '''); + return {"value": result['result']?['value']}; + } + + /// Get element CSS property. + Future> getCssProperty( + String key, String property) async { + final result = await _evalJs(''' + (() => { + const el = document.getElementById('$key') || document.querySelector('[data-testid="$key"]'); + if (!el) return null; + return getComputedStyle(el).getPropertyValue('$property'); + })() + '''); + return {"value": result['result']?['value']}; + } + + /// Get element bounding box (public). + Future> getBoundingBox(String key) async { + final bounds = await _getElementBounds(key); + if (bounds == null) + return {"success": false, "message": "Element not found"}; + return {"success": true, "bounds": bounds}; + } + + /// Count elements matching selector. + Future countElements(String selector) async { + final result = + await _evalJs('document.querySelectorAll("$selector").length'); + return result['result']?['value'] as int? ?? 0; + } + + /// Check if element is visible. + Future isVisible(String key) async { + final result = await _evalJs(''' + (() => { + const el = document.getElementById('$key') || document.querySelector('[data-testid="$key"]'); + if (!el) return false; + const rect = el.getBoundingClientRect(); + const style = getComputedStyle(el); + return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'; + })() + '''); + return result['result']?['value'] == true; + } + + /// Focus element. + Future> focus(String key) async { + final result = await _evalJs(''' + (() => { + const el = document.getElementById('$key') || document.querySelector('[data-testid="$key"]'); + if (!el) return false; + el.focus(); + return true; + })() + '''); + return {"success": result['result']?['value'] == true}; + } + + /// Blur (unfocus) element. + Future> blur(String key) async { + await _evalJs(''' + (() => { + const el = document.getElementById('$key') || document.querySelector('[data-testid="$key"]'); + if (el) el.blur(); + })() + '''); + return {"success": true}; + } + + /// Get page title. + Future getTitle() async { + final result = await _evalJs('document.title'); + return (result['result']?['value'] as String?) ?? ''; + } + + /// Get page HTML with optional cleaning. + Future getPageSource({ + String? selector, + bool removeScripts = false, + bool removeStyles = false, + bool removeComments = false, + bool removeMeta = false, + bool minify = false, + bool cleanHtml = false, + }) async { + if (cleanHtml) { + removeScripts = true; + removeStyles = true; + removeComments = true; + removeMeta = true; + } + final selectorJs = selector != null + ? 'var el = document.querySelector(${jsonEncode(selector)}); el ? el.outerHTML : ""' + : 'document.documentElement.outerHTML'; + final result = await _evalJs(selectorJs); + var html = (result['result']?['value'] as String?) ?? ''; + if (removeScripts) + html = html.replaceAll( + RegExp(r'', caseSensitive: false), ''); + if (removeStyles) + html = html.replaceAll( + RegExp(r'', caseSensitive: false), ''); + if (removeComments) html = html.replaceAll(RegExp(r''), ''); + if (removeMeta) + html = + html.replaceAll(RegExp(r']*/?>', caseSensitive: false), ''); + if (minify) html = html.replaceAll(RegExp(r'\s+'), ' ').trim(); + return html; + } + + /// Get visible text content (skips hidden elements). + Future getVisibleText({String? selector}) async { + final root = selector != null + ? 'document.querySelector(${jsonEncode(selector)})' + : 'document.body'; + final js = ''' +(function() { + var root = $root; + if (!root) return ''; + var walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT, { + acceptNode: function(node) { + var el = node.parentElement; + if (!el) return NodeFilter.FILTER_REJECT; + var style = window.getComputedStyle(el); + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return NodeFilter.FILTER_REJECT; + return NodeFilter.FILTER_ACCEPT; + } + }); + var texts = []; + while (walker.nextNode()) { + var t = walker.currentNode.textContent.trim(); + if (t) texts.push(t); + } + return texts.join(' '); +})() +'''; + final result = await _evalJs(js); + return (result['result']?['value'] as String?) ?? ''; + } + + /// Set geolocation. + Future> setGeolocation(double latitude, double longitude, + {double accuracy = 100}) async { + await _call('Emulation.setGeolocationOverride', { + 'latitude': latitude, + 'longitude': longitude, + 'accuracy': accuracy, + }); + return {"success": true}; + } + + /// Set timezone. + Future> setTimezone(String timezone) async { + await _call('Emulation.setTimezoneOverride', {'timezoneId': timezone}); + return {"success": true}; + } + + /// Set dark/light mode. + Future> setColorScheme(String scheme) async { + await _call('Emulation.setEmulatedMedia', { + 'features': [ + {'name': 'prefers-color-scheme', 'value': scheme} + ], + }); + return {"success": true, "scheme": scheme}; + } + + /// Block URLs (e.g. ads, trackers). + Future> blockUrls(List patterns) async { + await _call('Network.setBlockedURLs', {'urls': patterns}); + return {"success": true, "blocked": patterns}; + } + + /// Throttle network (simulate slow connections). + Future> throttleNetwork( + {int latencyMs = 0, int downloadKbps = -1, int uploadKbps = -1}) async { + await _call('Network.emulateNetworkConditions', { + 'offline': false, + 'latency': latencyMs, + 'downloadThroughput': downloadKbps > 0 ? downloadKbps * 1024 / 8 : -1, + 'uploadThroughput': uploadKbps > 0 ? uploadKbps * 1024 / 8 : -1, + }); + return {"success": true}; + } + + /// Disable network (offline mode). + Future> goOffline() async { + await _call('Network.emulateNetworkConditions', { + 'offline': true, + 'latency': 0, + 'downloadThroughput': -1, + 'uploadThroughput': -1, + }); + return {"success": true, "offline": true}; + } + + /// Go back online — restore normal network conditions. + Future> goOnline() async { + await _call('Network.emulateNetworkConditions', { + 'offline': false, + 'latency': 0, + 'downloadThroughput': -1, + 'uploadThroughput': -1, + }); + return {"success": true, "offline": false}; + } + + /// Clear browser data. + Future> clearBrowserData() async { + await _call('Network.clearBrowserCookies'); + await _call('Network.clearBrowserCache'); + await _evalJs("localStorage.clear(); sessionStorage.clear()"); + return {"success": true}; + } + + // ── File upload ── + + /// Upload file to input[type=file]. + /// Pierces Shadow DOM automatically to find deeply nested file inputs. + Future> uploadFile( + String selector, List filePaths) async { + final escapedSelector = + selector.replaceAll('\\', '\\\\').replaceAll("'", "\\'"); + + // Helper: find element via light DOM or Shadow DOM + Future _resolveNodeId() async { + final doc = await _call('DOM.getDocument'); + final rootNodeId = doc['root']?['nodeId'] as int? ?? 0; + final result = await _call('DOM.querySelector', { + 'nodeId': rootNodeId, + 'selector': selector, + }); + final nodeId = result['nodeId'] as int?; + if (nodeId != null && nodeId != 0) return nodeId; + return null; + } + + Future _resolveBackendNodeId() async { + final jsObj = await _call('Runtime.evaluate', { + 'expression': '''(() => { + function dq(s,r){let e=r.querySelector(s);if(e)return e;for(const n of r.querySelectorAll("*"))if(n.shadowRoot){e=dq(s,n.shadowRoot);if(e)return e;}return null;} + return dq('$escapedSelector',document); + })()''', + 'returnByValue': false, + }); + final objectId = jsObj['result']?['objectId'] as String?; + if (objectId == null) return null; + final desc = await _call('DOM.describeNode', {'objectId': objectId}); + await _call('Runtime.releaseObject', {'objectId': objectId}) + .catchError((_) => {}); + return desc['node']?['backendNodeId'] as int?; + } + + Future _setFiles() async { + final nodeId = await _resolveNodeId(); + if (nodeId != null) { + await _call( + 'DOM.setFileInputFiles', {'nodeId': nodeId, 'files': filePaths}); + return; + } + final backendNodeId = await _resolveBackendNodeId(); + if (backendNodeId != null) { + await _call('DOM.setFileInputFiles', + {'backendNodeId': backendNodeId, 'files': filePaths}); + } + } + + // ── Strategy 1: File chooser interception (works with React/Vue/Angular) ── + // Intercept the native file dialog, click the input with a trusted gesture, + // then feed files. This triggers framework event handlers properly. + bool fileChooserFired = false; + int? chooserBackendNodeId; + try { + await _call('Page.setInterceptFileChooserDialog', {'enabled': true}); + + onEvent('Page.fileChooserOpened', (params) { + fileChooserFired = true; + chooserBackendNodeId = params['backendNodeId'] as int?; + }); + + // Click the file input using DOM.focus + Enter key (trusted gesture) + // Or find the parent clickable element + final clickResult = await _call('Runtime.evaluate', { + 'expression': '''(() => { + function dq(s,r){let e=r.querySelector(s);if(e)return e;for(const n of r.querySelectorAll("*"))if(n.shadowRoot){e=dq(s,n.shadowRoot);if(e)return e;}return null;} + const el = dq('$escapedSelector',document); + if (!el) return JSON.stringify({found:false}); + // Find the visible parent that users would click + let target = el.parentElement; + while (target && (target.offsetWidth === 0 || target.offsetHeight === 0)) { + target = target.parentElement; + } + if (!target) target = el; + const r = target.getBoundingClientRect(); + return JSON.stringify({found:true, x:r.x+r.width/2, y:r.y+r.height/2}); + })()''', + 'returnByValue': true, + }); + + final clickJson = clickResult['result']?['value'] as String? ?? '{}'; + final clickData = jsonDecode(clickJson) as Map; + + if (clickData['found'] == true) { + final cx = (clickData['x'] as num).toDouble(); + final cy = (clickData['y'] as num).toDouble(); + // Hover first (some UIs require hover to reveal the clickable area) + await _dispatchMouseEvent('mouseMoved', cx, cy); + await Future.delayed(const Duration(milliseconds: 200)); + // Trusted CDP mouse click on the visible parent + await _dispatchMouseEvent('mousePressed', cx, cy, + button: 'left', clickCount: 1); + await _dispatchMouseEvent('mouseReleased', cx, cy, + button: 'left', clickCount: 1); + + // Wait for file chooser + for (int i = 0; i < 20 && !fileChooserFired; i++) { + await Future.delayed(const Duration(milliseconds: 100)); + } + } + + if (fileChooserFired) { + // Use the backendNodeId from the event to set files on the exact input + // that triggered the file chooser. This triggers native change events. + if (chooserBackendNodeId != null) { + await _call('DOM.setFileInputFiles', { + 'backendNodeId': chooserBackendNodeId, + 'files': filePaths, + }); + } else { + await _setFiles(); + } + await Future.delayed(const Duration(milliseconds: 300)); + try { + await _call('Page.setInterceptFileChooserDialog', {'enabled': false}); + } catch (_) {} + removeEventListeners('Page.fileChooserOpened'); + // Verify files were actually set + final fcVerified = await _verifyFilesSet(escapedSelector); + return { + "success": fcVerified, + "files": fcVerified ? filePaths : [], + "method": "fileChooser", + if (!fcVerified) + "error": "File chooser completed but files.length is 0" + }; + } + + // File chooser didn't fire — disable interception and fall through + try { + await _call('Page.setInterceptFileChooserDialog', {'enabled': false}); + } catch (_) {} + removeEventListeners('Page.fileChooserOpened'); + } catch (e) { + // File chooser interception not supported — continue to fallback + try { + await _call('Page.setInterceptFileChooserDialog', {'enabled': false}); + } catch (_) {} + removeEventListeners('Page.fileChooserOpened'); + } + + // ── Strategy 2: Direct setFileInputFiles + dispatch events (legacy) ── + await _setFiles(); + await _dispatchFileEvents(selector); + + // ── Verify files were actually set ── + final verified = await _verifyFilesSet(escapedSelector); + if (!verified) { + // Strategy 3: Try iframe-aware upload + final iframeResult = await _tryIframeUpload(filePaths); + if (iframeResult != null) return iframeResult; + return { + "success": false, + "files": [], + "method": "direct", + "error": + "Files not set after setFileInputFiles — input.files.length is 0" + }; + } + return {"success": true, "files": filePaths, "method": "direct"}; + } + + /// Dispatch change + input events on a file input so frameworks (React, Vue, Angular) detect it. + /// `DOM.setFileInputFiles` sets files but does NOT fire DOM events automatically. + /// For React: also calls the React onChange handler directly via __reactProps$. + Future _dispatchFileEvents(String selector) async { + final escapedSelector = + selector.replaceAll('\\', '\\\\').replaceAll("'", "\\'"); + await _call('Runtime.evaluate', { + 'expression': ''' + (() => { + function deepQuery(sel, root) { + let el = root.querySelector(sel); + if (el) return el; + for (const n of root.querySelectorAll('*')) { + if (n.shadowRoot) { + el = deepQuery(sel, n.shadowRoot); + if (el) return el; + } + } + return null; + } + const el = deepQuery('$escapedSelector', document); + if (!el) return; + + // 1. Standard DOM events + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + + // 2. React synthetic event — React ignores native events on file inputs + const propsKey = Object.keys(el).find(k => k.startsWith('__reactProps')); + if (propsKey && el[propsKey] && typeof el[propsKey].onChange === 'function') { + el[propsKey].onChange({ target: el, currentTarget: el }); + } + })() + ''', + 'returnByValue': true, + }); + } + + /// Verify that files were actually set on the file input. + /// Returns true if input.files.length > 0. + Future _verifyFilesSet(String escapedSelector) async { + try { + final result = await _call('Runtime.evaluate', { + 'expression': '''(() => { + function dq(s,r){let e=r.querySelector(s);if(e)return e;for(const n of r.querySelectorAll("*"))if(n.shadowRoot){e=dq(s,n.shadowRoot);if(e)return e;}return null;} + const el = dq('$escapedSelector', document); + return el ? el.files.length : -1; + })()''', + 'returnByValue': true, + }); + final count = result['result']?['value'] as int? ?? 0; + return count > 0; + } catch (_) { + return false; + } + } + + /// Try to find and upload to file inputs inside iframes. + /// Google, some social sites use iframes for profile picture dialogs. + Future?> _tryIframeUpload(List filePaths) async { + try { + // Find file inputs inside iframes + final result = await _call('Runtime.evaluate', { + 'expression': '''(() => { + const iframes = document.querySelectorAll('iframe'); + for (const iframe of iframes) { + try { + const doc = iframe.contentDocument; + if (!doc) continue; + const inp = doc.querySelector('input[type=file]'); + if (inp) { + return JSON.stringify({found: true, iframeSrc: iframe.src.substring(0, 100)}); + } + } catch(e) { /* cross-origin */ } + } + return JSON.stringify({found: false}); + })()''', + 'returnByValue': true, + }); + final data = jsonDecode(result['result']?['value'] as String? ?? '{}') + as Map; + if (data['found'] != true) return null; + + // Get the iframe's file input backendNodeId + final nodeResult = await _call('Runtime.evaluate', { + 'expression': '''(() => { + const iframes = document.querySelectorAll('iframe'); + for (const iframe of iframes) { + try { + const doc = iframe.contentDocument; + if (!doc) continue; + const inp = doc.querySelector('input[type=file]'); + if (inp) return inp; + } catch(e) {} + } + return null; + })()''', + 'returnByValue': false, + }); + final objectId = nodeResult['result']?['objectId'] as String?; + if (objectId == null) return null; + + final desc = await _call('DOM.describeNode', {'objectId': objectId}); + await _call('Runtime.releaseObject', {'objectId': objectId}) + .catchError((_) => {}); + final backendNodeId = desc['node']?['backendNodeId'] as int?; + if (backendNodeId == null) return null; + + await _call('DOM.setFileInputFiles', + {'backendNodeId': backendNodeId, 'files': filePaths}); + await Future.delayed(const Duration(milliseconds: 300)); + + // Dispatch events in iframe context + await _call('Runtime.evaluate', { + 'expression': '''(() => { + const iframes = document.querySelectorAll('iframe'); + for (const iframe of iframes) { + try { + const doc = iframe.contentDocument; + if (!doc) continue; + const inp = doc.querySelector('input[type=file]'); + if (inp) { + inp.dispatchEvent(new Event('input', {bubbles: true})); + inp.dispatchEvent(new Event('change', {bubbles: true})); + return true; + } + } catch(e) {} + } + return false; + })()''', + 'returnByValue': true, + }); + + return {"success": true, "files": filePaths, "method": "iframe"}; + } catch (_) { + return null; + } + } + + // ── Dialog handling ── + + // _dialogHandlerInstalled field is in CdpDriver class + + Future installDialogHandler({bool autoAccept = true}) async { + if (_dialogHandlerInstalled) return; + _dialogHandlerInstalled = true; + // Dialog events come as CDP events — handled in _onMessage + } + + Future> handleDialog(bool accept, + {String? promptText}) async { + await _call('Page.handleJavaScriptDialog', { + 'accept': accept, + if (promptText != null) 'promptText': promptText, + }); + return {"success": true, "action": accept ? "accepted" : "dismissed"}; + } + + // ── Iframe support ── + + Future> getFrames() async { + final tree = await _call('Page.getFrameTree'); + List> flatten(Map frame) { + final result = >[]; + final f = frame['frame'] as Map?; + if (f != null) { + result.add({ + 'id': f['id'], + 'url': f['url'], + 'name': f['name'] ?? '', + 'securityOrigin': f['securityOrigin'], + }); + } + final children = frame['childFrames'] as List?; + if (children != null) { + for (final child in children) { + result.addAll(flatten(child as Map)); + } + } + return result; + } + + final frames = flatten(tree['frameTree'] as Map? ?? {}); + return {"frames": frames, "count": frames.length}; + } + + Future> evalInFrame( + String frameId, String expression) async { + // Create isolated world in the target frame + final world = await _call('Page.createIsolatedWorld', { + 'frameId': frameId, + 'worldName': 'flutter-skill-eval', + 'grantUniveralAccess': true, + }); + final contextId = world['executionContextId'] as int?; + if (contextId == null) + return {"success": false, "message": "Cannot access frame"}; + final result = await _call('Runtime.evaluate', { + 'expression': expression, + 'contextId': contextId, + 'returnByValue': true, + }); + return {"result": result['result']?['value']}; + } + + // ── Multi-tab management ── + + Future> getTabs() async { + final client = HttpClient(); + final request = + await client.getUrl(Uri.parse('http://127.0.0.1:$_port/json')); + final response = await request.close(); + final body = await response.transform(utf8.decoder).join(); + client.close(); + final tabs = (jsonDecode(body) as List) + .where((t) => t['type'] == 'page') + .map((t) => { + 'id': t['id'], + 'title': t['title'], + 'url': t['url'], + }) + .toList(); + return {"tabs": tabs, "count": tabs.length}; + } + + Future> newTab(String url) async { + // Enforce max_tabs limit to prevent runaway tab creation + try { + final tabs = await getTabs(); + final tabList = tabs['tabs'] as List?; + if (tabList != null && tabList.length >= _maxTabs) { + return { + "success": false, + "error": "Max tabs limit reached ($_maxTabs). Close some tabs first." + }; + } + } catch (_) {} + final result = await _call('Target.createTarget', {'url': url}); + return {"success": true, "targetId": result['targetId']}; + } + + Future> closeTab(String targetId) async { + await _call('Target.closeTarget', {'targetId': targetId}); + return {"success": true}; + } + + Future> switchTab(String targetId) async { + await _call('Target.activateTarget', {'targetId': targetId}); + return {"success": true}; + } + + // ── Network request interception/mocking ── + + // _interceptRules field is in CdpDriver class + + Future> interceptRequests(String urlPattern, + {int? statusCode, String? body, Map? headers}) async { + await _call('Fetch.enable', { + 'patterns': [ + {'urlPattern': urlPattern, 'requestStage': 'Response'} + ], + }); + _interceptRules[urlPattern] = { + 'statusCode': statusCode ?? 200, + 'body': body ?? '', + 'headers': headers ?? {}, + }; + return {"success": true, "pattern": urlPattern}; + } + + Future> clearInterceptions() async { + await _call('Fetch.disable'); + _interceptRules.clear(); + return {"success": true}; + } + + // ── Accessibility audit ── + + Future> accessibilityAudit() async { + final result = await _evalJs(''' + JSON.stringify((() => { + const issues = []; + // Check images without alt + document.querySelectorAll('img:not([alt])').forEach(img => { + issues.push({type: 'error', rule: 'img-alt', message: 'Image missing alt attribute', element: img.src?.substring(0, 80)}); + }); + // Check form inputs without labels + document.querySelectorAll('input:not([type="hidden"]):not([aria-label]):not([id])').forEach(el => { + issues.push({type: 'warning', rule: 'input-label', message: 'Input missing associated label', element: el.outerHTML?.substring(0, 80)}); + }); + // Check empty buttons + document.querySelectorAll('button').forEach(btn => { + if (!btn.textContent?.trim() && !btn.getAttribute('aria-label')) { + issues.push({type: 'error', rule: 'button-name', message: 'Button has no accessible name', element: btn.outerHTML?.substring(0, 80)}); + } + }); + // Check empty links + document.querySelectorAll('a').forEach(a => { + if (!a.textContent?.trim() && !a.getAttribute('aria-label')) { + issues.push({type: 'error', rule: 'link-name', message: 'Link has no accessible name', element: a.outerHTML?.substring(0, 80)}); + } + }); + // Check heading hierarchy + let lastLevel = 0; + document.querySelectorAll('h1,h2,h3,h4,h5,h6').forEach(h => { + const level = parseInt(h.tagName[1]); + if (level > lastLevel + 1 && lastLevel > 0) { + issues.push({type: 'warning', rule: 'heading-order', message: 'Heading level skipped: h' + lastLevel + ' → h' + level, element: h.textContent?.substring(0, 40)}); + } + lastLevel = level; + }); + // Check color contrast (basic) + document.querySelectorAll('*').forEach(el => { + const style = getComputedStyle(el); + if (style.color === style.backgroundColor && el.textContent?.trim()) { + issues.push({type: 'error', rule: 'color-contrast', message: 'Text same color as background', element: el.textContent?.substring(0, 40)}); + } + }); + // Check document language + if (!document.documentElement.lang) { + issues.push({type: 'warning', rule: 'html-lang', message: 'HTML element missing lang attribute'}); + } + // Check viewport meta + if (!document.querySelector('meta[name="viewport"]')) { + issues.push({type: 'warning', rule: 'viewport', message: 'Missing viewport meta tag'}); + } + return {issues: issues, total: issues.length, errors: issues.filter(i => i.type === 'error').length, warnings: issues.filter(i => i.type === 'warning').length}; + })()) + '''); + final v = result['result']?['value'] as String?; + if (v == null) return {"issues": [], "total": 0}; + return jsonDecode(v) as Map; + } + + // ── Visual regression ── + + Future> compareScreenshot(String baselinePath) async { + final current = await takeScreenshot(); + if (current == null) + return {"success": false, "message": "Screenshot failed"}; + + final currentFile = File( + '${Directory.systemTemp.path}/flutter_skill_compare_${DateTime.now().millisecondsSinceEpoch}.png'); + await currentFile.writeAsBytes(base64.decode(current)); + + final baselineFile = File(baselinePath); + if (!baselineFile.existsSync()) { + // No baseline — save current as baseline + await currentFile.copy(baselinePath); + return { + "success": true, + "action": "baseline_created", + "path": baselinePath + }; + } + + // Basic pixel comparison + final baselineBytes = await baselineFile.readAsBytes(); + final currentBytes = await currentFile.readAsBytes(); + + if (baselineBytes.length != currentBytes.length) { + return { + "success": false, + "match": false, + "reason": "Image sizes differ", + "baseline_size": baselineBytes.length, + "current_size": currentBytes.length, + "current_path": currentFile.path, + }; + } + + int diffPixels = 0; + for (int i = 0; i < baselineBytes.length; i++) { + if (baselineBytes[i] != currentBytes[i]) diffPixels++; + } + final diffPercent = (diffPixels / baselineBytes.length * 100); + final match = diffPercent < 1.0; // 1% tolerance + + return { + "success": true, + "match": match, + "diff_percent": double.parse(diffPercent.toStringAsFixed(2)), + "current_path": currentFile.path, + "baseline_path": baselinePath, + }; + } + + // ── Wait for network idle ── + + Future> waitForNetworkIdle( + {int timeoutMs = 10000, int idleMs = 500}) async { + final result = await _evalJs(''' + new Promise((resolve) => { + let pending = 0; + let timer = null; + const timeout = setTimeout(() => resolve(JSON.stringify({idle: false, reason: 'timeout'})), $timeoutMs); + const check = () => { + if (pending <= 0) { + clearTimeout(timer); + timer = setTimeout(() => { + clearTimeout(timeout); + resolve(JSON.stringify({idle: true})); + }, $idleMs); + } + }; + const origFetch = window.fetch; + window.fetch = function() { + pending++; + return origFetch.apply(this, arguments).finally(() => { pending--; check(); }); + }; + const origXHR = XMLHttpRequest.prototype.send; + XMLHttpRequest.prototype.send = function() { + pending++; + this.addEventListener('loadend', () => { pending--; check(); }); + return origXHR.apply(this, arguments); + }; + check(); + }) + '''); + final raw = result['result']?['value']; + if (raw == null) return {"idle": true}; + if (raw is Map) return Map.from(raw); + if (raw is String) return jsonDecode(raw) as Map; + return {"idle": true}; + } + + // ── Session/tab storage ── + + Future> getSessionStorage() async { + final result = await _evalJs( + 'JSON.stringify(Object.fromEntries(Object.entries(sessionStorage)))'); + final v = result['result']?['value'] as String?; + if (v == null) return {}; + return jsonDecode(v) as Map; + } + + /// Get all open handles (window.open references). + Future> getWindowHandles() async { + final result = await _evalJs('window.length'); + return {"window_count": result['result']?['value'] ?? 1}; + } +} diff --git a/packaging/npm/dart/lib/src/bridge/cdp_driver.dart b/packaging/npm/dart/lib/src/bridge/cdp_driver.dart new file mode 100644 index 00000000..0aea02e5 --- /dev/null +++ b/packaging/npm/dart/lib/src/bridge/cdp_driver.dart @@ -0,0 +1,3650 @@ +library; + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:math'; + +import 'package:http/http.dart' as http; + +import '../drivers/app_driver.dart'; +import 'device_presets.dart'; + +part 'cdp_browser_methods.dart'; +part 'cdp_appmcp_methods.dart'; + +/// Segment of text for typeText: either a control char or printable text run. +class _TypeSegment { + final String text; + final bool isControl; + _TypeSegment(this.text, this.isControl); +} + +/// AppDriver that communicates with any web page via Chrome DevTools Protocol. +/// +/// No SDK injection needed — connects directly to Chrome's debugging port +/// and controls any web page (React, Vue, Angular, plain HTML, etc.). +class CdpDriver implements AppDriver { + final String _url; + int _port; + final bool _launchChrome; + final bool _headless; + final String? _chromePath; + final String? _proxy; + final bool _ignoreSsl; + final int _maxTabs; + + WebSocket? _ws; + bool _connected = false; + int _nextId = 1; + Process? _chromeProcess; + + /// Pending CDP calls keyed by request id. + final Map>> _pending = {}; + final Map _eventSubscriptions = {}; + final Map)>> _eventListeners = + {}; + bool _dialogHandlerInstalled = false; + bool _isChrome146ConsentPort = false; + final Map> _interceptRules = {}; + + /// Create a CDP driver. + /// + /// [url] is the page to navigate to. + /// [port] is the Chrome remote debugging port. + /// [launchChrome] whether to launch a new Chrome instance. + /// [headless] run Chrome in headless mode (default: false). + /// [chromePath] custom Chrome/Chromium executable path. + /// [proxy] proxy server URL (e.g. 'http://proxy:8080'). + /// [ignoreSsl] ignore SSL certificate errors. + /// [maxTabs] maximum number of tabs to allow (prevents runaway tab creation). + CdpDriver({ + required String url, + int port = 9222, + bool launchChrome = true, + bool headless = false, + String? chromePath, + String? proxy, + bool ignoreSsl = false, + int maxTabs = 20, + }) : _url = url, + _port = port, + _launchChrome = launchChrome, + _headless = headless, + _chromePath = chromePath, + _proxy = proxy, + _ignoreSsl = ignoreSsl, + _maxTabs = maxTabs; + + @override + String get frameworkName => 'CDP (Web)'; + + @override + bool get isConnected => _connected; + + /// Whether connect() found an existing tab matching the target URL + /// (skipped navigation to avoid duplicate tabs). + bool connectedToExistingTab = false; + + Future connect() async { + // Auto-assign random port if 0 + if (_port == 0) { + final server = await ServerSocket.bind(InternetAddress.loopbackIPv4, 0); + _port = server.port; + await server.close(); + } + + // Check if CDP debug port is already responding. + final cdpAlive = await _isCdpPortAlive(); + if (!cdpAlive) { + final chromeRunning = await _isChromeRunning(); + if (chromeRunning) { + // Step 1: Chrome may already have remote debugging enabled on a + // different port (Chrome 136+ picks a random port when the + // chrome://inspect/#remote-debugging checkbox is ticked). + // Scan all Chrome TCP ports before touching anything. + final existingPort = await _discoverChromeCdpPort(); + if (existingPort != null) { + _port = existingPort; + // CDP is already live — fall through to _discoverTarget(). + } else if (_launchChrome) { + // Step 2: Try to enable via chrome://inspect/#remote-debugging. + // Navigate Chrome to that page, tick the checkbox via the + // macOS Accessibility API, then discover the new port. + // This is zero-disruption: no restart, all tabs/sessions intact. + final enabledPort = await _enableChromeRemoteDebugging(); + if (enabledPort != null) { + _port = enabledPort; + } else { + // Step 3: Fallback — quit Chrome and relaunch with debug port. + await _restartChromeWithDebugPort(); + await _waitForCdpReady(); + } + } + // launch_chrome: false + no debug port found → fall through to error. + } else if (_launchChrome) { + // Chrome is not running — launch a fresh flutter-skill profile. + await _launchChromeProcess(); + await _waitForCdpReady(); + } + // else: Chrome not running + launch_chrome: false → fall through to error. + } + + // Discover tabs via CDP JSON endpoint + var wsUrl = await _discoverTarget(); + + // No suitable tab found — create a new one via CDP HTTP API + if (wsUrl == null && _url.isNotEmpty) { + try { + final client = HttpClient() + ..connectionTimeout = const Duration(seconds: 2); + final encodedUrl = Uri.encodeComponent(_url); + // Chrome 145+ requires PUT for /json/new + final request = await client.openUrl( + 'PUT', Uri.parse('http://127.0.0.1:$_port/json/new?$encodedUrl')); + final response = await request.close(); + final body = await response.transform(utf8.decoder).join(); + client.close(); + final tab = jsonDecode(body) as Map; + wsUrl = tab['webSocketDebuggerUrl'] as String?; + // Tab was just created with our URL — skip later navigation + if (wsUrl != null) connectedToExistingTab = true; + } catch (_) {} + } + + if (wsUrl == null) { + final chromeRunning = await _isChromeRunning(); + final hint = chromeRunning + ? 'Chrome is running but remote debugging is not enabled. ' + 'flutter-skill tried to auto-launch a debug profile but could not ' + 'connect. Try: connect_cdp(url: "$_url", launch_chrome: true)' + : 'Chrome is not running. ' + 'Use connect_cdp(url: "$_url") to auto-launch Chrome with remote ' + 'debugging enabled.'; + throw Exception(hint); + } + + _ws = await _connectCdpWebSocket(wsUrl); + _connected = true; + + _ws!.listen( + _onMessage, + onDone: _onDisconnect, + onError: (_) => _onDisconnect(), + cancelOnError: false, + ); + + // Enable required CDP domains in parallel. + // Each call has an individual timeout so a bad/special tab (e.g. chrome:// + // pages, crashed renderers) cannot hang the entire connect() forever. + await Future.wait([ + _call('Page.enable') + .timeout(const Duration(seconds: 10), onTimeout: () => {}), + _call('DOM.enable') + .timeout(const Duration(seconds: 10), onTimeout: () => {}), + _call('Runtime.enable') + .timeout(const Duration(seconds: 10), onTimeout: () => {}), + ]); + + // Check if _discoverTarget already found a tab with our exact URL. + // If so, skip navigation to avoid reloading/duplicating. + final currentUrl = await _getCurrentUrl(); + final alreadyOnTarget = currentUrl == _url || + (currentUrl != null && _url.isNotEmpty && currentUrl.startsWith(_url)); + + // Also check root domain match (e.g. passport.csdn.net redirected to csdn.net) + final sameRootDomain = !alreadyOnTarget && + currentUrl != null && + _url.isNotEmpty && + (() { + final targetHost = Uri.tryParse(_url)?.host ?? ''; + final currentHost = Uri.tryParse(currentUrl)?.host ?? ''; + if (targetHost.isEmpty || currentHost.isEmpty) return false; + final tp = targetHost.split('.'); + final cp = currentHost.split('.'); + final tr = + tp.length >= 2 ? tp.sublist(tp.length - 2).join('.') : targetHost; + final cr = cp.length >= 2 + ? cp.sublist(cp.length - 2).join('.') + : currentHost; + return tr == cr; + })(); + + // Navigate to URL and wait for load event. + final skipNav = _url.isEmpty || + _url == 'about:blank' || + _url.contains('localhost:$_port') || + _url.contains('127.0.0.1:$_port') || + alreadyOnTarget || + (connectedToExistingTab && sameRootDomain); + if (!skipNav) { + await _call('Page.navigate', {'url': _url}); + try { + await _waitForLoad(); + } catch (_) { + // Timeout is acceptable — page may be slow but still usable + } + } else if (alreadyOnTarget) { + connectedToExistingTab = true; + } + } + + /// Check if CDP port is already responding. + /// Handles both standard CDP (HTTP /json/version) and Chrome 146+'s + /// consent-based port (WebSocket /devtools/browser/{uuid} — no HTTP). + Future _isCdpPortAlive() async { + // Standard CDP: HTTP /json/version + try { + final client = HttpClient() + ..connectionTimeout = const Duration(seconds: 2); + final request = await client + .getUrl(Uri.parse('http://127.0.0.1:$_port/json/version')); + final response = await request.close(); + await response.drain(); + client.close(); + _isChrome146ConsentPort = false; + return true; + } catch (_) {} + + // Chrome 146+ consent port: HTTP returns 404, but WebSocket to + // /devtools/browser/{uuid} either connects or hangs (waiting for Allow). + // A quick 1s probe: if the socket connects (not refused), port is alive. + try { + final sock = await Socket.connect( + InternetAddress.loopbackIPv4, + _port, + timeout: const Duration(seconds: 1), + ); + await sock.close(); + // Port is open. Check if it's a Chrome 146 consent port by probing /json/version. + // 404 = Chrome 146 consent port. Connection refused = not alive. + _isChrome146ConsentPort = true; + return true; + } catch (_) {} + + return false; + } + + /// Check if a Chrome/Chromium process is running (any instance, debug port or not). + /// Used to decide whether to auto-launch a flutter-skill Chrome profile when the + /// user's Chrome is running without --remote-debugging-port. + Future _isChromeRunning() async { + try { + ProcessResult result; + if (Platform.isMacOS) { + result = await Process.run('pgrep', ['-x', 'Google Chrome']); + if (result.exitCode == 0) return true; + // Also check Chromium + result = await Process.run('pgrep', ['-x', 'Chromium']); + return result.exitCode == 0; + } else if (Platform.isLinux) { + result = await Process.run('pgrep', ['-x', 'google-chrome']); + if (result.exitCode == 0) return true; + result = await Process.run('pgrep', ['-x', 'chromium']); + return result.exitCode == 0; + } else if (Platform.isWindows) { + result = await Process.run( + 'tasklist', ['/FI', 'IMAGENAME eq chrome.exe', '/NH']); + return result.stdout.toString().contains('chrome.exe'); + } + } catch (_) {} + return false; + } + + /// Scan all TCP ports that Chrome is currently listening on and return the + /// first port that responds as a valid CDP endpoint. + /// Handles Chrome 136+ which assigns a random port when the user enables + /// chrome://inspect/#remote-debugging (not necessarily 9222). + Future _discoverChromeCdpPort() async { + try { + final result = await Process.run('bash', + ['-c', 'lsof -c "Google Chrome" -a -i TCP -sTCP:LISTEN 2>/dev/null']); + final portRegex = RegExp(r'127\.0\.0\.1:(\d+)'); + final ports = portRegex + .allMatches(result.stdout.toString()) + .map((m) => int.tryParse(m.group(1)!) ?? 0) + .where((p) => p > 1024) + .toSet() + .toList(); + for (final port in ports) { + final prev = _port; + _port = port; + if (await _isCdpPortAlive()) return port; + _port = prev; + } + } catch (_) {} + return null; + } + + /// Enable Chrome remote debugging via the chrome://inspect/#remote-debugging + /// page WITHOUT restarting Chrome. + /// + /// Chrome 136+ added a "Allow remote debugging for this browser instance" + /// checkbox on that page. This method: + /// 1. Navigates Chrome to chrome://inspect/#remote-debugging + /// 2. Ticks the checkbox via the macOS Accessibility API (AXCheckBox) + /// 3. Scans Chrome's new listening port and returns it + /// + /// Returns the enabled CDP port, or null if the approach failed (caller + /// should fall back to restarting Chrome). + Future _enableChromeRemoteDebugging() async { + if (!Platform.isMacOS) return null; + try { + // Snapshot of ports before we do anything (to detect the new one). + final portsBefore = await _getChromeTcpPorts(); + + // Navigate to the remote-debugging settings page. + await Process.run('osascript', [ + '-e', + 'tell application "Google Chrome" to open location ' + '"chrome://inspect/#remote-debugging"' + ]); + await Process.run( + 'osascript', ['-e', 'tell application "Google Chrome" to activate']); + await Future.delayed(const Duration(milliseconds: 2000)); + + // Try to tick the checkbox via the macOS Accessibility API. + // Chrome DevTools WebUI pages expose their content as AX elements. + final axResult = await Process.run('osascript', [ + '-e', + r''' +tell application "System Events" + tell process "Google Chrome" + activate + try + set allElems to entire contents of front window + repeat with e in allElems + try + if role of e is "AXCheckBox" then + if value of e is 0 then + click e + end if + return "ok:" & (value of e as string) + end if + end try + end repeat + end try + return "notfound" + end tell +end tell +''' + ]).timeout(const Duration(seconds: 8), onTimeout: () { + return ProcessResult(0, 0, 'timeout', ''); + }); + + final axOut = axResult.stdout.toString().trim(); + if (axOut == 'notfound' || axOut == 'timeout') { + // AX approach failed — try clicking at the approximate screen position + // of the checkbox (Chrome devtools page layout is consistent). + final boundsResult = await Process.run('osascript', [ + '-e', + ''' +tell application "System Events" + tell process "Google Chrome" + set w to front window + return ((item 1 of position of w) as string) & "," & ¬ + ((item 2 of position of w) as string) & "," & ¬ + ((item 1 of size of w) as string) & "," & ¬ + ((item 2 of size of w) as string) + end tell +end tell +''' + ]); + final parts = boundsResult.stdout + .toString() + .trim() + .split(',') + .map((s) => double.tryParse(s.trim())) + .whereType() + .toList(); + if (parts.length >= 4) { + // Checkbox is roughly at: sidebar(230) + margin(30), toolbar(130) + 100 + final cx = (parts[0] + 265).toInt(); + final cy = (parts[1] + 230).toInt(); + await Process.run('osascript', [ + '-e', + 'tell application "System Events" to click at {$cx, $cy}' + ]); + } + } + + await Future.delayed(const Duration(milliseconds: 1200)); + + // Discover the newly opened CDP port (port that wasn't there before). + final portsAfter = await _getChromeTcpPorts(); + final newPorts = + portsAfter.where((p) => !portsBefore.contains(p)).toList(); + + // Check new ports first, then all Chrome ports. + for (final port in [...newPorts, ...portsAfter]) { + _port = port; + if (await _isCdpPortAlive()) return port; + } + } catch (_) {} + return null; + } + + /// Returns all TCP ports Chrome is currently listening on (127.0.0.1 only). + Future> _getChromeTcpPorts() async { + try { + final result = await Process.run('bash', + ['-c', 'lsof -c "Google Chrome" -a -i TCP -sTCP:LISTEN 2>/dev/null']); + final portRegex = RegExp(r'127\.0\.0\.1:(\d+)'); + return portRegex + .allMatches(result.stdout.toString()) + .map((m) => int.tryParse(m.group(1)!) ?? 0) + .where((p) => p > 1024) + .toSet() + .toList(); + } catch (_) { + return []; + } + } + + /// Returns the OS-default Chrome user-data-dir (the user's actual profile). + String _defaultChromeUserDataDir() { + final home = Platform.environment['HOME'] ?? + Platform.environment['USERPROFILE'] ?? + ''; + if (Platform.isMacOS) { + return '$home/Library/Application Support/Google/Chrome'; + } else if (Platform.isLinux) { + return '$home/.config/google-chrome'; + } else if (Platform.isWindows) { + final appData = + Platform.environment['LOCALAPPDATA'] ?? '$home/AppData/Local'; + return '$appData\\Google\\Chrome\\User Data'; + } + return '$home/.config/google-chrome'; + } + + /// Path for the session-copy profile directory. + /// Uses a non-default path so Chrome 136+ allows --remote-debugging-port. + static String get _sessionCopyProfileDir { + final home = Platform.environment['HOME'] ?? + Platform.environment['USERPROFILE'] ?? + '.'; + return '$home/.flutter-skill/chrome-session'; + } + + /// Find the most recently modified profile subdirectory within [userDataDir]. + /// Checks Default, Profile 1 … Profile 9, and returns the one whose Cookies + /// file was most recently modified (best proxy for the active profile). + Future _findActiveProfileDir(String userDataDir) async { + final names = [ + 'Default', + ...List.generate(9, (i) => 'Profile ${i + 1}'), + ]; + String? latest; + DateTime? latestMod; + for (final name in names) { + final cookies = File('$userDataDir/$name/Cookies'); + if (!cookies.existsSync()) continue; + final mod = cookies.statSync().modified; + if (latestMod == null || mod.isAfter(latestMod)) { + latestMod = mod; + latest = '$userDataDir/$name'; + } + } + return latest; + } + + /// Copy a directory recursively, skipping locked / unreadable files silently. + Future _copyDirectory(Directory src, Directory dest) async { + if (!dest.existsSync()) dest.createSync(recursive: true); + try { + await for (final entity in src.list()) { + final name = entity.uri.pathSegments + .lastWhere((s) => s.isNotEmpty, orElse: () => ''); + if (name.isEmpty) continue; + if (entity is File) { + try { + await entity.copy('${dest.path}/$name'); + } catch (_) {} + } else if (entity is Directory) { + await _copyDirectory(entity, Directory('${dest.path}/$name')); + } + } + } catch (_) {} + } + + /// Create (or refresh) a session-copy profile at [_sessionCopyProfileDir] by + /// copying the essential login/cookie files from the user's active Chrome profile. + /// Returns the path to use as --user-data-dir when launching Chrome. + Future _createSessionCopyProfile(String userDataDir) async { + final destRoot = _sessionCopyProfileDir; + final destProfile = '$destRoot/Default'; + await Directory(destProfile).create(recursive: true); + + final sourceProfile = await _findActiveProfileDir(userDataDir); + if (sourceProfile != null) { + // Essential files for session preservation. + for (final filename in [ + 'Cookies', + 'Extension Cookies', + 'Login Data', + 'Login Data For Account', + 'Preferences', + 'Secure Preferences', + ]) { + final src = File('$sourceProfile/$filename'); + if (src.existsSync()) { + try { + await src.copy('$destProfile/$filename'); + } catch (_) {} + } + } + // Local Storage contains IndexedDB / localStorage session tokens. + final lsDir = Directory('$sourceProfile/Local Storage'); + if (lsDir.existsSync()) { + await _copyDirectory(lsDir, Directory('$destProfile/Local Storage')); + } + } + + // Local State holds multi-profile config and the encryption key reference. + final localState = File('$userDataDir/Local State'); + if (localState.existsSync()) { + try { + await localState.copy('$destRoot/Local State'); + } catch (_) {} + } + + return destRoot; + } + + /// Quit Chrome gracefully (AppleScript on macOS, SIGTERM on Linux, taskkill on Windows). + Future _quitChrome() async { + try { + if (Platform.isMacOS) { + await Process.run( + 'osascript', ['-e', 'tell application "Google Chrome" to quit']); + } else if (Platform.isLinux) { + await Process.run('pkill', ['-x', 'google-chrome']); + await Process.run('pkill', ['-x', 'chromium']); + } else if (Platform.isWindows) { + await Process.run('taskkill', ['/F', '/IM', 'chrome.exe']); + } + } catch (_) {} + } + + /// Poll until Chrome process is no longer running (max 3 seconds). + Future _waitForChromeExit() async { + for (var i = 0; i < 30; i++) { + if (!await _isChromeRunning()) return; + await Future.delayed(const Duration(milliseconds: 100)); + } + } + + /// Quit the user's running Chrome and relaunch it with the same profile + + /// --remote-debugging-port, so remote debugging is enabled without losing + /// the user's existing session / logins. + /// + /// Chrome 136+ blocks --remote-debugging-port on the default user-data-dir. + /// If that happens (process exits fast), we automatically fall back to + /// launching with a separate flutter-skill profile (no user session). + Future _restartChromeWithDebugPort() async { + final userDataDir = _defaultChromeUserDataDir(); + + // Quit the running Chrome instance. + await _quitChrome(); + await _waitForChromeExit(); + + // Find the Chrome executable (same search order as _launchChromeProcess). + String? chromePath; + if (_chromePath != null) { + chromePath = _chromePath; + } else if (Platform.isMacOS) { + chromePath = + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'; + } else if (Platform.isLinux) { + for (final p in ['google-chrome', 'google-chrome-stable', 'chromium']) { + final r = await Process.run('which', [p]); + if (r.exitCode == 0) { + chromePath = r.stdout.toString().trim(); + break; + } + } + } else if (Platform.isWindows) { + chromePath = r'C:\Program Files\Google\Chrome\Application\chrome.exe'; + } + + if (chromePath == null) { + await _launchChromeProcess(); + return; + } + + final args = [ + '--remote-debugging-port=$_port', + '--remote-allow-origins=*', + '--user-data-dir=$userDataDir', + '--no-first-run', + '--no-default-browser-check', + if (_headless) '--headless=new', + if (_proxy != null) '--proxy-server=$_proxy', + if (_ignoreSsl) '--ignore-certificate-errors', + if (_url.isNotEmpty) _url, + ]; + + try { + if (Platform.isMacOS && chromePath.contains('.app/')) { + final appBundle = chromePath.substring( + 0, chromePath.indexOf('.app/') + '.app/'.length - 1); + await Process.run('xattr', ['-cr', appBundle]); + } + _chromeProcess = await Process.start(chromePath, args); + } catch (_) { + // Could not start Chrome — fall back to flutter-skill profile. + await _launchChromeProcess(); + return; + } + + // Chrome 136+ may reject --remote-debugging-port on the default user-data-dir + // in two ways: + // a) Exit immediately (early Chrome 136 behaviour) + // b) Stay alive but silently ignore the flag (Chrome 145+ behaviour) + // Check both: process exit AND port actually opening. + await Future.delayed(const Duration(milliseconds: 800)); + final code = await _chromeProcess!.exitCode + .timeout(const Duration(milliseconds: 200), onTimeout: () => -1); + if (code != -1) { + // Chrome exited immediately — rejected the debug port. + _chromeProcess = null; + await _launchChromeProcess(); + return; + } + + // Chrome is still running — verify the debug port actually opened. + final portOpened = await _pollCdpPort( + timeout: const Duration(seconds: 4), + interval: const Duration(milliseconds: 200)); + if (!portOpened) { + // Chrome silently ignored --remote-debugging-port (Chrome 145+ behaviour). + // Kill this instance, then try session-copy profile (copies user's cookies/storage + // to a non-default path so Chrome allows the debug port while preserving logins). + try { + _chromeProcess?.kill(); + } catch (_) {} + _chromeProcess = null; + + try { + final sessionDir = await _createSessionCopyProfile(userDataDir); + final sessionArgs = [ + '--remote-debugging-port=$_port', + '--remote-allow-origins=*', + '--user-data-dir=$sessionDir', + '--no-first-run', + '--no-default-browser-check', + if (_headless) '--headless=new', + if (_proxy != null) '--proxy-server=$_proxy', + if (_ignoreSsl) '--ignore-certificate-errors', + if (_url.isNotEmpty) _url, + ]; + _chromeProcess = await Process.start(chromePath, sessionArgs); + final sessionPortOpened = await _pollCdpPort( + timeout: const Duration(seconds: 6), + interval: const Duration(milliseconds: 200), + ); + if (sessionPortOpened) + return; // Session copy worked with debug port open. + try { + _chromeProcess?.kill(); + } catch (_) {} + _chromeProcess = null; + } catch (_) {} + + // Final fallback: flutter-skill's own blank profile (no user session). + await _launchChromeProcess(); + } + // else: Chrome accepted and port is open — running with user's profile. + } + + /// Get the current page URL via Runtime.evaluate + Future _getCurrentUrl() async { + try { + final result = await _call('Runtime.evaluate', { + 'expression': 'location.href', + 'returnByValue': true, + }); + return result['result']?['value'] as String?; + } catch (_) { + return null; + } + } + + /// Reconnect to a new WebSocket URL (e.g., after target navigates away) + Future reconnectTo(String wsUrl) async { + _connected = false; + _reconnecting = true; // Prevent _autoReconnect from overriding this + _failAllPending('Reconnecting'); + try { + await _ws?.close(); + } catch (_) {} + _ws = null; + + _ws = await _connectCdpWebSocket(wsUrl); + _connected = true; + + _ws!.listen( + _onMessage, + onDone: _onDisconnect, + onError: (_) => _onDisconnect(), + cancelOnError: false, + ); + + // Re-enable required CDP domains + await Future.wait([ + _call('Page.enable'), + _call('DOM.enable'), + _call('Runtime.enable'), + ]); + _reconnecting = false; + } + + @override + Future disconnect() async { + _connected = false; + _failAllPending('Disconnected'); + try { + await _ws?.close(); + } catch (_) {} + _ws = null; + + if (_chromeProcess != null) { + _chromeProcess!.kill(); + _chromeProcess = null; + } + } + + @override + Future> tap( + {String? key, String? text, String? ref}) async { + // Find element and get its center coordinates + final selector = _buildSelector(key: key, text: text, ref: ref); + final bounds = await _getElementBounds(selector, text: text); + if (bounds == null) { + return { + 'success': false, + 'error': {'message': 'Element not found: ${key ?? text ?? ref}'}, + }; + } + + final cx = bounds['cx'] as double; + final cy = bounds['cy'] as double; + + await _dispatchMouseEvent('mousePressed', cx, cy, + button: 'left', clickCount: 1); + await _dispatchMouseEvent('mouseReleased', cx, cy, + button: 'left', clickCount: 1); + await _ensureFocusAtPoint(cx, cy); + + return { + 'success': true, + 'position': {'x': cx, 'y': cy}, + }; + } + + @override + Future> enterText(String? key, String text, + {String? ref}) async { + // Focus the element first + if (key != null || ref != null) { + final selector = _buildSelector(key: key, ref: ref); + final result = await _evalJs(''' + (() => { + const el = document.querySelector('$selector'); + if (!el) return false; + el.focus(); + el.value = ''; + return true; + })() + '''); + if (result['result']?['value'] != true) { + return { + 'success': false, + 'error': {'message': 'Element not found: ${key ?? ref}'}, + }; + } + } + + // Type each character + for (final char in text.codeUnits) { + await _call('Input.dispatchKeyEvent', { + 'type': 'keyDown', + 'text': String.fromCharCode(char), + 'key': String.fromCharCode(char), + 'unmodifiedText': String.fromCharCode(char), + }); + await _call('Input.dispatchKeyEvent', { + 'type': 'keyUp', + 'key': String.fromCharCode(char), + }); + } + + // Dispatch input event for frameworks that listen to it + if (key != null || ref != null) { + final selector = _buildSelector(key: key, ref: ref); + await _evalJs(''' + (() => { + const el = document.querySelector('$selector'); + if (el) { + el.dispatchEvent(new Event('input', {bubbles: true})); + el.dispatchEvent(new Event('change', {bubbles: true})); + } + })() + '''); + } + + return {'success': true}; + } + + @override + Future swipe( + {required String direction, double distance = 300, String? key}) async { + // Get viewport dimensions + final metrics = await _call('Page.getLayoutMetrics'); + final vw = + (metrics['cssLayoutViewport']?['clientWidth'] as num?)?.toDouble() ?? + 800.0; + final vh = + (metrics['cssLayoutViewport']?['clientHeight'] as num?)?.toDouble() ?? + 600.0; + + double startX = vw / 2, startY = vh / 2, endX = vw / 2, endY = vh / 2; + + switch (direction) { + case 'up': + startY = vh / 2 + distance / 2; + endY = vh / 2 - distance / 2; + break; + case 'down': + startY = vh / 2 - distance / 2; + endY = vh / 2 + distance / 2; + break; + case 'left': + startX = vw / 2 + distance / 2; + endX = vw / 2 - distance / 2; + break; + case 'right': + startX = vw / 2 - distance / 2; + endX = vw / 2 + distance / 2; + break; + } + + await _dispatchMouseEvent('mousePressed', startX, startY, button: 'left'); + await _dispatchMouseEvent('mouseMoved', endX, endY, button: 'left'); + await _dispatchMouseEvent('mouseReleased', endX, endY, button: 'left'); + return true; + } + + @override + Future> getInteractiveElements( + {bool includePositions = true}) async { + final result = await _evalJs(''' + (() => { + const selectors = 'a, button, input, select, textarea, [role="button"], [role="link"], [role="checkbox"], [role="radio"], [role="tab"], [onclick], [tabindex], [contenteditable="true"]'; + // Recursive query that traverses Shadow DOM + function deepQueryAll(root, sel) { + const results = Array.from(root.querySelectorAll(sel)); + root.querySelectorAll('*').forEach(el => { + if (el.shadowRoot) results.push(...deepQueryAll(el.shadowRoot, sel)); + }); + return results; + } + const elements = deepQueryAll(document, selectors); + return elements.filter(el => { + const style = window.getComputedStyle(el); + return style.display !== 'none' && style.visibility !== 'hidden' && el.offsetParent !== null; + }).map((el, i) => { + const rect = el.getBoundingClientRect(); + const tag = el.tagName.toLowerCase(); + const type = el.getAttribute('type') || ''; + const role = el.getAttribute('role') || ''; + const text = (el.textContent || '').trim().substring(0, 100); + const id = el.id || ''; + const name = el.getAttribute('name') || ''; + const ariaLabel = el.getAttribute('aria-label') || ''; + const placeholder = el.getAttribute('placeholder') || ''; + const value = el.value || ''; + return { + index: i, + tag: tag, + type: type || tag, + role: role, + text: text, + key: id || name || null, + label: ariaLabel || placeholder || '', + value: value, + bounds: { + x: Math.round(rect.left), + y: Math.round(rect.top), + w: Math.round(rect.width), + h: Math.round(rect.height) + }, + center: { + x: Math.round(rect.left + rect.width / 2), + y: Math.round(rect.top + rect.height / 2) + }, + visible: true, + clickable: true, + coordinatesReliable: true + }; + }); + })() + '''); + + final value = result['result']?['value']; + if (value is List) return value; + // If result is a remote object, need to get properties + final objectId = result['result']?['objectId'] as String?; + if (objectId != null) { + final props = await _call('Runtime.getProperties', { + 'objectId': objectId, + 'ownProperties': true, + }); + // Parse array-like properties + final elements = []; + for (final prop in (props['result'] as List? ?? [])) { + if (prop is Map && + prop['value']?['type'] == 'object' && + prop['name'] != '__proto__' && + prop['name'] != 'length') { + final elObjId = prop['value']['objectId'] as String?; + if (elObjId != null) { + final elProps = await _call('Runtime.getProperties', { + 'objectId': elObjId, + 'ownProperties': true, + }); + final map = {}; + for (final p in (elProps['result'] as List? ?? [])) { + if (p is Map && p['name'] != '__proto__') { + final v = p['value']; + if (v?['type'] == 'object' && v?['objectId'] != null) { + // nested object (bounds, center) — get properties + final nested = await _call('Runtime.getProperties', { + 'objectId': v['objectId'], + 'ownProperties': true, + }); + final nestedMap = {}; + for (final np in (nested['result'] as List? ?? [])) { + if (np is Map && np['name'] != '__proto__') { + nestedMap[np['name']] = np['value']?['value']; + } + } + map[p['name']] = nestedMap; + } else { + map[p['name']] = v?['value']; + } + } + } + elements.add(map); + } + } + } + return elements; + } + return []; + } + + @override + Future> getInteractiveElementsStructured() async { + final result = await _evalJs(''' + (() => { + const selectors = 'a, button, input, select, textarea, [role="button"], [role="link"], [role="checkbox"], [role="radio"], [role="tab"], [onclick], [tabindex], [contenteditable="true"]'; + function deepQueryAll(root, sel) { + const results = Array.from(root.querySelectorAll(sel)); + root.querySelectorAll('*').forEach(el => { + if (el.shadowRoot) results.push(...deepQueryAll(el.shadowRoot, sel)); + }); + return results; + } + const elements = deepQueryAll(document, selectors); + const visible = elements.filter(el => { + const style = window.getComputedStyle(el); + return style.display !== 'none' && style.visibility !== 'hidden' && el.offsetParent !== null; + }); + const mapped = visible.map((el, i) => { + const rect = el.getBoundingClientRect(); + const tag = el.tagName.toLowerCase(); + const type = el.getAttribute('type') || ''; + const role = el.getAttribute('role') || tag; + const text = (el.textContent || '').trim().substring(0, 100); + const id = el.id || ''; + const name = el.getAttribute('name') || ''; + const ariaLabel = el.getAttribute('aria-label') || ''; + const placeholder = el.getAttribute('placeholder') || ''; + + let refName = ''; + if (tag === 'button' || role === 'button') refName = 'button:' + (text || ariaLabel || id || ('idx_' + i)); + else if (tag === 'input' || tag === 'textarea') refName = 'input:' + (ariaLabel || placeholder || name || id || ('idx_' + i)); + else if (tag === 'a' || role === 'link') refName = 'link:' + (text || ariaLabel || id || ('idx_' + i)); + else if (tag === 'select') refName = 'select:' + (ariaLabel || name || id || ('idx_' + i)); + else refName = role + ':' + (text || ariaLabel || id || ('idx_' + i)); + + const actions = []; + actions.push('tap'); + if (tag === 'input' || tag === 'textarea') actions.push('enter_text'); + if (tag === 'select') actions.push('select'); + + return { + type: tag, + text: text, + label: ariaLabel || placeholder, + ref: refName, + selector: id ? ('#' + id) : (name ? ('[name="' + name + '"]') : null), + actions: actions, + bounds: { x: Math.round(rect.left), y: Math.round(rect.top), w: Math.round(rect.width), h: Math.round(rect.height) }, + enabled: !el.disabled, + visible: true, + value: el.value || null + }; + }); + + const summary = 'Found ' + mapped.length + ' interactive elements'; + return JSON.stringify({elements: mapped, summary: summary}); + })() + '''); + + final value = result['result']?['value']; + if (value is String) { + return (jsonDecode(value) as Map); + } + return {'elements': [], 'summary': 'Failed to inspect elements'}; + } + + /// Fast accessibility snapshot via single JS evaluation. + /// Scans all interactive + landmark elements, assigns ref IDs, returns compact text. + /// Benchmarked at ~10-50ms (vs 60ms+ for CDP Accessibility.getFullAXTree). + /// Pierces Shadow DOM automatically. + Future> getAccessibilitySnapshot() async { + final result = await _evalJs(r''' +(() => { + const t0 = performance.now(); + // Deep query helper for Shadow DOM + function dqAll(sel, root) { + root = root || document; + let r = Array.from(root.querySelectorAll(sel)); + for (const n of root.querySelectorAll('*')) { + if (n.shadowRoot) r = r.concat(dqAll(sel, n.shadowRoot)); + } + return r; + } + + // ===== Detect page-level context ===== + const framework = (() => { + if (window.__NEXT_DATA__) return 'nextjs'; + if (window.__NUXT__) return 'nuxt'; + if (document.querySelector('[ng-version]')) return 'angular'; + if (document.querySelector('[data-reactroot]') || document.querySelector('#__next')) return 'react'; + if (document.querySelector('[data-v-]')) return 'vue'; + return 'unknown'; + })(); + + // Detect editor type + const editorType = (() => { + if (document.querySelector('.CodeMirror')) return 'codemirror'; + if (document.querySelector('.cm-editor')) return 'codemirror6'; + if (document.querySelector('.DraftEditor-root')) return 'draft-js'; + if (document.querySelector('.tiptap.ProseMirror')) return 'tiptap'; + if (document.querySelector('.ProseMirror')) return 'prosemirror'; + if (document.querySelector('.ql-editor')) return 'quill'; + if (document.querySelector('[contenteditable="true"]')) return 'contenteditable'; + return 'none'; + })(); + + // Recommend best input method based on framework + editor + const inputMethod = (() => { + if (editorType === 'codemirror' || editorType === 'codemirror6') return 'api:CodeMirror.setValue()'; + if (editorType === 'draft-js') return 'clipboard:paste-event'; + if (editorType === 'tiptap' || editorType === 'prosemirror') return 'html:innerHTML+input-event'; + if (editorType === 'quill') return 'api:quill.clipboard.dangerouslyPasteHTML()'; + if (framework === 'react') return 'cdp:Input.insertText'; + return 'cdp:Input.insertText'; + })(); + + const interactiveSel = 'a,button,input,select,textarea,[role="button"],[role="link"],[role="textbox"],[role="searchbox"],[role="combobox"],[role="checkbox"],[role="radio"],[role="switch"],[role="tab"],[role="menuitem"],[role="option"],[role="slider"],[contenteditable="true"]'; + const landmarkSel = 'h1,h2,h3,h4,h5,h6,nav,main,header,footer,[role="heading"],[role="navigation"],[role="main"],[role="banner"],[role="complementary"],[role="dialog"],[role="alert"],[role="status"],img[alt],label,[class*="error"],[class*="warning"],[aria-invalid]'; + + const allEls = dqAll(interactiveSel + ',' + landmarkSel); + + // Also collect elements from same-origin iframes + const iframeEls = []; + const iframeOffsets = new WeakMap(); + try { + for (const iframe of document.querySelectorAll('iframe')) { + try { + const doc = iframe.contentDocument; + if (!doc) continue; + const iRect = iframe.getBoundingClientRect(); + if (iRect.width === 0 || iRect.height === 0) continue; + const sel = interactiveSel + ',' + landmarkSel; + for (const el of doc.querySelectorAll(sel)) { + iframeEls.push(el); + iframeOffsets.set(el, {dx: iRect.x, dy: iRect.y, src: iframe.src.substring(0, 60)}); + } + } catch(e) { /* cross-origin iframe — skip */ } + } + } catch(e) {} + // Second pass: find clickable custom elements inside shadow roots that the + // selector-based dqAll may have missed (e.g. Reddit's faceplate-button). + function findShadowInteractive(root) { + let results = []; + for (const el of root.querySelectorAll('*')) { + if (el.shadowRoot) { + for (const inner of el.shadowRoot.querySelectorAll('button, [role="button"], [type="submit"]')) { + if (!allEls.includes(inner)) results.push(inner); + } + results = results.concat(findShadowInteractive(el.shadowRoot)); + } + } + return results; + } + const shadowEls = findShadowInteractive(document); + const combinedEls = [...allEls, ...iframeEls, ...shadowEls]; + + const vw = window.innerWidth; + const vh = window.innerHeight; + let refN = 0; + const lines = []; + const refs = {}; + let interactiveCount = 0; + const requiredEmpty = []; + const errors = []; + + for (const el of combinedEls) { + const iframeOff = iframeOffsets.get(el); + const r = el.getBoundingClientRect(); + // For iframe elements, we store the offset but use raw rect for visibility check + if (r.width === 0 && r.height === 0) continue; + const s = getComputedStyle(el); + if (s.display === 'none' || s.visibility === 'hidden') continue; + + const tag = el.tagName.toLowerCase(); + const role = el.getAttribute('role') || ({'a':'link','button':'button','input':'textbox','select':'combobox','textarea':'textbox','h1':'heading','h2':'heading','h3':'heading','h4':'heading','h5':'heading','h6':'heading','nav':'navigation','img':'img','label':'label'}[tag] || tag); + const text = (el.textContent || '').trim().substring(0, 60); + const ariaLabel = el.getAttribute('aria-label') || ''; + const placeholder = el.getAttribute('placeholder') || ''; + const name = ariaLabel || placeholder || text; + const displayName = name.length > 55 ? name.substring(0, 52) + '...' : name; + const value = el.value || ''; + const type = el.getAttribute('type') || ''; + + const isInteractive = /^(link|button|textbox|searchbox|combobox|checkbox|radio|switch|tab|menuitem|option|slider)$/.test(role) || el.hasAttribute('contenteditable'); + + refN++; + const refId = 'e' + refN; + if (isInteractive) interactiveCount++; + + // ===== Enhanced state detection ===== + const states = []; + if (el.disabled) states.push('disabled'); + if (el.checked) states.push('checked'); + if (el.getAttribute('aria-expanded') === 'true') states.push('expanded'); + if (el.getAttribute('aria-selected') === 'true') states.push('selected'); + if (el.required || el.getAttribute('aria-required') === 'true') states.push('required'); + if (document.activeElement === el) states.push('focused'); + if (el.getAttribute('aria-invalid') === 'true') states.push('invalid'); + if (el.readOnly) states.push('readonly'); + + // Empty check for required fields + const isEmpty = (tag === 'input' || tag === 'textarea' || tag === 'select') && !value.trim(); + const isEmptyEditable = el.hasAttribute('contenteditable') && !el.textContent?.trim(); + if ((el.required || el.getAttribute('aria-required') === 'true') && (isEmpty || isEmptyEditable)) { + states.push('empty'); + requiredEmpty.push(displayName || role + '#' + refId); + } + + // ===== Validation info ===== + let validation = ''; + if (el.validity && !el.validity.valid && value) { + if (el.validity.tooShort) validation = ' minlen=' + el.minLength; + if (el.validity.tooLong) validation = ' maxlen=' + el.maxLength; + if (el.validity.patternMismatch) validation = ' pattern=' + el.pattern; + if (el.validity.typeMismatch) validation = ' invalid-format'; + } + if (el.minLength > 0) validation += ' minlen=' + el.minLength; + if (el.maxLength > 0 && el.maxLength < 10000) validation += ' maxlen=' + el.maxLength; + + // ===== Error message detection ===== + if (el.classList?.contains('error') || el.getAttribute('aria-invalid') === 'true' || + (el.className && /error|warning|invalid/i.test(el.className))) { + const errText = el.textContent?.trim(); + if (errText && errText.length < 100) errors.push(errText); + } + // Check aria-errormessage + const errMsgId = el.getAttribute('aria-errormessage') || el.getAttribute('aria-describedby'); + if (errMsgId) { + const errEl = document.getElementById(errMsgId); + if (errEl?.textContent?.trim()) errors.push(errEl.textContent.trim()); + } + + // ===== Select/dropdown options ===== + let optionsStr = ''; + if (tag === 'select') { + const opts = Array.from(el.options || []).map(o => o.text?.trim()).filter(Boolean).slice(0, 10); + if (opts.length) optionsStr = ' options=[' + opts.join(',') + ']'; + } + + // ===== Disabled button reason ===== + let disabledReason = ''; + if (el.disabled && role === 'button' && requiredEmpty.length > 0) { + disabledReason = ' reason="missing:' + requiredEmpty.join(',') + '"'; + } + + const stateStr = states.length ? ' [' + states.join(',') + ']' : ''; + const valueStr = value && type !== 'password' ? ' value="' + value.substring(0, 30) + '"' : ''; + const typeStr = type && type !== 'text' ? ' type=' + type : ''; + const refStr = isInteractive ? ' [ref=' + refId + ']' : ''; + + // Viewport indicator + const inView = r.top >= -10 && r.bottom <= vh + 10; + const offscreen = !inView && (r.bottom < -100 || r.top > vh + 100) ? ' (offscreen)' : ''; + // Beyond viewport width (important for dialog buttons) + const beyondVW = r.left > vw ? ' (beyond-viewport-x:' + Math.round(r.left) + ')' : ''; + + refs[refId] = displayName; + const iframeTag = iframeOff ? ' (iframe:' + iframeOff.src + ')' : ''; + lines.push(role + ' "' + displayName + '"' + typeStr + valueStr + optionsStr + refStr + stateStr + validation + disabledReason + offscreen + beyondVW + iframeTag); + } + + const elapsed = Math.round(performance.now() - t0); + const snapshot = lines.join('\n'); + + // ===== Form summary ===== + const formMeta = { + framework: framework, + editorType: editorType, + inputMethod: inputMethod, + requiredEmpty: requiredEmpty, + errors: errors, + viewport: vw + 'x' + vh, + }; + + return JSON.stringify({ + snapshot: snapshot, + interactiveCount: interactiveCount, + totalElements: refN, + tokenEstimate: Math.round(snapshot.length / 4), + elapsedMs: elapsed, + formMeta: formMeta, + hint: requiredEmpty.length > 0 + ? 'BLOCKED: Required empty fields: [' + requiredEmpty.join(', ') + ']. Fill these before submit. Use ' + inputMethod + ' for input.' + : 'Ready to submit. Use act(ref, action) to interact.', + refs: refs + }); +})() + '''); + + final parsed = _parseJsonEval(result); + if (parsed != null) return parsed; + // Fallback + return await getInteractiveElementsStructured(); + } + + /// Fast composite action — find + scroll + act in a SINGLE JS evaluation. + /// Benchmarked: ~1-15ms for click, ~2-20ms for fill (vs ~1000ms before). + /// Falls back to CDP Input.dispatch for actions that need real mouse events. + Future> act({ + String? ref, + String? text, + String? key, + required String action, + String? value, + int timeoutMs = 5000, + bool dispatchRealEvents = false, + }) async { + // Escape parameters for JS + final jsText = text + ?.replaceAll('\\', '\\\\') + .replaceAll("'", "\\'") + .replaceAll('\n', '\\n') ?? + ''; + final jsKey = key?.replaceAll('\\', '\\\\').replaceAll("'", "\\'") ?? ''; + final jsRef = ref?.replaceAll('\\', '\\\\').replaceAll("'", "\\'") ?? ''; + final jsValue = value + ?.replaceAll('\\', '\\\\') + .replaceAll("'", "\\'") + .replaceAll('\n', '\\n') ?? + ''; + + // Single JS eval: find + scroll + act + final result = await _evalJs(''' +(() => { + const t0 = performance.now(); + // Deep query helpers + function dq(sel, root) { + root = root || document; + let el = root.querySelector(sel); + if (el) return el; + for (const n of root.querySelectorAll('*')) { + if (n.shadowRoot) { el = dq(sel, n.shadowRoot); if (el) return el; } + } + return null; + } + function dqAll(sel, root) { + root = root || document; + let r = Array.from(root.querySelectorAll(sel)); + for (const n of root.querySelectorAll('*')) { + if (n.shadowRoot) r = r.concat(dqAll(sel, n.shadowRoot)); + } + return r; + } + + let el = null; + const refId = '$jsRef'; + const textQuery = '$jsText'; + const keyQuery = '$jsKey'; + + // Strategy 1: By CSS selector/key + if (keyQuery) { + el = dq(keyQuery) || dq('#' + keyQuery) || dq('[name="' + keyQuery + '"]') || dq('[data-testid="' + keyQuery + '"]'); + } + + // Strategy 2: By ref ID (e.g. "e5" from snapshot) + if (!el && refId) { + // Refs are positional — find the Nth interactive element + const refMatch = refId.match(/^e(\\d+)\$/); + if (refMatch) { + const idx = parseInt(refMatch[1]) - 1; + const interactiveSel = 'a,button,input,select,textarea,[role="button"],[role="link"],[role="textbox"],[role="searchbox"],[role="combobox"],[role="checkbox"],[role="radio"],[role="switch"],[role="tab"],[role="menuitem"],[role="option"],[role="slider"],[contenteditable="true"],h1,h2,h3,h4,h5,h6,nav,main,header,footer,[role="heading"],[role="navigation"],[role="main"],[role="banner"],[role="complementary"],[role="dialog"],[role="alert"],[role="status"],img[alt],label'; + const all = dqAll(interactiveSel); + const visible = all.filter(e => { + const r = e.getBoundingClientRect(); + if (r.width === 0 && r.height === 0) return false; + const s = getComputedStyle(e); + return s.display !== 'none' && s.visibility !== 'hidden'; + }); + if (idx < visible.length) el = visible[idx]; + } + // Also try by ref-like selector + if (!el) el = dq('[data-ref="' + refId + '"]'); + } + + // Strategy 3: By text content (exact then partial) + if (!el && textQuery) { + const all = dqAll('a,button,input,select,textarea,label,span,div,h1,h2,h3,h4,h5,h6,[role="button"],[role="link"],[role="tab"],[role="menuitem"],[role="option"]'); + for (const e of all) { + if (e.textContent && e.textContent.trim() === textQuery) { el = e; break; } + } + if (!el) { + for (const e of all) { + if (e.textContent && e.textContent.trim().includes(textQuery)) { el = e; break; } + } + } + } + + if (!el) { + return JSON.stringify({success: false, error: 'Element not found: ' + (refId || textQuery || keyQuery), elapsedMs: Math.round(performance.now() - t0)}); + } + + // Scroll into view if needed + const rect = el.getBoundingClientRect(); + if (rect.top < 0 || rect.bottom > window.innerHeight) { + el.scrollIntoView({behavior: 'instant', block: 'center'}); + } + + const action = '$action'; + const fillValue = '$jsValue'; + const cx = Math.round(rect.left + rect.width / 2); + const cy = Math.round(rect.top + rect.height / 2); + + switch (action) { + case 'click': + case 'tap': + el.focus(); + el.click(); + return JSON.stringify({success: true, action: 'click', position: {x: cx, y: cy}, elapsedMs: Math.round(performance.now() - t0)}); + + case 'fill': { + el.focus(); + if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.tagName === 'SELECT') { + el.value = ''; + el.value = fillValue; + el.dispatchEvent(new Event('input', {bubbles: true})); + el.dispatchEvent(new Event('change', {bubbles: true})); + } else if (el.isContentEditable || el.getAttribute('contenteditable') === 'true') { + el.innerHTML = fillValue; + el.dispatchEvent(new Event('input', {bubbles: true})); + } + return JSON.stringify({success: true, action: 'fill', value: fillValue, elapsedMs: Math.round(performance.now() - t0)}); + } + + case 'select': { + if (el.tagName === 'SELECT') { + el.value = fillValue; + el.dispatchEvent(new Event('change', {bubbles: true})); + } else { + el.click(); + } + return JSON.stringify({success: true, action: 'select', value: fillValue, elapsedMs: Math.round(performance.now() - t0)}); + } + + case 'hover': + el.dispatchEvent(new MouseEvent('mouseenter', {bubbles: true})); + el.dispatchEvent(new MouseEvent('mouseover', {bubbles: true})); + return JSON.stringify({success: true, action: 'hover', position: {x: cx, y: cy}, elapsedMs: Math.round(performance.now() - t0)}); + + case 'check': + el.click(); + return JSON.stringify({success: true, action: 'check', checked: el.checked, elapsedMs: Math.round(performance.now() - t0)}); + + default: + return JSON.stringify({success: false, error: 'Unknown action: ' + action}); + } +})() + '''); + + final parsed = _parseJsonEval(result); + if (parsed != null) { + // For click actions needing real mouse events (e.g., custom components), + // fall back to CDP Input.dispatch + if (dispatchRealEvents && + parsed['success'] == true && + parsed['position'] != null) { + final pos = parsed['position'] as Map; + final cx = (pos['x'] as num).toDouble(); + final cy = (pos['y'] as num).toDouble(); + await _dispatchMouseEvent('mousePressed', cx, cy, + button: 'left', clickCount: 1); + await _dispatchMouseEvent('mouseReleased', cx, cy, + button: 'left', clickCount: 1); + } + return parsed; + } + + return {'success': false, 'error': 'Failed to parse act result'}; + } + + @override + Future takeScreenshot({double quality = 1.0, int? maxWidth}) async { + // Default to JPEG@80 for speed; use PNG only when quality=1.0 explicitly AND no maxWidth + final useJpeg = quality < 1.0 || maxWidth != null; + final params = { + 'format': useJpeg + ? 'jpeg' + : 'jpeg', // Always JPEG for CDP — 3-5x faster than PNG + 'quality': (quality * 80).round().clamp(30, 100), + }; + if (maxWidth != null) { + // CDP supports clip parameter for region, use viewport scaling + params['optimizeForSpeed'] = true; + } + final result = await _call('Page.captureScreenshot', params); + return result['data'] as String?; + } + + @override + Future> getLogs() async { + // CDP doesn't have a built-in log store; return console messages if captured + return []; + } + + @override + Future clearLogs() async { + // No-op for CDP + } + + @override + Future hotReload() async { + await _call('Page.reload'); + } + + // ── Extended methods used by server.dart ── + + /// Take a screenshot of a specific region. + Future takeRegionScreenshot( + double x, double y, double width, double height) async { + try { + final result = await _call('Page.captureScreenshot', { + 'format': 'jpeg', + 'quality': 80, + 'clip': { + 'x': x, + 'y': y, + 'width': width, + 'height': height, + 'scale': 1, + }, + }).timeout(const Duration(seconds: 10)); + return result['data'] as String?; + } catch (_) { + // Fallback: full screenshot (clip times out on some Chrome versions) + final full = await takeScreenshot(quality: 0.8); + return full; + } + } + + /// Take a screenshot of a specific element. + Future takeElementScreenshot(String selector) async { + final bounds = await _getElementBounds(selector); + if (bounds == null) return null; + return takeRegionScreenshot( + bounds['x'] as double, + bounds['y'] as double, + bounds['w'] as double, + bounds['h'] as double, + ); + } + + /// Scroll an element into view. + Future> scrollTo({String? key, String? text}) async { + final selector = _buildSelector(key: key, text: text); + final result = await _evalJs(''' + (() => { + const el = ${_jsFindElement(selector, text: text)}; + if (!el) return false; + el.scrollIntoView({behavior: 'smooth', block: 'center'}); + return true; + })() + '''); + final found = result['result']?['value'] == true; + return { + 'success': found, + 'message': found ? 'Scrolled to element' : 'Element not found' + }; + } + + /// Navigate back. + Future goBack() async { + await _evalJs("history.back()"); + await Future.delayed(const Duration(milliseconds: 300)); + return true; + } + + /// Get current URL. + Future getCurrentRoute() async { + final result = await _evalJs('window.location.href'); + return (result['result']?['value'] as String?) ?? ''; + } + + /// Evaluate JavaScript. + Future> evaluate(String expression) async { + return _evalJs(expression); + } + + /// Wait for an element to appear. + Future waitForElement( + {String? key, String? text, int timeout = 5000}) async { + final selector = _buildSelector(key: key, text: text); + final sw = Stopwatch()..start(); + while (sw.elapsedMilliseconds < timeout) { + final result = await _evalJs(''' + (() => { + const el = ${_jsFindElement(selector, text: text)}; + return el !== null && el !== undefined; + })() + '''); + if (result['result']?['value'] == true) return true; + await Future.delayed(const Duration(milliseconds: 200)); + } + return false; + } + + /// Wait for an element to disappear. + Future waitForGone( + {String? key, String? text, int timeout = 5000}) async { + final selector = _buildSelector(key: key, text: text); + final sw = Stopwatch()..start(); + while (sw.elapsedMilliseconds < timeout) { + final result = await _evalJs(''' + (() => { + const el = ${_jsFindElement(selector, text: text)}; + return el === null || el === undefined; + })() + '''); + if (result['result']?['value'] == true) return true; + await Future.delayed(const Duration(milliseconds: 200)); + } + return false; + } + + /// Assert visibility of an element. + Future assertVisible({String? key, String? text}) async { + final selector = _buildSelector(key: key, text: text); + final result = await _evalJs(''' + (() => { + const el = ${_jsFindElement(selector, text: text)}; + if (!el) return false; + const style = window.getComputedStyle(el); + return style.display !== 'none' && style.visibility !== 'hidden' && el.offsetParent !== null; + })() + '''); + return result['result']?['value'] == true; + } + + /// Tap at specific coordinates. + Future tapAt(double x, double y) async { + await _dispatchMouseEvent('mousePressed', x, y, + button: 'left', clickCount: 1); + await _dispatchMouseEvent('mouseReleased', x, y, + button: 'left', clickCount: 1); + await _ensureFocusAtPoint(x, y); + } + + /// Long press an element. + Future longPress( + {String? key, String? text, int duration = 500}) async { + final selector = _buildSelector(key: key, text: text); + final bounds = await _getElementBounds(selector, text: text); + if (bounds == null) return false; + final cx = bounds['cx'] as double; + final cy = bounds['cy'] as double; + await _dispatchMouseEvent('mousePressed', cx, cy, button: 'left'); + await Future.delayed(Duration(milliseconds: duration)); + await _dispatchMouseEvent('mouseReleased', cx, cy, button: 'left'); + return true; + } + + /// Double-tap an element. + Future doubleTap({String? key, String? text}) async { + final selector = _buildSelector(key: key, text: text); + final bounds = await _getElementBounds(selector, text: text); + if (bounds == null) return false; + final cx = bounds['cx'] as double; + final cy = bounds['cy'] as double; + await _dispatchMouseEvent('mousePressed', cx, cy, + button: 'left', clickCount: 1); + await _dispatchMouseEvent('mouseReleased', cx, cy, + button: 'left', clickCount: 1); + await _dispatchMouseEvent('mousePressed', cx, cy, + button: 'left', clickCount: 2); + await _dispatchMouseEvent('mouseReleased', cx, cy, + button: 'left', clickCount: 2); + return true; + } + + /// Get text value of an input element. + Future getTextValue(String key) async { + final result = await _evalJs(''' + (() => { + const el = document.querySelector('#$key') || document.querySelector('[name="$key"]'); + return el ? (el.value || el.textContent || '') : null; + })() + '''); + return result['result']?['value'] as String?; + } + + /// Get all text content on the page. + Future getTextContent() async { + final result = await _evalJs('document.body.innerText'); + return (result['result']?['value'] as String?) ?? ''; + } + + /// Get navigation stack (just current URL for web). + Future> getNavigationStack() async { + final url = await getCurrentRoute(); + return [url]; + } + + // ── Extended interaction methods ── + + /// Drag from one point to another. + Future> drag( + double startX, double startY, double endX, double endY) async { + try { + return await Future(() async { + await _dispatchMouseEvent('mousePressed', startX, startY, + button: 'left', clickCount: 1); + const steps = 10; + for (var i = 1; i <= steps; i++) { + final x = startX + (endX - startX) * i / steps; + final y = startY + (endY - startY) * i / steps; + await _dispatchMouseEvent('mouseMoved', x, y, button: 'left'); + } + await _dispatchMouseEvent('mouseReleased', endX, endY, + button: 'left', clickCount: 1); + return {"success": true} as Map; + }).timeout(const Duration(seconds: 10)); + } on TimeoutException { + return { + "success": false, + "error": "Drag timed out — mouse event not acknowledged by browser" + }; + } + } + + /// Long press at coordinates. + Future longPressAt(double x, double y) async { + await _dispatchMouseEvent('mousePressed', x, y, + button: 'left', clickCount: 1); + await Future.delayed(const Duration(milliseconds: 800)); + await _dispatchMouseEvent('mouseReleased', x, y, + button: 'left', clickCount: 1); + } + + /// Swipe between coordinates. + Future> swipeCoordinates( + double startX, double startY, double endX, double endY, + {int durationMs = 300}) async { + try { + return await Future(() async { + await _dispatchMouseEvent('mousePressed', startX, startY, + button: 'left', clickCount: 1); + const steps = 8; + for (var i = 1; i <= steps; i++) { + final x = startX + (endX - startX) * i / steps; + final y = startY + (endY - startY) * i / steps; + await _dispatchMouseEvent('mouseMoved', x, y, button: 'left'); + await Future.delayed(Duration(milliseconds: durationMs ~/ steps)); + } + await _dispatchMouseEvent('mouseReleased', endX, endY, + button: 'left', clickCount: 1); + return {"success": true} as Map; + }).timeout(const Duration(seconds: 10)); + } on TimeoutException { + return { + "success": false, + "error": "Swipe timed out — mouse event not acknowledged by browser" + }; + } + } + + /// Edge swipe (simulate from edge of viewport). + Future> edgeSwipe(String direction, + {String edge = 'left', int distance = 200}) async { + final viewport = + await _evalJs('[window.innerWidth, window.innerHeight].join(",")'); + final dims = + (viewport['result']?['value'] as String? ?? '1280,720').split(','); + final w = double.parse(dims[0]); + final h = double.parse(dims[1]); + double startX, startY, endX, endY; + switch (direction) { + case 'right': + startX = 5; + startY = h / 2; + endX = distance.toDouble(); + endY = h / 2; + break; + case 'left': + startX = w - 5; + startY = h / 2; + endX = w - distance; + endY = h / 2; + break; + case 'up': + startX = w / 2; + startY = h - 5; + endX = w / 2; + endY = h - distance; + break; + case 'down': + default: + startX = w / 2; + startY = 5; + endX = w / 2; + endY = distance.toDouble(); + } + return swipeCoordinates(startX, startY, endX, endY); + } + + /// Custom gesture (series of points). + Future> gesture( + List> points) async { + if (points.isEmpty) return {"success": false, "message": "No points"}; + try { + return await Future(() async { + final first = points.first; + await _dispatchMouseEvent('mousePressed', + (first['x'] as num).toDouble(), (first['y'] as num).toDouble(), + button: 'left'); + for (var i = 1; i < points.length; i++) { + await _dispatchMouseEvent( + 'mouseMoved', + (points[i]['x'] as num).toDouble(), + (points[i]['y'] as num).toDouble(), + button: 'left'); + await Future.delayed(const Duration(milliseconds: 20)); + } + final last = points.last; + await _dispatchMouseEvent('mouseReleased', + (last['x'] as num).toDouble(), (last['y'] as num).toDouble(), + button: 'left'); + return {"success": true} as Map; + }).timeout(const Duration(seconds: 10)); + } on TimeoutException { + return { + "success": false, + "error": "Gesture timed out — mouse event not acknowledged by browser" + }; + } + } + + /// Scroll until element is visible. + Future> scrollUntilVisible(String key, + {int maxScrolls = 10, String direction = 'down'}) async { + for (var i = 0; i < maxScrolls; i++) { + final result = await _evalJs(''' + (() => { + const el = ${_jsResolveElement(key)}; + if (!el) return false; + const rect = el.getBoundingClientRect(); + return rect.top >= 0 && rect.bottom <= window.innerHeight; + })() + '''); + if (result['result']?['value'] == true) + return {"success": true, "scrolls": i}; + final dy = direction == 'up' ? -300 : 300; + await _evalJs('window.scrollBy(0, $dy)'); + await Future.delayed(const Duration(milliseconds: 200)); + } + return { + "success": false, + "message": "Element '$key' not visible after $maxScrolls scrolls" + }; + } + + /// Get checkbox state. + Future> getCheckboxState(String key) async { + final result = await _evalJs(''' + (() => { + let el = ${_jsResolveElement(key)}; + // Fallback: search checkboxes by label/value + if (!el) { + for (const cb of document.querySelectorAll('input[type="checkbox"], [role="checkbox"]')) { + const label = cb.closest('label') || document.querySelector('label[for="' + cb.id + '"]'); + const t = (label ? label.textContent : cb.getAttribute('aria-label') || '').trim(); + if (t.toLowerCase().includes('$key'.toLowerCase()) || cb.value === '$key') { el = cb; break; } + } + } + if (!el) return JSON.stringify({ success: false, error: "Element not found" }); + if (el.type === 'checkbox') return JSON.stringify({ success: true, checked: el.checked }); + const cb = el.querySelector && el.querySelector('input[type="checkbox"]'); + if (cb) return JSON.stringify({ success: true, checked: cb.checked }); + return JSON.stringify({ success: true, checked: el.getAttribute('aria-checked') === 'true' }); + })() + '''); + return _parseJsonEval(result) ?? + {"success": false, "error": "Element not found"}; + } + + /// Get slider value. + Future> getSliderValue(String key) async { + final result = await _evalJs(''' + (() => { + let el = ${_jsResolveElement(key)}; + // Fallback: search sliders by label/name + if (!el) { + for (const s of document.querySelectorAll('input[type="range"], [role="slider"]')) { + const label = s.closest('label') || document.querySelector('label[for="' + s.id + '"]'); + const t = (label ? label.textContent : s.getAttribute('aria-label') || '').trim(); + if (t.toLowerCase().includes('$key'.toLowerCase()) || s.name === '$key') { el = s; break; } + } + } + if (!el) return JSON.stringify({ success: false, error: "Element not found" }); + return JSON.stringify({ + success: true, + value: parseFloat(el.value || el.getAttribute('aria-valuenow') || 0), + min: parseFloat(el.min || el.getAttribute('aria-valuemin') || 0), + max: parseFloat(el.max || el.getAttribute('aria-valuemax') || 100) + }); + })() + '''); + return _parseJsonEval(result) ?? + {"success": false, "error": "Element not found"}; + } + + /// Get page state (title, url, scroll, viewport). + Future> getPageState() async { + final result = await _evalJs(''' + JSON.stringify({ + title: document.title, + url: window.location.href, + scrollX: window.scrollX, + scrollY: window.scrollY, + viewportWidth: window.innerWidth, + viewportHeight: window.innerHeight, + documentHeight: document.documentElement.scrollHeight, + readyState: document.readyState, + visibilityState: document.visibilityState + }) + '''); + final value = result['result']?['value'] as String?; + if (value == null) return {}; + return jsonDecode(value) as Map; + } + + /// Get interactable elements (alias for getInteractiveElements). + Future> getInteractableElements() async { + return getInteractiveElements(); + } + + /// Get performance metrics. + Future> getPerformance() async { + final result = await _evalJs(''' + JSON.stringify((() => { + const nav = performance.getEntriesByType('navigation')[0] || {}; + return { + loadTime: nav.loadEventEnd - nav.startTime, + domContentLoaded: nav.domContentLoadedEventEnd - nav.startTime, + firstPaint: (performance.getEntriesByType('paint').find(p => p.name === 'first-paint') || {}).startTime || 0, + firstContentfulPaint: (performance.getEntriesByType('paint').find(p => p.name === 'first-contentful-paint') || {}).startTime || 0, + resourceCount: performance.getEntriesByType('resource').length + }; + })()) + '''); + final value = result['result']?['value'] as String?; + if (value == null) return {}; + return jsonDecode(value) as Map; + } + + /// Get frame stats via Performance API. + Future> getFrameStats() async { + final result = await _evalJs(''' + JSON.stringify({ + fps: 60, + frameCount: performance.getEntriesByType('frame').length || 0, + longTasks: performance.getEntriesByType('longtask').length || 0, + timestamp: performance.now() + }) + '''); + final value = result['result']?['value'] as String?; + if (value == null) return {}; + return jsonDecode(value) as Map; + } + + /// Get memory stats. + Future> getMemoryStats() async { + final result = await _evalJs(''' + JSON.stringify(performance.memory ? { + usedJSHeapSize: performance.memory.usedJSHeapSize, + totalJSHeapSize: performance.memory.totalJSHeapSize, + jsHeapSizeLimit: performance.memory.jsHeapSizeLimit + } : { message: "memory API not available" }) + '''); + final value = result['result']?['value'] as String?; + if (value == null) return {"message": "not available"}; + return jsonDecode(value) as Map; + } + + /// Assert text exists on page. + Future> assertText(String text, {String? key}) async { + final result = await _evalJs(''' + (() => { + ${key != null ? "const el = ${_jsResolveElement(key)}; return el ? el.textContent.includes('$text') : false;" : "return document.body.innerText.includes('$text');"} + })() + '''); + final found = result['result']?['value'] == true; + return {"success": found, "found": found, "text": text}; + } + + /// Assert element count. + Future> assertElementCount( + String selector, int expectedCount) async { + final result = + await _evalJs('document.querySelectorAll("$selector").length'); + final count = result['result']?['value'] as int? ?? 0; + return { + "success": count == expectedCount, + "actual_count": count, + "expected_count": expectedCount, + }; + } + + /// Wait for idle. + Future> waitForIdle({int timeoutMs = 5000}) async { + await Future.delayed( + Duration(milliseconds: timeoutMs > 2000 ? 2000 : timeoutMs)); + return {"success": true, "message": "Page idle"}; + } + + /// Diagnose connection. + Future> diagnose() async { + return { + "mode": "cdp", + "port": _port, + "connected": _connected, + "url": await getCurrentRoute(), + }; + } + + // ── Advanced CDP tools (beyond Playwright MCP) ── + + /// Evaluate JavaScript and return result. + Future> eval(String expression) async { + return _evalJs(expression); + } + + /// Press a keyboard key. + Future pressKey(String key, {List? modifiers}) async { + // Map common key names to CDP key codes + final keyMap = >{ + 'Enter': {'key': 'Enter', 'code': 'Enter', 'keyCode': 13}, + 'Tab': {'key': 'Tab', 'code': 'Tab', 'keyCode': 9}, + 'Escape': {'key': 'Escape', 'code': 'Escape', 'keyCode': 27}, + 'Backspace': {'key': 'Backspace', 'code': 'Backspace', 'keyCode': 8}, + 'Delete': {'key': 'Delete', 'code': 'Delete', 'keyCode': 46}, + 'ArrowUp': {'key': 'ArrowUp', 'code': 'ArrowUp', 'keyCode': 38}, + 'ArrowDown': {'key': 'ArrowDown', 'code': 'ArrowDown', 'keyCode': 40}, + 'ArrowLeft': {'key': 'ArrowLeft', 'code': 'ArrowLeft', 'keyCode': 37}, + 'ArrowRight': {'key': 'ArrowRight', 'code': 'ArrowRight', 'keyCode': 39}, + 'Space': {'key': ' ', 'code': 'Space', 'keyCode': 32}, + }; + final mapped = keyMap[key]; + final keyName = mapped?['key'] ?? key; + final code = mapped?['code'] ?? 'Key${key.toUpperCase()}'; + final keyCode = mapped?['keyCode'] ?? key.codeUnitAt(0); + + int modifierFlags = 0; + if (modifiers != null) { + if (modifiers.contains('Alt')) modifierFlags |= 1; + if (modifiers.contains('Control')) modifierFlags |= 2; + if (modifiers.contains('Meta')) modifierFlags |= 4; + if (modifiers.contains('Shift')) modifierFlags |= 8; + } + + await _call('Input.dispatchKeyEvent', { + 'type': 'keyDown', + 'key': keyName, + 'code': code, + 'windowsVirtualKeyCode': keyCode, + 'nativeVirtualKeyCode': keyCode, + 'modifiers': modifierFlags, + }); + await _call('Input.dispatchKeyEvent', { + 'type': 'keyUp', + 'key': keyName, + 'code': code, + 'windowsVirtualKeyCode': keyCode, + 'nativeVirtualKeyCode': keyCode, + 'modifiers': modifierFlags, + }); + } + + /// Type text into the focused element. + /// Complete US QWERTY keyboard layout mapping. + /// Maps every printable ASCII character to its physical key, keyCode, + /// and whether Shift is required. Based on Puppeteer's USKeyboardLayout. + static const _usKeyboard = >{ + // [code, keyCode, shifted] + // --- Letters (a-z unshifted, A-Z shifted) --- + 'a': ['KeyA', 65, false], 'b': ['KeyB', 66, false], + 'c': ['KeyC', 67, false], 'd': ['KeyD', 68, false], + 'e': ['KeyE', 69, false], 'f': ['KeyF', 70, false], + 'g': ['KeyG', 71, false], 'h': ['KeyH', 72, false], + 'i': ['KeyI', 73, false], 'j': ['KeyJ', 74, false], + 'k': ['KeyK', 75, false], 'l': ['KeyL', 76, false], + 'm': ['KeyM', 77, false], 'n': ['KeyN', 78, false], + 'o': ['KeyO', 79, false], 'p': ['KeyP', 80, false], + 'q': ['KeyQ', 81, false], 'r': ['KeyR', 82, false], + 's': ['KeyS', 83, false], 't': ['KeyT', 84, false], + 'u': ['KeyU', 85, false], 'v': ['KeyV', 86, false], + 'w': ['KeyW', 87, false], 'x': ['KeyX', 88, false], + 'y': ['KeyY', 89, false], 'z': ['KeyZ', 90, false], + 'A': ['KeyA', 65, true], 'B': ['KeyB', 66, true], + 'C': ['KeyC', 67, true], 'D': ['KeyD', 68, true], + 'E': ['KeyE', 69, true], 'F': ['KeyF', 70, true], + 'G': ['KeyG', 71, true], 'H': ['KeyH', 72, true], + 'I': ['KeyI', 73, true], 'J': ['KeyJ', 74, true], + 'K': ['KeyK', 75, true], 'L': ['KeyL', 76, true], + 'M': ['KeyM', 77, true], 'N': ['KeyN', 78, true], + 'O': ['KeyO', 79, true], 'P': ['KeyP', 80, true], + 'Q': ['KeyQ', 81, true], 'R': ['KeyR', 82, true], + 'S': ['KeyS', 83, true], 'T': ['KeyT', 84, true], + 'U': ['KeyU', 85, true], 'V': ['KeyV', 86, true], + 'W': ['KeyW', 87, true], 'X': ['KeyX', 88, true], + 'Y': ['KeyY', 89, true], 'Z': ['KeyZ', 90, true], + // --- Digits (unshifted) --- + '0': ['Digit0', 48, false], '1': ['Digit1', 49, false], + '2': ['Digit2', 50, false], '3': ['Digit3', 51, false], + '4': ['Digit4', 52, false], '5': ['Digit5', 53, false], + '6': ['Digit6', 54, false], '7': ['Digit7', 55, false], + '8': ['Digit8', 56, false], '9': ['Digit9', 57, false], + // --- Shifted digit symbols --- + ')': ['Digit0', 48, true], '!': ['Digit1', 49, true], + '@': ['Digit2', 50, true], '#': ['Digit3', 51, true], + '\$': ['Digit4', 52, true], '%': ['Digit5', 53, true], + '^': ['Digit6', 54, true], '&': ['Digit7', 55, true], + '*': ['Digit8', 56, true], '(': ['Digit9', 57, true], + // --- Punctuation (unshifted) --- + ' ': ['Space', 32, false], + '-': ['Minus', 189, false], '=': ['Equal', 187, false], + '[': ['BracketLeft', 219, false], ']': ['BracketRight', 221, false], + '\\': ['Backslash', 220, false], ';': ['Semicolon', 186, false], + "'": ['Quote', 222, false], '`': ['Backquote', 192, false], + ',': ['Comma', 188, false], '.': ['Period', 190, false], + '/': ['Slash', 191, false], + // --- Punctuation (shifted) --- + '_': ['Minus', 189, true], '+': ['Equal', 187, true], + '{': ['BracketLeft', 219, true], '}': ['BracketRight', 221, true], + '|': ['Backslash', 220, true], ':': ['Semicolon', 186, true], + '"': ['Quote', 222, true], '~': ['Backquote', 192, true], + '<': ['Comma', 188, true], '>': ['Period', 190, true], + '?': ['Slash', 191, true], + }; + + /// Type text into the currently focused element. + /// + /// **Strategy** (universal, works with all frameworks — React, Vue, Angular, + /// Svelte, vanilla, contenteditable, etc.): + /// + /// 1. Detect focused element type (input/textarea/contenteditable/other). + /// 2. For form elements (``, `">', +]; + +/// Sensitive data patterns +final sensitivePatterns = { + 'API Key (sk-)': RegExp(r'sk-[a-zA-Z0-9]{20,}'), + 'API Key (pk_)': RegExp(r'pk_[a-zA-Z0-9]{20,}'), + 'API Key (api_key)': RegExp('api_key[=:]\\s*["\']?[a-zA-Z0-9_-]{16,}'), + 'API Key (apikey)': RegExp('apikey[=:]\\s*["\']?[a-zA-Z0-9_-]{16,}'), + 'AWS Access Key': RegExp(r'AKIA[0-9A-Z]{16}'), + 'AWS Secret Key': + RegExp('aws_secret_access_key[=:]\\s*["\']?[a-zA-Z0-9/+=]{40}'), + 'JWT Token': + RegExp(r'eyJ[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_-]{10,}'), + 'Credit Card (Visa)': RegExp(r'\b4[0-9]{12}(?:[0-9]{3})?\b'), + 'Credit Card (Mastercard)': RegExp(r'\b5[1-5][0-9]{14}\b'), + 'Credit Card (Amex)': RegExp(r'\b3[47][0-9]{13}\b'), + 'Password in URL': RegExp(r'[?&]password=[^&]+'), + 'Password field value': RegExp('password["\\s]*[:=]\\s*["\'][^"\']{1,}["\']'), + 'Private Key': RegExp(r'-----BEGIN\s+(RSA\s+)?PRIVATE KEY-----'), + 'Email Address': + RegExp(r'\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b'), + 'Bearer Token': RegExp(r'Bearer\s+[a-zA-Z0-9_.=-]{20,}'), + 'GitHub Token': RegExp(r'gh[pousr]_[A-Za-z0-9_]{36,}'), + 'Google API Key': RegExp(r'AIza[0-9A-Za-z_-]{35}'), + 'Slack Token': RegExp(r'xox[baprs]-[0-9a-zA-Z]{10,}'), +}; + +/// Security headers to check +const expectedSecurityHeaders = { + 'Content-Security-Policy': { + 'severity': 'High', + 'description': 'Prevents XSS and data injection attacks', + }, + 'Strict-Transport-Security': { + 'severity': 'High', + 'description': 'Enforces HTTPS connections', + }, + 'X-Frame-Options': { + 'severity': 'Medium', + 'description': 'Prevents clickjacking attacks', + }, + 'X-Content-Type-Options': { + 'severity': 'Medium', + 'description': 'Prevents MIME type sniffing', + }, + 'X-XSS-Protection': { + 'severity': 'Low', + 'description': 'Legacy XSS filter (deprecated but still useful)', + }, + 'Referrer-Policy': { + 'severity': 'Low', + 'description': 'Controls referrer information sent with requests', + }, + 'Permissions-Policy': { + 'severity': 'Medium', + 'description': 'Controls browser features and APIs', + }, +}; + +class SecurityFinding { + final String category; + final String severity; // Critical, High, Medium, Low, Info + final String title; + final String description; + final String? url; + final String? evidence; + final String? recommendation; + + SecurityFinding({ + required this.category, + required this.severity, + required this.title, + required this.description, + this.url, + this.evidence, + this.recommendation, + }); + + Map toJson() => { + 'category': category, + 'severity': severity, + 'title': title, + 'description': description, + if (url != null) 'url': url, + if (evidence != null) 'evidence': evidence, + if (recommendation != null) 'recommendation': recommendation, + }; +} + +class SecurityScanner { + final String startUrl; + final int maxDepth; + final int cdpPort; + final bool headless; + final String reportPath; + final int maxPages; + final int maxLinks; + + late CdpDriver _cdp; + final List _findings = []; + final Map _visited = {}; + final List<(String, int)> _queue = []; + final Map> _responseHeaders = {}; + + SecurityScanner({ + required this.startUrl, + required this.maxDepth, + required this.cdpPort, + required this.headless, + required this.reportPath, + this.maxPages = 10, + this.maxLinks = 20, + }); + + Future> run() async { + print('📡 Launching Chrome and connecting via CDP...'); + _cdp = CdpDriver( + url: startUrl, + port: cdpPort, + launchChrome: cdpPort == 0, + headless: headless, + ); + await _cdp.connect(); + print('✅ Connected'); + + // Enable Network domain for header checking + await _cdp.call('Network.enable'); + + // Set up dialog handler for XSS detection + await _cdp.call('Page.enable'); + + // Crawl and scan + _queue.add((startUrl, 0)); + + while (_queue.isNotEmpty) { + if (_visited.length >= maxPages) break; + + final (pageUrl, currentDepth) = _queue.removeAt(0); + final normalizedUrl = _normalizeUrl(pageUrl); + + if (_visited.containsKey(normalizedUrl)) continue; + if (currentDepth > maxDepth) continue; + _visited[normalizedUrl] = true; + + print(''); + print('🔍 [$currentDepth/$maxDepth] Scanning: $pageUrl'); + + await _scanPage(pageUrl, currentDepth); + } + + await _cdp.disconnect(); + + // Generate report + print(''); + print('📊 Generating security report...'); + await _generateHtmlReport(); + + return _findings; + } + + /// Run security scan with specific checks (for MCP tool use) + Future> runChecks({ + required CdpDriver cdp, + List checks = const [ + 'xss', + 'csrf', + 'headers', + 'sensitive_data', + 'mixed_content' + ], + List? customXssPayloads, + }) async { + _cdp = cdp; + + final currentUrl = (await _cdp.evaluate('window.location.href'))['result'] + ?['value'] as String? ?? + startUrl; + + if (checks.contains('headers')) { + await _checkSecurityHeaders(currentUrl); + } + if (checks.contains('xss')) { + await _xssScan(currentUrl, customPayloads: customXssPayloads); + } + if (checks.contains('sensitive_data')) { + await _sensitiveDataScan(currentUrl); + } + if (checks.contains('mixed_content')) { + await _mixedContentScan(currentUrl); + } + if (checks.contains('csrf')) { + await _csrfCheck(currentUrl); + } + if (checks.contains('open_redirect')) { + await _openRedirectCheck(currentUrl); + } + if (checks.contains('clickjacking')) { + await _clickjackingCheck(currentUrl); + } + + return _findings; + } + + Future _scanPage(String pageUrl, int currentDepth) async { + try { + await _cdp.call('Page.navigate', {'url': pageUrl}); + await Future.delayed(const Duration(seconds: 2)); + + // Capture response headers + await _captureHeaders(pageUrl); + + // Run all checks + await _checkSecurityHeaders(pageUrl); + await _xssScan(pageUrl); + await _sensitiveDataScan(pageUrl); + await _mixedContentScan(pageUrl); + await _csrfCheck(pageUrl); + await _openRedirectCheck(pageUrl); + await _clickjackingCheck(pageUrl); + + // Discover links for crawling + if (currentDepth < maxDepth) { + final linksResult = await _cdp.evaluate(''' + JSON.stringify( + Array.from(document.querySelectorAll('a[href]')) + .map(a => a.href) + .filter(h => h && !h.startsWith('javascript:') && !h.startsWith('mailto:')) + ) + '''); + final linksJson = linksResult['result']?['value'] as String?; + if (linksJson != null) { + final links = (jsonDecode(linksJson) as List) + .cast() + .take(maxLinks) + .toList(); + for (final link in links) { + final normLink = _normalizeUrl(link); + if (!_visited.containsKey(normLink) && + _isSameOrigin(link, startUrl)) { + _queue.add((link, currentDepth + 1)); + } + } + } + } + } catch (e) { + print(' ❌ Error scanning: $e'); + } + } + + Future _captureHeaders(String pageUrl) async { + try { + // Use JavaScript to fetch headers via a same-origin request + final result = await _cdp.evaluate(''' + (async () => { + try { + const resp = await fetch(window.location.href, {method: 'HEAD', cache: 'no-cache'}); + const headers = {}; + resp.headers.forEach((v, k) => headers[k] = v); + return JSON.stringify(headers); + } catch(e) { return '{}'; } + })() + '''); + // Handle both awaited and non-awaited results + final val = result['result']?['value'] as String?; + if (val != null && val != '{}') { + _responseHeaders[pageUrl] = + (jsonDecode(val) as Map).cast(); + } + } catch (_) {} + } + + Future _checkSecurityHeaders(String pageUrl) async { + print(' 🔒 Checking security headers...'); + + final headers = _responseHeaders[pageUrl] ?? {}; + + // If no headers captured via fetch, try evaluating meta tags + if (headers.isEmpty) { + try { + final metaResult = await _cdp.evaluate(''' + JSON.stringify({ + csp: document.querySelector('meta[http-equiv="Content-Security-Policy"]')?.content || null + }) + '''); + final metaJson = metaResult['result']?['value'] as String?; + if (metaJson != null) { + final meta = jsonDecode(metaJson) as Map; + if (meta['csp'] != null) { + headers['content-security-policy'] = meta['csp'] as String; + } + } + } catch (_) {} + } + + // Only report truly critical missing headers (CSP and HSTS) + const criticalHeaders = { + 'Content-Security-Policy': { + 'severity': 'High', + 'description': 'Prevents XSS and data injection attacks', + }, + 'Strict-Transport-Security': { + 'severity': 'High', + 'description': 'Enforces HTTPS connections', + }, + }; + + for (final entry in criticalHeaders.entries) { + final headerName = entry.key; + final config = entry.value; + final headerLower = headerName.toLowerCase(); + + final found = headers.keys.any((k) => k.toLowerCase() == headerLower); + if (!found) { + _findings.add(SecurityFinding( + category: 'Security Headers', + severity: config['severity']!, + title: 'Missing $headerName header', + description: config['description']!, + url: pageUrl, + recommendation: 'Add $headerName header to server responses.', + )); + } + } + } + + Future _xssScan(String pageUrl, {List? customPayloads}) async { + print(' 💉 Testing XSS vulnerabilities...'); + + final payloads = customPayloads ?? xssPayloads; + + // Find input fields + final fieldsResult = await _cdp.evaluate(''' + JSON.stringify( + Array.from(document.querySelectorAll('input, textarea, [contenteditable]')) + .map((el, i) => ({ + index: i, + tag: el.tagName.toLowerCase(), + type: el.type || 'text', + name: el.name || '', + id: el.id || '', + selector: el.id ? '#' + el.id : (el.name ? '[name="' + el.name + '"]' : el.tagName.toLowerCase() + ':nth-of-type(' + (i+1) + ')') + })) + ) + '''); + + final fieldsJson = fieldsResult['result']?['value'] as String?; + if (fieldsJson == null) return; + + final fields = + (jsonDecode(fieldsJson) as List).cast>(); + + if (fields.isEmpty) { + print(' 📝 No input fields found'); + return; + } + + // Install dialog handler to detect alert() execution + bool dialogDetected = false; + await _cdp.evaluate(''' + window.__xss_dialog_detected__ = false; + window.__original_alert__ = window.alert; + window.alert = function() { window.__xss_dialog_detected__ = true; }; + '''); + + for (final field in fields) { + final selector = field['selector'] as String; + final fieldName = + field['name'] as String? ?? field['id'] as String? ?? selector; + + for (final payload in payloads.take(5)) { + // Limit payloads per field + try { + // Clear and inject payload + await _cdp.evaluate(''' + (() => { + const el = document.querySelector('$selector'); + if (!el) return; + el.value = ''; + el.value = ${jsonEncode(payload)}; + el.dispatchEvent(new Event('input', {bubbles: true})); + el.dispatchEvent(new Event('change', {bubbles: true})); + })() + '''); + + await Future.delayed(const Duration(milliseconds: 200)); + + // Check if dialog was triggered + final dialogResult = + await _cdp.evaluate('window.__xss_dialog_detected__'); + dialogDetected = dialogResult['result']?['value'] == true; + + if (dialogDetected) { + _findings.add(SecurityFinding( + category: 'XSS', + severity: 'Critical', + title: 'XSS vulnerability detected in field "$fieldName"', + description: + 'JavaScript execution was triggered via alert() when injecting payload into the input field.', + url: pageUrl, + evidence: 'Payload: $payload', + recommendation: + 'Sanitize and escape all user input. Use Content-Security-Policy headers.', + )); + // Reset + await _cdp.evaluate('window.__xss_dialog_detected__ = false'); + } + + // Check if payload is reflected in DOM + final reflectedResult = await _cdp.evaluate(''' + document.body.innerHTML.includes(${jsonEncode(payload)}) + '''); + final reflected = reflectedResult['result']?['value'] == true; + + if (reflected) { + _findings.add(SecurityFinding( + category: 'XSS', + severity: 'High', + title: 'XSS payload reflected in DOM for field "$fieldName"', + description: + 'The injected payload was found unescaped in the page DOM, indicating potential reflected XSS.', + url: pageUrl, + evidence: 'Payload: $payload', + recommendation: + 'HTML-encode all user input before rendering in the DOM.', + )); + } + } catch (_) { + // Skip failures + } + } + } + + // Restore alert + await _cdp.evaluate(''' + if (window.__original_alert__) window.alert = window.__original_alert__; + '''); + } + + Future _sensitiveDataScan(String pageUrl) async { + print(' 🔑 Scanning for sensitive data exposure...'); + + // Check localStorage + await _scanStorage(pageUrl, 'localStorage', + 'JSON.stringify(Object.entries(localStorage))'); + + // Check sessionStorage + await _scanStorage(pageUrl, 'sessionStorage', + 'JSON.stringify(Object.entries(sessionStorage))'); + + // Check cookies + await _scanStorage(pageUrl, 'cookies', 'JSON.stringify(document.cookie)'); + + // Check page source + try { + final sourceResult = await _cdp + .evaluate('document.documentElement.outerHTML.substring(0, 50000)'); + final source = sourceResult['result']?['value'] as String? ?? ''; + + for (final entry in sensitivePatterns.entries) { + final matches = entry.value.allMatches(source); + if (matches.isNotEmpty) { + // Skip email patterns in mailto links and common false positives + if (entry.key == 'Email Address') continue; + + _findings.add(SecurityFinding( + category: 'Sensitive Data', + severity: entry.key.contains('Private Key') || + entry.key.contains('Password') + ? 'Critical' + : entry.key.contains('API Key') || + entry.key.contains('AWS') || + entry.key.contains('JWT') + ? 'High' + : entry.key.contains('Credit Card') + ? 'Critical' + : 'Medium', + title: '${entry.key} found in page source', + description: + 'Detected ${matches.length} potential ${entry.key} pattern(s) in the page HTML source.', + url: pageUrl, + evidence: + 'Pattern match: ${matches.first.group(0)?.substring(0, (matches.first.group(0)?.length ?? 0).clamp(0, 40))}...', + recommendation: + 'Remove sensitive data from client-side code. Use server-side APIs.', + )); + } + } + } catch (_) {} + + // Check console logs for sensitive data + try { + final consoleResult = await _cdp.evaluate(''' + JSON.stringify(window.__fs_console_log__ || []) + '''); + final consoleJson = consoleResult['result']?['value'] as String?; + if (consoleJson != null) { + final logs = consoleJson; + for (final entry in sensitivePatterns.entries) { + if (entry.value.hasMatch(logs)) { + _findings.add(SecurityFinding( + category: 'Sensitive Data', + severity: 'Medium', + title: '${entry.key} found in console output', + description: + 'Detected ${entry.key} pattern in console log output.', + url: pageUrl, + recommendation: + 'Remove sensitive data from console.log statements in production.', + )); + } + } + } + } catch (_) {} + } + + Future _scanStorage( + String pageUrl, String storageName, String jsExpr) async { + try { + final result = await _cdp.evaluate(jsExpr); + final val = result['result']?['value'] as String? ?? ''; + + for (final entry in sensitivePatterns.entries) { + if (entry.value.hasMatch(val)) { + _findings.add(SecurityFinding( + category: 'Sensitive Data', + severity: entry.key.contains('Password') || + entry.key.contains('Private Key') + ? 'Critical' + : 'High', + title: '${entry.key} found in $storageName', + description: 'Detected ${entry.key} pattern in $storageName.', + url: pageUrl, + recommendation: + 'Avoid storing sensitive data in $storageName. Use secure, HttpOnly cookies for tokens.', + )); + } + } + } catch (_) {} + } + + Future _mixedContentScan(String pageUrl) async { + if (!pageUrl.startsWith('https://')) return; + + print(' 🔀 Checking for mixed content...'); + + try { + final result = await _cdp.evaluate(''' + JSON.stringify( + Array.from(document.querySelectorAll('img, script, link, iframe, video, audio, source, object, embed')) + .filter(el => { + const src = el.src || el.href || el.data || ''; + return src.startsWith('http://'); + }) + .map(el => ({ + tag: el.tagName.toLowerCase(), + src: el.src || el.href || el.data || '' + })) + ) + '''); + + final val = result['result']?['value'] as String?; + if (val == null) return; + + final mixedResources = + (jsonDecode(val) as List).cast>(); + + for (final resource in mixedResources) { + final tag = resource['tag'] as String; + final src = resource['src'] as String; + final isActive = tag == 'script' || tag == 'iframe'; + + _findings.add(SecurityFinding( + category: 'Mixed Content', + severity: isActive ? 'High' : 'Medium', + title: '${isActive ? "Active" : "Passive"} mixed content: <$tag>', + description: + 'HTTPS page loads ${isActive ? "active" : "passive"} HTTP resource.', + url: pageUrl, + evidence: 'Source: $src', + recommendation: + 'Use HTTPS for all resources. Update the URL to use https://.', + )); + } + } catch (_) {} + } + + Future _csrfCheck(String pageUrl) async { + print(' 🛡️ Checking CSRF protection...'); + + try { + // Check all forms (not just POST) for CSRF tokens + final result = await _cdp.evaluate(''' + JSON.stringify( + Array.from(document.querySelectorAll('form')) + .map((form, i) => ({ + index: i, + action: form.action, + method: (form.method || 'get').toUpperCase(), + hasCSRFToken: !!( + form.querySelector('input[name*="csrf"]') || + form.querySelector('input[name*="token"]') || + form.querySelector('input[name*="_token"]') || + form.querySelector('input[name="authenticity_token"]') || + document.querySelector('meta[name="csrf-token"]') + ), + hasInputs: form.querySelectorAll('input, textarea, select').length > 0 + })) + ) + '''); + + final val = result['result']?['value'] as String?; + if (val == null) return; + + final forms = (jsonDecode(val) as List).cast>(); + + // Only report forms that are POST and have inputs but no CSRF token + for (final form in forms) { + final method = form['method'] as String? ?? 'GET'; + final hasInputs = form['hasInputs'] == true; + if (method == 'POST' && hasInputs && form['hasCSRFToken'] != true) { + _findings.add(SecurityFinding( + category: 'CSRF', + severity: 'High', + title: 'POST form without CSRF token', + description: + 'A POST form with input fields was found without a CSRF protection token.', + url: pageUrl, + evidence: 'Form action: ${form['action']}', + recommendation: + 'Add CSRF tokens to all state-changing forms. Use SameSite cookie attribute.', + )); + } + } + } catch (_) {} + } + + Future _openRedirectCheck(String pageUrl) async { + print(' ↪️ Checking for open redirect...'); + + try { + final uri = Uri.parse(pageUrl); + final redirectParams = [ + 'redirect', + 'redirect_uri', + 'redirect_url', + 'callback', + 'next', + 'url', + 'return', + 'returnTo', + 'return_url', + 'continue', + 'dest', + 'destination', + 'go', + 'target', + 'rurl', + 'forward', + 'forward_url' + ]; + + for (final param in uri.queryParameters.keys) { + if (redirectParams.contains(param.toLowerCase())) { + final value = uri.queryParameters[param] ?? ''; + // Check if the redirect parameter accepts external URLs + if (value.startsWith('http') || value.startsWith('//')) { + _findings.add(SecurityFinding( + category: 'Open Redirect', + severity: 'Medium', + title: 'Potential open redirect via "$param" parameter', + description: + 'URL parameter "$param" contains an external URL, which may be exploitable for open redirect attacks.', + url: pageUrl, + evidence: '$param=$value', + recommendation: + 'Validate redirect URLs against a whitelist. Only allow relative URLs or specific domains.', + )); + } + } + } + + // Also check for redirect params in links on the page + final linksResult = await _cdp.evaluate(''' + JSON.stringify( + Array.from(document.querySelectorAll('a[href]')) + .map(a => a.href) + .filter(h => { + const lower = h.toLowerCase(); + return lower.includes('redirect=') || lower.includes('next=') || + lower.includes('url=') || lower.includes('callback=') || + lower.includes('return='); + }) + .slice(0, 10) + ) + '''); + final linksJson = linksResult['result']?['value'] as String?; + if (linksJson != null) { + final links = (jsonDecode(linksJson) as List).cast(); + for (final link in links) { + final linkUri = Uri.tryParse(link); + if (linkUri == null) continue; + for (final param in linkUri.queryParameters.keys) { + if (redirectParams.contains(param.toLowerCase())) { + final value = linkUri.queryParameters[param] ?? ''; + if (value.startsWith('http') || value.startsWith('//')) { + _findings.add(SecurityFinding( + category: 'Open Redirect', + severity: 'Medium', + title: 'Link with potential open redirect via "$param"', + description: + 'Found a link containing a redirect parameter with an external URL.', + url: pageUrl, + evidence: + link.length > 100 ? '${link.substring(0, 100)}...' : link, + recommendation: 'Validate redirect URLs server-side.', + )); + } + } + } + } + } + } catch (_) {} + } + + Future _clickjackingCheck(String pageUrl) async { + print(' 🖼️ Checking clickjacking protection...'); + + final headers = _responseHeaders[pageUrl] ?? {}; + + final hasXFrameOptions = + headers.keys.any((k) => k.toLowerCase() == 'x-frame-options'); + final csp = headers.entries + .where((e) => e.key.toLowerCase() == 'content-security-policy') + .map((e) => e.value) + .firstOrNull; + final hasFrameAncestors = csp != null && csp.contains('frame-ancestors'); + + if (!hasXFrameOptions && !hasFrameAncestors) { + _findings.add(SecurityFinding( + category: 'Clickjacking', + severity: 'Medium', + title: 'No clickjacking protection', + description: + 'Neither X-Frame-Options header nor CSP frame-ancestors directive was found.', + url: pageUrl, + recommendation: + 'Add X-Frame-Options: DENY or SAMEORIGIN header, or use CSP frame-ancestors directive.', + )); + } + } + + String _normalizeUrl(String url) { + try { + final uri = Uri.parse(url); + return uri.replace(fragment: '').toString().replaceAll(RegExp(r'/$'), ''); + } catch (_) { + return url; + } + } + + bool _isSameOrigin(String url, String baseUrl) { + try { + return Uri.parse(url).host == Uri.parse(baseUrl).host; + } catch (_) { + return false; + } + } + + Future _generateHtmlReport() async { + final buffer = StringBuffer(); + buffer.writeln(''); + buffer.writeln(''); + buffer.writeln(''); + buffer.writeln( + ''); + buffer.writeln('flutter-skill Security Report'); + buffer.writeln(''); + + final critical = _findings.where((f) => f.severity == 'Critical').length; + final high = _findings.where((f) => f.severity == 'High').length; + final medium = _findings.where((f) => f.severity == 'Medium').length; + final low = _findings.where((f) => f.severity == 'Low').length; + final info = _findings.where((f) => f.severity == 'Info').length; + + buffer.writeln('

🔒 flutter-skill Security Report

'); + buffer.writeln( + '

Generated ${DateTime.now().toIso8601String()} — Target: $startUrl

'); + + buffer.writeln('
'); + buffer.writeln( + '
${_findings.length}
Total Findings
'); + buffer.writeln( + '
$critical
Critical
'); + buffer.writeln( + '
$high
High
'); + buffer.writeln( + '
$medium
Medium
'); + buffer.writeln( + '
$low
Low
'); + buffer.writeln( + '
$info
Info
'); + buffer.writeln('
'); + + // Findings grouped by severity + final severityOrder = ['Critical', 'High', 'Medium', 'Low', 'Info']; + for (final sev in severityOrder) { + final sevFindings = _findings.where((f) => f.severity == sev).toList(); + if (sevFindings.isEmpty) continue; + + buffer.writeln( + '

$sev (${sevFindings.length})

'); + + for (final f in sevFindings) { + buffer.writeln('
'); + buffer.writeln('
'); + buffer.writeln( + '${f.severity}'); + buffer.writeln( + '${_htmlEscape(f.title)}'); + buffer.writeln( + '${_htmlEscape(f.category)}'); + buffer.writeln('
'); + buffer.writeln( + '
${_htmlEscape(f.description)}
'); + if (f.url != null) { + buffer.writeln( + '
URL: ${_htmlEscape(f.url!)}
'); + } + if (f.evidence != null) { + buffer.writeln( + '
${_htmlEscape(f.evidence!)}
'); + } + if (f.recommendation != null) { + buffer.writeln( + '
💡 ${_htmlEscape(f.recommendation!)}
'); + } + buffer.writeln('
'); + } + } + + if (_findings.isEmpty) { + buffer.writeln('
'); + buffer.writeln( + '

✅ No security issues found!

'); + buffer.writeln('
'); + } + + buffer.writeln(''); + + final file = File(reportPath); + await file.writeAsString(buffer.toString()); + print(' ✅ Report saved to $reportPath'); + } + + String _htmlEscape(String text) { + return text + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"'); + } +} diff --git a/packaging/npm/dart/lib/src/cli/serve.dart b/packaging/npm/dart/lib/src/cli/serve.dart new file mode 100644 index 00000000..413167e5 --- /dev/null +++ b/packaging/npm/dart/lib/src/cli/serve.dart @@ -0,0 +1,1425 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import '../bridge/cdp_driver.dart'; + +/// `flutter-skill serve` — Zero-config WebMCP server. +/// +/// Connects to any running app/website via CDP and exposes +/// all interactive UI elements as MCP tools. Supports hot reload +/// detection — when the page changes, tools auto-update. +/// +/// Usage: +/// flutter-skill serve --url=https://example.com +/// flutter-skill serve --cdp-port=9222 --no-launch +/// flutter-skill serve --url=https://example.com --port=3000 --headless +Future runServe(List args) async { + String url = 'about:blank'; + int cdpPort = 9222; + int serverPort = 3000; + bool headless = false; + bool launchChrome = true; + bool watch = true; + + for (final arg in args) { + if (arg.startsWith('--url=')) { + url = arg.substring(6); + } else if (arg.startsWith('--cdp-port=')) { + cdpPort = int.parse(arg.substring(11)); + } else if (arg.startsWith('--chrome-port=')) { + // Support --chrome-port as an alias for --cdp-port + cdpPort = int.parse(arg.substring(14)); + } else if (arg.startsWith('--port=')) { + serverPort = int.parse(arg.substring(7)); + } else if (arg == '--headless') { + headless = true; + } else if (arg == '--no-launch') { + launchChrome = false; + } else if (arg == '--no-watch') { + watch = false; + } else if (!arg.startsWith('-')) { + // Positional arg = URL + url = arg; + } + } + + if (url == 'about:blank' && launchChrome) { + print('Usage: flutter-skill serve '); + print(''); + print('Examples:'); + print(' flutter-skill serve https://example.com'); + print(' flutter-skill serve https://amazon.com --port=3000'); + print(' flutter-skill serve --cdp-port=9222 --no-launch'); + print(' flutter-skill serve https://app.com --headless'); + print(''); + print('Options:'); + print(' --port=N HTTP server port (default: 3000)'); + print(' --cdp-port=N Chrome DevTools port (default: 9222)'); + print(' --headless Run Chrome in headless mode'); + print(' --no-launch Connect to existing Chrome instance'); + print(' --no-watch Disable hot reload detection'); + print(''); + print('This starts a WebMCP-compatible server that auto-discovers'); + print('all interactive elements on the page and exposes them as tools.'); + print('Any MCP client can connect and control the app.'); + exit(1); + } + + print('🚀 flutter-skill serve — Zero-Config WebMCP'); + print(''); + print(' URL: $url'); + print(' CDP Port: $cdpPort'); + print(' Server Port: $serverPort'); + print(' Headless: $headless'); + print(' Hot Reload Watch: $watch'); + print(''); + + // Step 1: Connect via CDP (always connect to about:blank first to avoid + // creating duplicate tabs when target URL redirects to a different subdomain) + print('📡 Connecting to Chrome...'); + final cdp = CdpDriver( + url: '', // Connect to any existing tab, don't create one for target URL + port: cdpPort, + launchChrome: launchChrome, + headless: headless, + ); + await cdp.connect(); + print('✅ Connected via CDP'); + + // Step 1b: Navigate to target URL via smart navigation (reuses tabs by domain) + if (url.isNotEmpty && url != 'about:blank') { + await _navigateToUrl(cdp, url, cdpPort); + await Future.delayed(const Duration(seconds: 2)); + } + + // Step 2: Initial tool discovery + print('🔍 Scanning page for interactive elements...'); + var toolCache = await _discoverAndPrint(cdp); + + // Step 3: Set up hot reload detection + if (watch) { + print('👀 Watching for changes (HMR / hot reload / DOM mutations)...'); + await _setupHotReloadDetection(cdp); + } + + // Step 4: Start HTTP server + print(''); + print('═══════════════════════════════════════════════'); + print(' 📋 WebMCP Server: http://localhost:$serverPort'); + print(''); + print(' Endpoints:'); + print(' GET /tools/list — List all discovered tools'); + print(' POST /tools/call — Call a tool { name, arguments }'); + print(' GET /snapshot — Page text snapshot'); + print(' GET /screenshot — Page screenshot (JPEG)'); + print(' POST /navigate — Navigate { url }'); + print(' POST /refresh — Force re-scan tools'); + print(' GET /health — Server status'); + print('═══════════════════════════════════════════════'); + print(''); + print('Press Ctrl+C to stop.'); + + final server = await HttpServer.bind( + InternetAddress.anyIPv4, + serverPort, + shared: + true, // SO_REUSEADDR — avoids "Address already in use" on TIME_WAIT sockets + ); + + // Shared mutable state + final state = _ServeState(toolCache, DateTime.now(), cdpPort); + + // Background: poll for DOM changes + if (watch) { + Timer.periodic(const Duration(seconds: 2), (timer) async { + try { + final changed = await _checkForChanges(cdp); + if (changed) { + print('🔄 Page changed (hot reload?) — rescanning...'); + state.tools = await _discoverAndPrint(cdp); + state.updatedAt = DateTime.now(); + } + } catch (e) { + // CDP connection may drop — try to recover + if (e.toString().contains('-32000') || + e.toString().contains('Inspected target navigated or closed')) { + print('⚠️ CDP target lost, attempting to reconnect...'); + await _reconnectCdpTarget(cdp, cdpPort, + preferOrigin: state.lastNavigatedOrigin); + } + } + }); + } + + await for (final request in server) { + try { + // Refresh tools on each request if stale (> 30s) + if (DateTime.now().difference(state.updatedAt).inSeconds > 30) { + try { + state.tools = await _discoverTools(cdp); + state.updatedAt = DateTime.now(); + } catch (e) { + print(' ⚠️ Tool refresh failed: $e'); + } + } + await _handleRequest(request, cdp, state); + } catch (e) { + // Try to recover from CDP target loss on any request + if (e.toString().contains('-32000') || + e.toString().contains('Inspected target navigated or closed')) { + print(' ⚠️ CDP target lost, attempting to reconnect...'); + await _reconnectCdpTarget(cdp, state.cdpPort, + preferOrigin: state.lastNavigatedOrigin); + } + print(' ❌ Request error: $e'); + try { + request.response + ..statusCode = 500 + ..headers.contentType = ContentType.json + ..write(jsonEncode({'error': e.toString()})); + await request.response.close(); + } catch (_) {} + } + } +} + +/// Shared mutable state for the serve session +class _ServeState { + List> tools; + DateTime updatedAt; + final int cdpPort; + String? lastNavigatedOrigin; // Track which origin we're connected to + _ServeState(this.tools, this.updatedAt, this.cdpPort); +} + +/// Reconnect to the CDP target after it navigates or closes. +/// If [preferOrigin] is set, prefer a tab matching that origin. +Future _reconnectCdpTarget(CdpDriver cdp, int cdpPort, + {String? preferOrigin}) async { + try { + await Future.delayed(const Duration(seconds: 1)); + final client = HttpClient(); + final request = + await client.getUrl(Uri.parse('http://localhost:$cdpPort/json')); + final response = await request.close(); + final body = await response.transform(utf8.decoder).join(); + final tabs = (jsonDecode(body) as List).cast>(); + client.close(); + + // Prefer tab matching the last navigated origin + Map? pageTab; + if (preferOrigin != null) { + pageTab = tabs.cast?>().firstWhere( + (t) => + t!['type'] == 'page' && + (t['url'] as String? ?? '').startsWith(preferOrigin), + orElse: () => null, + ); + } + // Fallback to first page tab + pageTab ??= tabs.firstWhere( + (t) => t['type'] == 'page', + orElse: () => tabs.isNotEmpty ? tabs.first : {}, + ); + + final wsUrl = pageTab['webSocketDebuggerUrl'] as String?; + if (wsUrl != null && wsUrl.isNotEmpty) { + await cdp.reconnectTo(wsUrl); + print(' ✅ Reconnected to CDP target: ${pageTab['url']}'); + } + } catch (e) { + print(' ❌ Failed to reconnect CDP target: $e'); + } +} + +/// Navigate to a URL — find an existing tab with matching origin and reconnect, +/// or navigate in current tab if no match found. +Future _navigateToUrl(CdpDriver cdp, String url, int cdpPort) async { + try { + final targetUri = Uri.parse(url); + final targetOrigin = '${targetUri.scheme}://${targetUri.host}'; + + // Fetch all tabs + final client = HttpClient(); + final request = + await client.getUrl(Uri.parse('http://localhost:$cdpPort/json')); + final response = await request.close(); + final body = await response.transform(utf8.decoder).join(); + final tabs = (jsonDecode(body) as List).cast>(); + client.close(); + + // Find tab with matching origin or same root domain + final targetHost = targetUri.host; + final targetParts = targetHost.split('.'); + final targetRoot = targetParts.length >= 2 + ? targetParts.sublist(targetParts.length - 2).join('.') + : targetHost; + + Map? matchingTab; + + // 1. Exact origin match + matchingTab = tabs.cast?>().firstWhere( + (t) => + t!['type'] == 'page' && + (t['url'] as String? ?? '').startsWith(targetOrigin), + orElse: () => null, + ); + + // 2. Root domain match (e.g. passport.csdn.net → www.csdn.net) + if (matchingTab == null) { + matchingTab = tabs.cast?>().firstWhere( + (t) { + if (t!['type'] != 'page') return false; + final tabHost = Uri.tryParse(t['url']?.toString() ?? '')?.host ?? ''; + final tabParts = tabHost.split('.'); + final tabRoot = tabParts.length >= 2 + ? tabParts.sublist(tabParts.length - 2).join('.') + : tabHost; + return tabRoot == targetRoot && tabRoot.isNotEmpty; + }, + orElse: () => null, + ); + } + + if (matchingTab != null) { + // Connect to existing tab with same origin/domain + final wsUrl = matchingTab['webSocketDebuggerUrl'] as String?; + if (wsUrl != null && wsUrl.isNotEmpty) { + await cdp.reconnectTo(wsUrl); + print(' ✅ Connected to existing tab: ${matchingTab['url']}'); + } + final tabUrl = matchingTab['url'] as String? ?? ''; + if (tabUrl == url) { + // Already on exact URL — skip navigation + } else { + // Navigate (even for root domain matches — user explicitly requested this URL) + await cdp.call('Page.navigate', {'url': url}); + await Future.delayed(const Duration(seconds: 3)); + await _retryWithJsNavIfBroken(cdp, url); + } + } else { + // No matching tab — navigate in current tab + await cdp.call('Page.navigate', {'url': url}); + await Future.delayed(const Duration(seconds: 3)); + await _retryWithJsNavIfBroken(cdp, url); + print(' ✅ Navigated current tab to: $url'); + } + } catch (e) { + print(' ⚠️ Smart navigate failed ($e), using direct navigate'); + await cdp.call('Page.navigate', {'url': url}); + await Future.delayed(const Duration(seconds: 3)); + } +} + +/// Some SPAs (X/Twitter) detect CDP Page.navigate and serve static error pages +/// like "JavaScript is not available". If detected, retry via JS navigation +/// which triggers the SPA router instead of a full page reload. +/// Only triggers on definitive anti-bot error pages, NOT on slow-loading pages. +Future _retryWithJsNavIfBroken(CdpDriver cdp, String url) async { + try { + final result = await cdp.evaluate(''' + (() => { + const text = document.body?.innerText || ''; + // Only retry on definitive anti-bot/broken-JS error pages + if (text.includes('JavaScript is not available') || + text.includes('Enable JavaScript') || + text.includes('JavaScript is disabled') || + text.includes('browser is not supported')) { + return 'broken'; + } + return 'ok'; + })() + '''); + final status = result['result']?['value'] as String? ?? 'ok'; + if (status == 'broken') { + print( + ' ⚠️ Anti-bot error page detected, retrying via JS navigation...'); + final escaped = url.replaceAll("'", "\\'"); + await cdp.evaluate("window.location.href = '$escaped'"); + await Future.delayed(const Duration(seconds: 4)); + } + } catch (_) { + // Ignore errors in detection — non-critical + } +} + +/// Handle HTTP requests +Future _handleRequest( + HttpRequest request, + CdpDriver cdp, + _ServeState state, +) async { + final tools = state.tools; + final path = request.uri.path; + final method = request.method; + final response = request.response; + + // CORS + response.headers.add('Access-Control-Allow-Origin', '*'); + response.headers.add('Access-Control-Allow-Methods', 'GET, POST, OPTIONS'); + response.headers.add('Access-Control-Allow-Headers', 'Content-Type'); + + if (method == 'OPTIONS') { + response.statusCode = 200; + await response.close(); + return; + } + + switch (path) { + case '/': + case '/health': + response + ..statusCode = 200 + ..headers.contentType = ContentType.json + ..write(jsonEncode({ + 'status': 'ok', + 'service': 'flutter-skill-webmcp', + 'version': '0.8.5', + 'tools': tools.length, + 'capabilities': [ + 'auto-discover', + 'hot-reload', + 'zero-config', + 'snapshot', + 'screenshot', + ], + })); + await response.close(); + + case '/tools/list': + case '/tools': + final toolDefs = tools.map(_toMcpToolDef).toList(); + // Add built-in CDP tools + toolDefs.addAll(_builtInCdpToolDefs()); + response + ..statusCode = 200 + ..headers.contentType = ContentType.json + ..write(jsonEncode({'tools': toolDefs})); + await response.close(); + + case '/tools/call': + if (method != 'POST') { + response + ..statusCode = 405 + ..write('POST required'); + await response.close(); + return; + } + final body = jsonDecode(await utf8.decoder.bind(request).join()) + as Map; + final toolName = body['name'] as String?; + final toolArgs = + (body['arguments'] ?? body['params'] ?? {}) as Map; + + if (toolName == null) { + response + ..statusCode = 400 + ..headers.contentType = ContentType.json + ..write(jsonEncode({'error': 'Missing "name" field'})); + await response.close(); + return; + } + + print(' 🔧 Calling tool: $toolName'); + + // Built-in tools handled by serve + if (toolName == 'reset_app') { + final clearStorage = toolArgs['clear_storage'] ?? true; + final clearCookies = toolArgs['clear_cookies'] ?? true; + final actions = []; + if (clearStorage) { + await cdp.call('Runtime.evaluate', { + 'expression': 'localStorage.clear(); sessionStorage.clear();', + 'returnByValue': true, + }); + actions.add('storage cleared'); + } + if (clearCookies) { + try { + await cdp.call('Network.enable'); + await cdp.call('Network.clearBrowserCookies'); + } catch (_) {} + actions.add('cookies cleared'); + } + await cdp.call('Page.reload', {'ignoreCache': true}); + await Future.delayed(const Duration(seconds: 2)); + // Re-scan tools after reset + state.tools = await _discoverTools(cdp); + state.updatedAt = DateTime.now(); + actions.add('page reloaded'); + response + ..statusCode = 200 + ..headers.contentType = ContentType.json + ..write(jsonEncode({ + 'success': true, + 'actions': actions, + 'tools': state.tools.length, + })); + await response.close(); + return; + } + + if (toolName == 'snapshot') { + // Redirect to /snapshot + final result = await cdp.call('Runtime.evaluate', { + 'expression': _snapshotJs, + 'returnByValue': true, + }); + final text = result['result']?['value'] as String? ?? ''; + response + ..statusCode = 200 + ..headers.contentType = ContentType.json + ..write(jsonEncode({'snapshot': text, 'length': text.length})); + await response.close(); + return; + } + + // Route built-in CDP tools that need native Dart handling + final builtInResult = + await _handleBuiltInCdpTool(cdp, toolName, toolArgs, state); + if (builtInResult != null) { + response + ..statusCode = 200 + ..headers.contentType = ContentType.json + ..write(jsonEncode(builtInResult)); + await response.close(); + return; + } + + final result = await cdp.callTool(toolName, toolArgs); + response + ..statusCode = 200 + ..headers.contentType = ContentType.json + ..write(jsonEncode(result)); + await response.close(); + + case '/snapshot': + // Text-based page snapshot (token efficient) + final result = await cdp.call('Runtime.evaluate', { + 'expression': _snapshotJs, + 'returnByValue': true, + }); + final text = result['result']?['value'] as String? ?? ''; + response + ..statusCode = 200 + ..headers.contentType = ContentType.json + ..write(jsonEncode({'snapshot': text, 'length': text.length})); + await response.close(); + + case '/screenshot': + final data = await cdp.takeScreenshot(quality: 0.8, maxWidth: 1280); + if (data != null) { + response + ..statusCode = 200 + ..headers.set('Content-Type', 'image/jpeg') + ..add(base64Decode(data)); + } else { + response + ..statusCode = 500 + ..write('Screenshot failed'); + } + await response.close(); + + case '/navigate': + if (method != 'POST') { + response + ..statusCode = 405 + ..write('POST required'); + await response.close(); + return; + } + final body = jsonDecode(await utf8.decoder.bind(request).join()) + as Map; + final navUrl = body['url'] as String?; + if (navUrl == null) { + response + ..statusCode = 400 + ..write('Missing "url" field'); + await response.close(); + return; + } + print(' 🌐 Navigating to: $navUrl'); + try { + // Track the origin so reconnect logic prefers this tab + final navUri = Uri.parse(navUrl); + state.lastNavigatedOrigin = '${navUri.scheme}://${navUri.host}'; + // Navigate in the current tab — find matching tab or open new one + await _navigateToUrl(cdp, navUrl, state.cdpPort); + // Re-setup observers and re-scan tools + await _setupHotReloadDetection(cdp); + state.tools = await _discoverAndPrint(cdp); + state.updatedAt = DateTime.now(); + response + ..statusCode = 200 + ..headers.contentType = ContentType.json + ..write( + jsonEncode({'navigated': navUrl, 'tools': state.tools.length})); + } catch (e) { + // Try to recover from CDP target loss + if (e.toString().contains('-32000') || + e.toString().contains('Inspected target navigated or closed')) { + print(' ⚠️ CDP target lost during navigation, reconnecting...'); + await Future.delayed(const Duration(seconds: 1)); + await _reconnectCdpTarget(cdp, state.cdpPort); + state.tools = await _discoverAndPrint(cdp); + state.updatedAt = DateTime.now(); + response + ..statusCode = 200 + ..headers.contentType = ContentType.json + ..write(jsonEncode({ + 'navigated': navUrl, + 'tools': state.tools.length, + 'reconnected': true + })); + } else { + response + ..statusCode = 500 + ..headers.contentType = ContentType.json + ..write(jsonEncode({'error': e.toString()})); + } + } + await response.close(); + + case '/refresh': + print(' 🔄 Force refresh tools'); + state.tools = await _discoverAndPrint(cdp); + state.updatedAt = DateTime.now(); + response + ..statusCode = 200 + ..headers.contentType = ContentType.json + ..write(jsonEncode({'tools': state.tools.length, 'refreshed': true})); + await response.close(); + + default: + response + ..statusCode = 404 + ..write('Not found: $path'); + await response.close(); + } +} + +/// Convert internal tool format to MCP tool definition +Map _toMcpToolDef(Map tool) { + final params = tool['params'] as Map? ?? {}; + final properties = {}; + final required = []; + + params.forEach((key, value) { + if (value is Map) { + properties[key] = { + 'type': value['type'] ?? 'string', + 'description': value['description'] ?? '', + }; + if (value['required'] == true) required.add(key); + } + }); + + return { + 'name': tool['name'], + 'description': tool['description'] ?? '', + 'inputSchema': { + 'type': 'object', + 'properties': properties, + if (required.isNotEmpty) 'required': required, + }, + }; +} + +/// Discover tools and print summary +Future>> _discoverAndPrint(CdpDriver cdp) async { + final tools = await _discoverTools(cdp); + + // Group by source + final bySource = {}; + for (final t in tools) { + final source = t['source'] as String? ?? 'unknown'; + bySource[source] = (bySource[source] ?? 0) + 1; + } + + print(' 📦 ${tools.length} tools discovered:'); + bySource.forEach((source, count) { + final icon = switch (source) { + 'auto-ui' => '🔍 UI elements', + 'auto-form' => '📝 Forms', + 'js-registered' => '📦 JS registered', + 'data-mcp-tool' => '🏷️ Annotated', + 'well-known' => '🌐 .well-known', + 'link-manifest' => '🔗 Manifest', + _ => '❓ $source', + }; + print(' $icon: $count'); + }); + + return tools; +} + +/// Raw tool discovery +Future>> _discoverTools(CdpDriver cdp) async { + final result = await cdp.discoverTools(); + return (result['tools'] as List?)?.cast>() ?? []; +} + +/// Set up hot reload / HMR / DOM mutation detection +Future _setupHotReloadDetection(CdpDriver cdp) async { + await cdp.call('Runtime.evaluate', { + 'expression': ''' + (() => { + if (window.__fs_serve_observer__) return 'already active'; + + window.__fs_dom_change_count__ = 0; + window.__fs_last_checked__ = 0; + + // 1. DOM MutationObserver — catches all framework renders + const observer = new MutationObserver((mutations) => { + const significant = mutations.some(m => + m.type === 'childList' && (m.addedNodes.length > 0 || m.removedNodes.length > 0) + ); + if (significant) window.__fs_dom_change_count__++; + }); + observer.observe(document.body || document.documentElement, { + childList: true, + subtree: true, + }); + window.__fs_serve_observer__ = observer; + + // 2. Navigation events + window.addEventListener('popstate', () => window.__fs_dom_change_count__++); + window.addEventListener('hashchange', () => window.__fs_dom_change_count__++); + + // 3. Vite HMR + try { + if (import.meta?.hot) { + import.meta.hot.on('vite:afterUpdate', () => window.__fs_dom_change_count__ += 10); + import.meta.hot.on('vite:fullReload', () => window.__fs_dom_change_count__ += 10); + } + } catch(e) {} + + // 4. Webpack HMR + try { + if (module?.hot) { + module.hot.addStatusHandler((status) => { + if (status === 'apply') window.__fs_dom_change_count__ += 10; + }); + } + } catch(e) {} + + // 5. React Fast Refresh (detect re-renders) + // React DevTools hook fires on commit + try { + if (window.__REACT_DEVTOOLS_GLOBAL_HOOK__) { + const orig = window.__REACT_DEVTOOLS_GLOBAL_HOOK__.onCommitFiberRoot; + if (orig) { + window.__REACT_DEVTOOLS_GLOBAL_HOOK__.onCommitFiberRoot = function(...args) { + window.__fs_dom_change_count__++; + return orig.apply(this, args); + }; + } + } + } catch(e) {} + + // 6. Flutter Web hot reload (service worker message) + try { + navigator.serviceWorker?.addEventListener('message', (e) => { + if (e.data?.type === 'FLUTTER_REASSEMBLE') { + window.__fs_dom_change_count__ += 10; + } + }); + } catch(e) {} + + return 'observers installed'; + })() + ''', + 'returnByValue': true, + }); +} + +/// Check if page changed since last check +Future _checkForChanges(CdpDriver cdp) async { + final result = await cdp.call('Runtime.evaluate', { + 'expression': ''' + (() => { + const c = window.__fs_dom_change_count__ || 0; + const last = window.__fs_last_checked__ || 0; + window.__fs_last_checked__ = c; + return c > last; + })() + ''', + 'returnByValue': true, + }); + return result['result']?['value'] == true; +} + +/// JS for text-based page snapshot +const _snapshotJs = ''' +(() => { + const lines = []; + const walk = (node, depth) => { + if (node.nodeType === 3) { + const t = node.textContent.trim(); + if (t) lines.push(' '.repeat(depth) + t); + return; + } + if (node.nodeType !== 1) return; + const el = node; + const tag = el.tagName.toLowerCase(); + if (['script','style','noscript','svg','path'].includes(tag)) return; + if (el.hidden || el.style.display === 'none') return; + + const role = el.getAttribute('role') || ''; + const label = el.getAttribute('aria-label') || ''; + const href = el.getAttribute('href') || ''; + + let prefix = ''; + if (tag === 'button' || role === 'button') prefix = '[button] '; + else if (tag === 'a' && href) prefix = '[link] '; + else if (tag === 'input') prefix = '[input:' + (el.type||'text') + '] '; + else if (tag === 'textarea') prefix = '[textarea] '; + else if (tag === 'select') prefix = '[select] '; + else if (tag === 'img') { lines.push(' '.repeat(depth) + '[img] ' + (el.alt||'')); return; } + else if (['h1','h2','h3','h4','h5','h6'].includes(tag)) prefix = '[' + tag + '] '; + + if (prefix && label) { + lines.push(' '.repeat(depth) + prefix + label); + } + + for (const child of el.childNodes) walk(child, depth + (prefix ? 1 : 0)); + }; + walk(document.body, 0); + return lines.join('\\n'); +})() +'''; + +/// Built-in CDP tool definitions for /tools/list +List> _builtInCdpToolDefs() => [ + { + 'name': 'tap', + 'description': + 'Tap/click an element by text, CSS selector, ref, or x,y coordinates', + 'inputSchema': { + 'type': 'object', + 'properties': { + 'text': { + 'type': 'string', + 'description': 'Visible text of element to tap' + }, + 'selector': {'type': 'string', 'description': 'CSS selector'}, + 'key': { + 'type': 'string', + 'description': 'CSS selector (alias for selector)' + }, + 'ref': { + 'type': 'string', + 'description': 'Element ref from snapshot' + }, + 'x': {'type': 'number', 'description': 'X coordinate'}, + 'y': {'type': 'number', 'description': 'Y coordinate'}, + } + } + }, + { + 'name': 'type_text', + 'description': + 'Type text via keyboard (into focused element or specified selector)', + 'inputSchema': { + 'type': 'object', + 'properties': { + 'text': {'type': 'string', 'description': 'Text to type'}, + }, + 'required': ['text'] + } + }, + { + 'name': 'screenshot', + 'description': 'Take a screenshot of the current page', + 'inputSchema': { + 'type': 'object', + 'properties': { + 'quality': { + 'type': 'number', + 'description': 'JPEG quality 0.0-1.0 (default 0.8)' + }, + 'max_width': { + 'type': 'integer', + 'description': 'Max width in pixels (default 1280)' + }, + } + } + }, + { + 'name': 'snapshot', + 'description': + 'Get a text-based accessibility snapshot of the page (token-efficient)', + 'inputSchema': {'type': 'object', 'properties': {}} + }, + { + 'name': 'upload_file', + 'description': + 'Upload file(s) to an input[type=file] element. Supports Shadow DOM.', + 'inputSchema': { + 'type': 'object', + 'properties': { + 'selector': { + 'type': 'string', + 'description': + 'CSS selector for file input (default: input[type="file"])' + }, + 'files': { + 'type': 'array', + 'items': {'type': 'string'}, + 'description': 'List of absolute file paths to upload' + }, + }, + 'required': ['files'] + } + }, + { + 'name': 'navigate', + 'description': + 'Navigate to a URL (finds matching tab or navigates current tab)', + 'inputSchema': { + 'type': 'object', + 'properties': { + 'url': {'type': 'string', 'description': 'URL to navigate to'}, + }, + 'required': ['url'] + } + }, + { + 'name': 'evaluate', + 'description': + 'Execute JavaScript in the browser and return the result', + 'inputSchema': { + 'type': 'object', + 'properties': { + 'expression': { + 'type': 'string', + 'description': 'JavaScript expression to evaluate' + }, + }, + 'required': ['expression'] + } + }, + { + 'name': 'scroll', + 'description': 'Scroll to an element by CSS selector or text', + 'inputSchema': { + 'type': 'object', + 'properties': { + 'key': { + 'type': 'string', + 'description': 'CSS selector to scroll to' + }, + 'text': { + 'type': 'string', + 'description': 'Text of element to scroll to' + }, + } + } + }, + { + 'name': 'hover', + 'description': 'Hover over an element', + 'inputSchema': { + 'type': 'object', + 'properties': { + 'key': {'type': 'string', 'description': 'CSS selector'}, + 'text': {'type': 'string', 'description': 'Visible text'}, + 'ref': { + 'type': 'string', + 'description': 'Element ref from snapshot' + }, + } + } + }, + { + 'name': 'press_key', + 'description': 'Press a keyboard key (Enter, Tab, Escape, etc.)', + 'inputSchema': { + 'type': 'object', + 'properties': { + 'key': { + 'type': 'string', + 'description': + 'Key name (Enter, Tab, Escape, Backspace, ArrowDown, etc.)' + }, + 'modifiers': { + 'type': 'string', + 'description': + 'Comma-separated modifiers: Alt, Control, Meta, Shift' + }, + }, + 'required': ['key'] + } + }, + { + 'name': 'select_option', + 'description': 'Select an option from a dropdown", + "inputSchema": { + "type": "object", + "properties": { + "key": {"type": "string", "description": "Element ID or test ID"}, + "value": {"type": "string", "description": "Option value to select"} + }, + "required": ["key", "value"] + } + }, + { + "name": "set_checkbox", + "description": "Check or uncheck a checkbox", + "inputSchema": { + "type": "object", + "properties": { + "key": {"type": "string"}, + "checked": {"type": "boolean"} + }, + "required": ["key"] + } + }, + { + "name": "fill", + "description": + "Fill an input field (clear + set value — faster than enter_text for forms)", + "inputSchema": { + "type": "object", + "properties": { + "key": {"type": "string"}, + "value": {"type": "string"} + }, + "required": ["key", "value"] + } + }, + { + "name": "get_cookies", + "description": "Get all browser cookies for the current page", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "set_cookie", + "description": "Set a browser cookie", + "inputSchema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "value": {"type": "string"}, + "domain": {"type": "string"}, + "path": {"type": "string"} + }, + "required": ["name", "value"] + } + }, + { + "name": "clear_cookies", + "description": "Clear all browser cookies", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "get_local_storage", + "description": "Get all localStorage key-value pairs", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "set_local_storage", + "description": "Set a localStorage value", + "inputSchema": { + "type": "object", + "properties": { + "key": {"type": "string"}, + "value": {"type": "string"} + }, + "required": ["key", "value"] + } + }, + { + "name": "clear_local_storage", + "description": "Clear all localStorage data", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "get_console_messages", + "description": "Get browser console log messages", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "get_network_requests", + "description": + "Get all network requests made by the page (via Performance API)", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "set_viewport", + "description": "Set browser viewport size (responsive testing)", + "inputSchema": { + "type": "object", + "properties": { + "width": {"type": "integer"}, + "height": {"type": "integer"}, + "device_scale_factor": {"type": "number"} + }, + "required": ["width", "height"] + } + }, + { + "name": "emulate_device", + "description": + "Emulate a device viewport + user agent. 143+ presets: iPhone 12-16 (all sizes), SE, Pixel 5-9, Galaxy S21-S24, Z Fold/Flip, OnePlus, Xiaomi, Huawei, iPad Pro/Air/Mini, Galaxy Tab, Surface Pro, MacBook Air/Pro, Dell XPS, desktop resolutions (1080p/1440p/4K) with Chrome/Firefox/Safari/Edge UAs. Supports flexible naming: 'iPhone 14 Pro', 'iphone-14-pro', 'iphone14pro' all work. Pass empty device to list all available presets.", + "inputSchema": { + "type": "object", + "properties": { + "device": { + "type": "string", + "description": + "Device name (e.g. 'iphone-16-pro-max', 'pixel-8', 'galaxy-s24-ultra', 'ipad-pro-11', 'macbook-pro-16', 'desktop-1080p'). Empty string lists all devices." + } + }, + "required": ["device"] + } + }, + { + "name": "generate_pdf", + "description": "Generate a PDF of the current page", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "navigate", + "description": "Navigate to a URL", + "inputSchema": { + "type": "object", + "properties": { + "url": {"type": "string"} + }, + "required": ["url"] + } + }, + { + "name": "go_forward", + "description": "Navigate forward in browser history", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "reload", + "description": "Reload the current page", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "get_attribute", + "description": "Get an HTML element's attribute value", + "inputSchema": { + "type": "object", + "properties": { + "key": {"type": "string"}, + "attribute": {"type": "string"} + }, + "required": ["key", "attribute"] + } + }, + { + "name": "get_css_property", + "description": "Get computed CSS property of an element", + "inputSchema": { + "type": "object", + "properties": { + "key": {"type": "string"}, + "property": {"type": "string"} + }, + "required": ["key", "property"] + } + }, + { + "name": "get_bounding_box", + "description": "Get element position and size", + "inputSchema": { + "type": "object", + "properties": { + "key": {"type": "string"} + }, + "required": ["key"] + } + }, + { + "name": "focus", + "description": "Focus an element", + "inputSchema": { + "type": "object", + "properties": { + "key": {"type": "string"} + }, + "required": ["key"] + } + }, + { + "name": "blur", + "description": "Remove focus from an element", + "inputSchema": { + "type": "object", + "properties": { + "key": {"type": "string"} + }, + "required": ["key"] + } + }, + { + "name": "count_elements", + "description": "Count elements matching a CSS selector", + "inputSchema": { + "type": "object", + "properties": { + "selector": {"type": "string"} + }, + "required": ["selector"] + } + }, + { + "name": "is_visible", + "description": "Check if an element is visible on page", + "inputSchema": { + "type": "object", + "properties": { + "key": {"type": "string"} + }, + "required": ["key"] + } + }, + { + "name": "get_page_source", + "description": + "Get the HTML source of the current page with optional cleaning", + "inputSchema": { + "type": "object", + "properties": { + "selector": { + "type": "string", + "description": + "CSS selector to get HTML for a specific element only" + }, + "remove_scripts": { + "type": "boolean", + "description": "Strip