diff --git a/lib/flutter_skill.dart b/lib/flutter_skill.dart index a9b63aa7..69264625 100644 --- a/lib/flutter_skill.dart +++ b/lib/flutter_skill.dart @@ -5,8 +5,10 @@ import 'dart:math'; import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; +import 'package:flutter/gestures.dart' show PointerAddedEvent; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; +import 'package:flutter/scheduler.dart'; import 'package:flutter/services.dart'; import 'flutter_skill_semantic_refs.dart'; @@ -409,10 +411,10 @@ class FlutterSkillBinding { final success = await _performSwipe( direction: direction, distance: distance, key: key); return developer.ServiceExtensionResponse.result( - jsonEncode({ + jsonEncode(_withFrameStatus({ 'success': success, 'message': success ? 'Swipe successful' : 'Swipe failed' - }), + })), ); } catch (e, stack) { return _errorResponse(e, stack); @@ -425,12 +427,14 @@ class FlutterSkillBinding { try { final fromKey = parameters['fromKey']; final toKey = parameters['toKey']; - final success = await _performDrag(fromKey: fromKey, toKey: toKey); + final holdMs = int.tryParse(parameters['hold'] ?? '0') ?? 0; + final success = + await _performDrag(fromKey: fromKey, toKey: toKey, holdMs: holdMs); return developer.ServiceExtensionResponse.result( - jsonEncode({ + jsonEncode(_withFrameStatus({ 'success': success, 'message': success ? 'Drag successful' : 'Drag failed' - }), + })), ); } catch (e, stack) { return _errorResponse(e, stack); @@ -561,10 +565,20 @@ class FlutterSkillBinding { try { final quality = double.tryParse(parameters['quality'] ?? '1.0') ?? 1.0; final maxWidth = int.tryParse(parameters['maxWidth'] ?? ''); - final base64Image = + final shot = await _takeScreenshot(quality: quality, maxWidth: maxWidth); - return developer.ServiceExtensionResponse.result( - jsonEncode({'image': base64Image})); + // Logical size + DPR let callers map image pixels back to the + // logical coordinates that tap_at/swipe expect. + final view = WidgetsBinding.instance.platformDispatcher.views.first; + final logical = view.physicalSize / view.devicePixelRatio; + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'image': shot?.base64, + 'imageWidth': shot?.width, + 'imageHeight': shot?.height, + 'logicalWidth': logical.width, + 'logicalHeight': logical.height, + 'devicePixelRatio': view.devicePixelRatio, + })); } catch (e, stack) { return _errorResponse(e, stack); } @@ -613,7 +627,7 @@ class FlutterSkillBinding { try { final route = _getCurrentRoute(); return developer.ServiceExtensionResponse.result( - jsonEncode({'route': route})); + jsonEncode({'route': route, 'routes': _routeEntries()})); } catch (e, stack) { return _errorResponse(e, stack); } @@ -713,6 +727,144 @@ class FlutterSkillBinding { } }); + // ==================== FOCUS / TYPING / TOGGLES / HOVER ==================== + + // type_text: insert into the focused field the way an IME commits text + // (replaces the current selection), so onChanged fires as for a user. + developer.registerExtension('ext.flutter.flutter_skill.typeText', + (method, parameters) async { + try { + final text = parameters['text'] ?? ''; + final field = _findFocusedTextField(); + if (field == null) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'No focused text field. Use focus(key) or tap it first.', + })); + } + final value = field.textEditingValue; + final sel = value.selection.isValid + ? value.selection + : TextSelection.collapsed(offset: value.text.length); + final newText = value.text.replaceRange(sel.start, sel.end, text); + field.updateEditingValue(TextEditingValue( + text: newText, + selection: TextSelection.collapsed(offset: sel.start + text.length), + )); + await _settle(); + return developer.ServiceExtensionResponse.result(jsonEncode( + {'success': true, 'message': 'Typed "$text"', 'value': newText})); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // focus / blur: drive the field's own FocusNode. + developer.registerExtension('ext.flutter.flutter_skill.focus', + (method, parameters) async { + try { + final key = parameters['key']; + final field = key == null ? null : _findEditableText(key); + if (field == null) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'No text field with key "$key"', + })); + } + field.widget.focusNode.requestFocus(); + await _settle(); + return developer.ServiceExtensionResponse.result( + jsonEncode({'success': true, 'message': 'Focused "$key"'})); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + developer.registerExtension('ext.flutter.flutter_skill.blur', + (method, parameters) async { + try { + final key = parameters['key']; + final field = + key == null ? _findFocusedTextField() : _findEditableText(key); + if (field == null) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'No focused text field to blur', + })); + } + field.widget.focusNode.unfocus(); + await _settle(); + return developer.ServiceExtensionResponse.result( + jsonEncode({'success': true, 'message': 'Blurred'})); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // set_checkbox: tap the toggle until it shows the requested state. + developer.registerExtension('ext.flutter.flutter_skill.setCheckbox', + (method, parameters) async { + try { + final key = parameters['key'] ?? ''; + final wanted = parameters['checked'] == 'true'; + final current = _getCheckboxState(key); + if (current == null) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'No Checkbox/Switch (or *ListTile) with key "$key"', + })); + } + if (current != wanted) { + await _performTapWithDetails(key: key); + } + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': _getCheckboxState(key) == wanted, + 'checked': _getCheckboxState(key), + })); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // hover: move a mouse pointer over the element (tooltips, hover styles). + developer.registerExtension('ext.flutter.flutter_skill.hover', + (method, parameters) async { + try { + final element = + _findElement(key: parameters['key'], text: parameters['text']); + final box = element?.renderObject; + if (element == null || box is! RenderBox || !box.hasSize) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'Element not found or not laid out', + })); + } + final position = box.localToGlobal(box.size.center(Offset.zero)); + final binding = WidgetsBinding.instance; + // One synthetic mouse device, added once: MouseTracker asserts on a + // repeated PointerAddedEvent for a device that was never removed. + if (!_syntheticMouseAdded) { + binding.handlePointerEvent(PointerAddedEvent( + position: position, + kind: ui.PointerDeviceKind.mouse, + device: _syntheticMouseDevice, + timeStamp: _pointerClock.elapsed)); + _syntheticMouseAdded = true; + } + binding.handlePointerEvent(PointerHoverEvent( + position: position, + kind: ui.PointerDeviceKind.mouse, + device: _syntheticMouseDevice, + timeStamp: _pointerClock.elapsed)); + await _settle(); + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': true, + 'position': {'x': position.dx.round(), 'y': position.dy.round()}, + })); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + // ==================== PRESS KEY ==================== developer.registerExtension('ext.flutter.flutter_skill.pressKey', @@ -765,55 +917,21 @@ class FlutterSkillBinding { // ignore: unused_local_variable final isMeta = modifiers.contains('meta'); - // Simulate key press through the focus system - final focusNode = FocusManager.instance.primaryFocus; - if (focusNode != null) { - // Use HardwareKeyboard simulation - // ignore: unused_local_variable - final binding = WidgetsBinding.instance; - // ignore: unused_local_variable - final pointer = _pointerCounter++; - - // Create a RawKeyDownEvent and dispatch through the focus system - // ignore: unused_local_variable - final keyDown = KeyDownEvent( - physicalKey: PhysicalKeyboardKey.findKeyByCode(logicalKey.keyId) ?? - PhysicalKeyboardKey.enter, - logicalKey: logicalKey, - timeStamp: - Duration(milliseconds: DateTime.now().millisecondsSinceEpoch), - ); - - // Dispatch through ServicesBinding - await ServicesBinding.instance.keyEventManager - .handleKeyData(ui.KeyData( - type: ui.KeyEventType.down, - timeStamp: - Duration(milliseconds: DateTime.now().millisecondsSinceEpoch), - physical: (PhysicalKeyboardKey.findKeyByCode(logicalKey.keyId) ?? - PhysicalKeyboardKey.enter) - .usbHidUsage, - logical: logicalKey.keyId, - character: key.length == 1 ? key : null, - synthesized: false, - )); - - await Future.delayed(const Duration(milliseconds: 50)); - - await ServicesBinding.instance.keyEventManager - .handleKeyData(ui.KeyData( - type: ui.KeyEventType.up, - timeStamp: - Duration(milliseconds: DateTime.now().millisecondsSinceEpoch), - physical: (PhysicalKeyboardKey.findKeyByCode(logicalKey.keyId) ?? - PhysicalKeyboardKey.enter) - .usbHidUsage, - logical: logicalKey.keyId, - character: null, - synthesized: false, - )); + // Dispatch as a real hardware key: down (with modifiers held), up. + // Enter on a focused text field also performs the platform's "done" + // action, which is what fires onSubmitted on desktop. + final handled = await _pressHardwareKey(logicalKey, modifiers, + character: key.length == 1 ? key : null); + if (logicalKey == LogicalKeyboardKey.enter) { + _findFocusedTextField()?.performAction(TextInputAction.done); } - + await _settle(); + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': true, + 'message': 'Key pressed: $key', + 'handled': handled, + })); + // ignore: dead_code return developer.ServiceExtensionResponse.result( jsonEncode({'success': true, 'message': 'Key pressed: $key'}), ); @@ -879,9 +997,9 @@ class FlutterSkillBinding { final x = double.tryParse(parameters['x'] ?? '0') ?? 0; final y = double.tryParse(parameters['y'] ?? '0') ?? 0; await _performTapAt(x, y); - return developer.ServiceExtensionResponse.result( - jsonEncode({'success': true, 'message': 'Tapped at ($x, $y)'}), - ); + return developer.ServiceExtensionResponse.result(jsonEncode( + _withFrameStatus( + {'success': true, 'message': 'Tapped at ($x, $y)'}))); } catch (e, stack) { return _errorResponse(e, stack); } @@ -912,13 +1030,14 @@ class FlutterSkillBinding { final endX = double.tryParse(parameters['endX'] ?? '0') ?? 0; final endY = double.tryParse(parameters['endY'] ?? '0') ?? 0; final duration = int.tryParse(parameters['duration'] ?? '300') ?? 300; + final holdMs = int.tryParse(parameters['hold'] ?? '0') ?? 0; await _performSwipeCoordinates(startX, startY, endX, endY, - duration: duration); + duration: duration, holdMs: holdMs); return developer.ServiceExtensionResponse.result( - jsonEncode({ + jsonEncode(_withFrameStatus({ 'success': true, 'message': 'Swiped from ($startX, $startY) to ($endX, $endY)' - }), + })), ); } catch (e, stack) { return _errorResponse(e, stack); @@ -1507,27 +1626,28 @@ class FlutterSkillBinding { _indicatorOverlay!.showTap(center, hint: "Tapping '$targetText'"); } - await _dispatchTap(center); - - // Fallback: directly invoke the callback if the widget supports it - _tryInvokeCallback(element); - - // Wait for frame to pump after callback invocation - await Future.delayed(const Duration(milliseconds: 300)); + // Exactly one activation: drive the tap through the pointer pipeline + // when the element is really hittable at its center; otherwise (covered + // by an overlay, pointer ignored) invoke the widget's callback directly. + if (_isHittableAt(element, center)) { + await _dispatchTap(center); + } else { + _tryInvokeCallback(element); + await _settle(); + } _log('Tap completed on (key: $key, text: $text)'); - return { + return _withFrameStatus({ 'success': true, 'message': 'Tap successful', 'target': {'key': key, 'text': text}, 'position': {'x': center.dx.round(), 'y': center.dy.round()}, - }; + }); } - /// Try to directly invoke the onPressed/onTap callback of a widget. - /// This is a fallback for when pointer dispatch doesn't trigger the callback - /// (e.g., when an overlay intercepts events or hit testing fails). + /// Directly invoke the onPressed/onTap callback of a widget — used only + /// when the element is not reachable by a pointer (see [_isHittableAt]). static void _tryInvokeCallback(Element element) { if (!element.mounted) return; final widget = element.widget; @@ -1643,10 +1763,197 @@ class FlutterSkillBinding { return scored.map((e) => e.key).toList(); } - static Future _dispatchTap(Offset position) async { + // ==================== SYNTHETIC KEYBOARD ==================== + + /// USB HID usage codes for the physical keys we can synthesize; the + /// framework requires physical/logical pairs to be consistent. + static const Map _physicalUsageByLogicalId = { + 0x100000008: 0x0007002a, // backspace + 0x100000009: 0x0007002b, // tab + 0x10000000d: 0x00070028, // enter + 0x10000001b: 0x00070029, // escape + 0x10000007f: 0x0007004c, // delete + 0x100000301: 0x00070051, // arrowDown + 0x100000302: 0x00070050, // arrowLeft + 0x100000303: 0x0007004f, // arrowRight + 0x100000304: 0x00070052, // arrowUp + 0x100000305: 0x0007004d, // end + 0x100000306: 0x0007004a, // home + 0x100000307: 0x0007004e, // pageDown + 0x100000308: 0x0007004b, // pageUp + 0x20: 0x0007002c, // space + }; + + /// Device id of the synthetic mouse used by `hover` (added on first use). + static const int _syntheticMouseDevice = 0x5ca1; + static bool _syntheticMouseAdded = false; + + static PhysicalKeyboardKey _physicalFor(LogicalKeyboardKey logical) { + final id = logical.keyId; + final mapped = _physicalUsageByLogicalId[id]; + if (mapped != null) return PhysicalKeyboardKey(mapped); + // Unicode letters/digits: keyA..keyZ = 0x70004.., digit1..9,0 = 0x7001e.. + if (id >= 0x61 && id <= 0x7a) + return PhysicalKeyboardKey(0x00070004 + id - 0x61); + if (id >= 0x31 && id <= 0x39) + return PhysicalKeyboardKey(0x0007001e + id - 0x31); + if (id == 0x30) return const PhysicalKeyboardKey(0x00070027); + return PhysicalKeyboardKey.findKeyByCode(id) ?? PhysicalKeyboardKey.enter; + } + + static const Map + _modifierKeys = { + 'shift': (LogicalKeyboardKey.shiftLeft, PhysicalKeyboardKey.shiftLeft), + 'ctrl': (LogicalKeyboardKey.controlLeft, PhysicalKeyboardKey.controlLeft), + 'control': ( + LogicalKeyboardKey.controlLeft, + PhysicalKeyboardKey.controlLeft + ), + 'alt': (LogicalKeyboardKey.altLeft, PhysicalKeyboardKey.altLeft), + 'meta': (LogicalKeyboardKey.metaLeft, PhysicalKeyboardKey.metaLeft), + }; + + /// Delivers one synthetic [KeyEvent] the way the engine's key path does + /// once a platform message is decoded: [HardwareKeyboard] records the + /// pressed state, then the [KeyMessage] goes to the global handler + /// (FocusManager → Shortcuts/Actions → text-editing shortcuts). + /// Returns whether a handler claimed it. + static bool _deliverKey(KeyEvent event) { + HardwareKeyboard.instance.handleKeyEvent(event); + final handler = ServicesBinding.instance.keyEventManager.keyMessageHandler; + return handler?.call(KeyMessage([event], null)) ?? false; + } + + /// Sends a physical key press (down, up) with [modifiers] held around it. + static Future _pressHardwareKey( + LogicalKeyboardKey logical, List modifiers, + {String? character}) async { + Duration now() => _pointerClock.elapsed; + final held = <(LogicalKeyboardKey, PhysicalKeyboardKey)>[]; + for (final m in modifiers) { + final pair = _modifierKeys[m.toLowerCase()]; + if (pair == null) continue; + _deliverKey(KeyDownEvent( + physicalKey: pair.$2, logicalKey: pair.$1, timeStamp: now())); + held.add(pair); + } + final physical = _physicalFor(logical); + // Platforms report no character while a modifier is held (Ctrl+A is a + // shortcut, not text input). + final handled = _deliverKey(KeyDownEvent( + physicalKey: physical, + logicalKey: logical, + character: held.isEmpty ? character : null, + timeStamp: now(), + )); + await Future.delayed(const Duration(milliseconds: 40)); + _deliverKey(KeyUpEvent( + physicalKey: physical, logicalKey: logical, timeStamp: now())); + for (final pair in held.reversed) { + _deliverKey(KeyUpEvent( + physicalKey: pair.$2, logicalKey: pair.$1, timeStamp: now())); + } + return handled; + } + + // ==================== SYNTHETIC POINTER ENGINE ==================== + // + // Every emulated input goes through [_emitGesture]. Flutter's gesture + // recognizers and scroll physics derive velocities from event + // timestamps, so each event carries a monotonically increasing + // [PointerEvent.timeStamp]; the pointer is always released, even if a + // handler throws; and the call returns only after the framework has + // rendered the frame that reflects the input, so a state query issued + // right afterwards observes the result. + + static final Stopwatch _pointerClock = Stopwatch()..start(); + + static Future _emitGesture({ + required Offset from, + Offset? to, + Duration hold = Duration.zero, + Duration travel = const Duration(milliseconds: 300), + }) async { final binding = WidgetsBinding.instance; final pointer = _pointerCounter++; + final end = to ?? from; + Duration now() => _pointerClock.elapsed; + binding.handlePointerEvent( + PointerDownEvent(position: from, pointer: pointer, timeStamp: now())); + try { + if (hold > Duration.zero) await Future.delayed(hold); + if (end != from) { + // About one move per frame; enough steps to clear the touch slop + // early and give the velocity tracker a smooth sample history. + final steps = (travel.inMilliseconds / 16).round().clamp(4, 60); + final stepDelay = travel ~/ steps; + var last = from; + for (var i = 1; i <= steps; i++) { + await Future.delayed(stepDelay); + final position = Offset.lerp(from, end, i / steps)!; + binding.handlePointerEvent(PointerMoveEvent( + position: position, + delta: position - last, + pointer: pointer, + timeStamp: now(), + )); + last = position; + } + } + } finally { + binding.handlePointerEvent( + PointerUpEvent(position: end, pointer: pointer, timeStamp: now())); + } + await _settle(); + } + + /// Completes once two frames have rendered: the one that runs the input's + /// callbacks and the one that paints the resulting state. Each wait is + /// bounded: a minimized/occluded window produces no frames on some + /// platforms (Windows), and an action must never hang on that. + static Future _settle() async { + _framesStalled = !await _nextFrame() && !await _nextFrame(); + } + + /// True when the last action saw no frame at all — the app is most likely + /// minimized or hidden, so nothing it did is observable yet. + static bool _framesStalled = false; + + static const _stalledWarning = + 'No frame was rendered after this action; the app window is probably ' + 'minimized or hidden. Restore it before reading state or taking a ' + 'screenshot.'; + + /// Adds the stalled-frames warning to an action response when relevant. + static Map _withFrameStatus(Map response) => + _framesStalled ? {...response, 'warning': _stalledWarning} : response; + + static Future _nextFrame() => SchedulerBinding.instance.endOfFrame + .then((_) => true) + .timeout(const Duration(milliseconds: 500), onTimeout: () => false); + + /// Whether a pointer at [position] would reach [element] (or a descendant), + /// i.e. nothing opaque covers it and it takes part in hit testing. + static bool _isHittableAt(Element element, Offset position) { + final target = element.renderObject; + if (target == null) return false; + final binding = WidgetsBinding.instance; + final result = HitTestResult(); + binding.hitTestInView( + result, position, binding.platformDispatcher.views.first.viewId); + for (final entry in result.path) { + RenderObject? node = + entry.target is RenderObject ? entry.target as RenderObject : null; + while (node != null) { + if (identical(node, target)) return true; + node = node.parent; + } + } + return false; + } + + static Future _dispatchTap(Offset position) async { // Show animated character walking to position and tapping if (_indicatorsEnabled && _indicatorOverlay != null) { // Use last position as starting point for walking animation @@ -1661,18 +1968,12 @@ class FlutterSkillBinding { await Future.delayed(const Duration(milliseconds: 200)); } - binding.handlePointerEvent( - PointerDownEvent(position: position, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 50)); - binding.handlePointerEvent( - PointerUpEvent(position: position, pointer: pointer)); + await _emitGesture(from: position, hold: const Duration(milliseconds: 50)); // Show tap ripple after character if (_indicatorsEnabled && _indicatorOverlay != null) { _indicatorOverlay!.showTap(position); } - - await Future.delayed(const Duration(milliseconds: 100)); } // ignore: unused_element @@ -1866,11 +2167,16 @@ class FlutterSkillBinding { static Future> _performScroll( {String? key, String? text}) async { - final element = _findElement(key: key, text: text); + var element = _findElement(key: key, text: text); + // Lazily built lists (ListView.builder, slivers) only mount the rows in + // view, so an off-screen target has no element yet. Page through every + // scrollable until it appears, then let ensureVisible do the fine work. + element ??= await _revealByPaging(() => _findElement(key: key, text: text)); if (element == null) { return { 'success': false, - 'error': 'Element not found (key: $key, text: $text)' + 'error': 'Element not found (key: $key, text: $text), ' + 'also after paging through all scrollables', }; } @@ -1914,15 +2220,7 @@ class FlutterSkillBinding { final center = renderObject.localToGlobal(renderObject.size.center(Offset.zero)); - final binding = WidgetsBinding.instance; - final pointer = _pointerCounter++; - - binding.handlePointerEvent( - PointerDownEvent(position: center, pointer: pointer)); - await Future.delayed(Duration(milliseconds: duration)); - binding - .handlePointerEvent(PointerUpEvent(position: center, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + await _emitGesture(from: center, hold: Duration(milliseconds: duration)); _log('Long press completed (key: $key, text: $text)'); return true; @@ -1964,31 +2262,14 @@ class FlutterSkillBinding { return false; } - final pointer = _pointerCounter++; - final end = start + delta; - - binding.handlePointerEvent( - PointerDownEvent(position: start, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 16)); - - const steps = 10; - for (int i = 1; i <= steps; i++) { - final current = Offset.lerp(start, end, i / steps)!; - binding.handlePointerEvent(PointerMoveEvent( - position: current, - pointer: pointer, - delta: delta / steps.toDouble())); - await Future.delayed(const Duration(milliseconds: 16)); - } - - binding.handlePointerEvent(PointerUpEvent(position: end, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + await _emitGesture(from: start, to: start + delta); _log('Swipe $direction completed'); return true; } - static Future _performDrag({String? fromKey, String? toKey}) async { + static Future _performDrag( + {String? fromKey, String? toKey, int holdMs = 0}) async { if (fromKey == null || toKey == null) return false; final fromElement = _findElementByKey(fromKey); @@ -2002,23 +2283,14 @@ class FlutterSkillBinding { final start = fromRender.localToGlobal(fromRender.size.center(Offset.zero)); final end = toRender.localToGlobal(toRender.size.center(Offset.zero)); - final binding = WidgetsBinding.instance; - final pointer = _pointerCounter++; - - binding.handlePointerEvent( - PointerDownEvent(position: start, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); - - const steps = 20; - for (int i = 1; i <= steps; i++) { - final current = Offset.lerp(start, end, i / steps)!; - binding.handlePointerEvent( - PointerMoveEvent(position: current, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 16)); - } - - binding.handlePointerEvent(PointerUpEvent(position: end, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + // `hold` lets LongPressDraggable (and long-press-to-select lists) + // recognise the drag before the pointer starts moving. + await _emitGesture( + from: start, + to: end, + hold: Duration(milliseconds: holdMs), + travel: const Duration(milliseconds: 400), + ); _log('Drag from $fromKey to $toKey completed'); return true; @@ -2045,16 +2317,13 @@ class FlutterSkillBinding { // ==================== COORDINATE-BASED ACTIONS ==================== static Future _performTapAt(double x, double y) async { - final position = Offset(x, y); - await _dispatchTap(position); + await _dispatchTap(Offset(x, y)); _log('Tap at coordinates ($x, $y) completed'); } static Future _performLongPressAt(double x, double y, {int duration = 500}) async { final position = Offset(x, y); - final binding = WidgetsBinding.instance; - final pointer = _pointerCounter++; // Show animated character holding if (_indicatorsEnabled && _indicatorOverlay != null) { @@ -2067,12 +2336,7 @@ class FlutterSkillBinding { ); } - binding.handlePointerEvent( - PointerDownEvent(position: position, pointer: pointer)); - await Future.delayed(Duration(milliseconds: duration)); - binding.handlePointerEvent( - PointerUpEvent(position: position, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + await _emitGesture(from: position, hold: Duration(milliseconds: duration)); _log('Long press at coordinates ($x, $y) completed'); } @@ -2083,10 +2347,8 @@ class FlutterSkillBinding { double endX, double endY, { int duration = 300, + int holdMs = 0, }) async { - final binding = WidgetsBinding.instance; - final pointer = _pointerCounter++; - final start = Offset(startX, startY); final end = Offset(endX, endY); final delta = end - start; @@ -2100,25 +2362,12 @@ class FlutterSkillBinding { _indicatorOverlay!.showSwipe(start, end); } - binding.handlePointerEvent( - PointerDownEvent(position: start, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 16)); - - final steps = (duration / 16).round().clamp(5, 30); - final stepDuration = duration ~/ steps; - - for (int i = 1; i <= steps; i++) { - final current = Offset.lerp(start, end, i / steps)!; - binding.handlePointerEvent(PointerMoveEvent( - position: current, - pointer: pointer, - delta: delta / steps.toDouble(), - )); - await Future.delayed(Duration(milliseconds: stepDuration)); - } - - binding.handlePointerEvent(PointerUpEvent(position: end, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + await _emitGesture( + from: start, + to: end, + hold: Duration(milliseconds: holdMs), + travel: Duration(milliseconds: duration), + ); _log('Swipe from ($startX, $startY) to ($endX, $endY) completed'); } @@ -2352,6 +2601,12 @@ class FlutterSkillBinding { }; entry['visible'] = hasValidSize; entry['coordinatesReliable'] = coordinatesReliable; + // `visible` only says the box has a size; `hittable` says a pointer + // at its center actually reaches it (not offstage, not covered by + // a bar/overlay) — the flag to trust before tapping/dragging. + entry['hittable'] = coordinatesReliable && + _isHittableAt( + element, offset + renderObject.size.center(Offset.zero)); // Add warning for unreliable coordinates if (!coordinatesReliable && isAtOrigin) { @@ -2573,6 +2828,8 @@ class FlutterSkillBinding { offset.dx.isFinite && offset.dy.isFinite; state['visible'] = isVisible; + state['hittable'] = isVisible && + _isHittableAt(element, offset + size.center(Offset.zero)); // Cache element for performance optimization if (isVisible && refId.isNotEmpty) { @@ -2582,6 +2839,7 @@ class FlutterSkillBinding { // Element has no render box or size - set default bounds elementEntry['bounds'] = {'x': 0, 'y': 0, 'w': 0, 'h': 0}; state['visible'] = false; + state['hittable'] = false; } // Merge state information @@ -2878,15 +3136,32 @@ class FlutterSkillBinding { if (element == null) return null; final widget = element.widget; - if (widget is Checkbox) { - return widget.value; - } - if (widget is Switch) { - return widget.value; - } + if (widget is Checkbox) return widget.value; + if (widget is Switch) return widget.value; + if (widget is CheckboxListTile) return widget.value; + if (widget is SwitchListTile) return widget.value; return null; } + /// The EditableText inside the widget with [key] (TextField, TextFormField + /// or a wrapper), or null. + static EditableTextState? _findEditableText(String key) { + final element = _findElementByKey(key); + if (element == null) return null; + EditableTextState? found; + void visit(Element e) { + if (found != null) return; + if (e is StatefulElement && e.state is EditableTextState) { + found = e.state as EditableTextState; + return; + } + e.visitChildren(visit); + } + + visit(element); + return found; + } + static double? _getSliderValue(String key) { final element = _findElementByKey(key); if (element == null) return null; @@ -2931,7 +3206,8 @@ class FlutterSkillBinding { /// avoiding the bug where DFS finds a RepaintBoundary from an old route. /// Returns the image in logical pixel dimensions scaled by [pixelRatio]. static Future _captureFullScene({double pixelRatio = 1.0}) async { - await WidgetsBinding.instance.endOfFrame; + // Bounded: a hidden window may never deliver a frame (see _nextFrame). + await _nextFrame(); final binding = WidgetsBinding.instance; // ignore: invalid_use_of_protected_member @@ -2982,7 +3258,7 @@ class FlutterSkillBinding { return boundary!.toImage(pixelRatio: pixelRatio); } - static Future _takeScreenshot( + static Future<_Screenshot?> _takeScreenshot( {double quality = 1.0, int? maxWidth}) async { try { // Use quality as pixel ratio (lower = smaller image) @@ -3014,7 +3290,8 @@ class FlutterSkillBinding { final byteData = await image.toByteData(format: ui.ImageByteFormat.png); if (byteData == null) return null; - return base64Encode(byteData.buffer.asUint8List()); + return _Screenshot(base64Encode(byteData.buffer.asUint8List()), + image.width, image.height); } catch (e) { _log('Screenshot failed: $e'); return null; @@ -3126,26 +3403,56 @@ class FlutterSkillBinding { // ==================== NAVIGATION ==================== - static String? _getCurrentRoute() { - String? currentRoute; - final binding = WidgetsBinding.instance; - - void visit(Element element) { - if (element.widget is ModalRoute) { - final route = ModalRoute.of(element); - if (route != null) { - currentRoute = route.settings.name; + /// Every route currently mounted, in stack order (outer navigator first, + /// each navigator bottom → top, nested navigators after the route that + /// hosts them). Each entry: `name` (RouteSettings.name — go_router sets + /// the route name or path), `key` (Page key), `isCurrent`, `depth` + /// (navigator nesting), `type`. + /// + /// Routes are found through the `_ModalScopeStatus` inherited widget that + /// every ModalRoute installs above its content: walking the element tree + /// meets each route exactly once and does not register dependencies (as + /// `ModalRoute.of` would). Field access is dynamic because the class is + /// private to the framework; its members are public and stable. + static List> _routeEntries() { + final entries = >[]; + void visit(Element element, int depth) { + var next = depth; + final widget = element.widget; + if (widget.runtimeType.toString() == '_ModalScopeStatus') { + try { + final route = (widget as dynamic).route as Route; + final settings = route.settings; + entries.add({ + 'name': settings.name, + 'key': settings is Page ? settings.key?.toString() : null, + 'isCurrent': route.isCurrent, + 'depth': depth, + 'type': route.runtimeType.toString(), + }); + next = depth + 1; + } catch (_) { + // framework internals changed shape; skip silently } } - element.visitChildren(visit); + element.visitChildren((child) => visit(child, next)); } // ignore: invalid_use_of_protected_member - if (binding.rootElement != null) { - visit(binding.rootElement!); - } + final rootElement = WidgetsBinding.instance.rootElement; + if (rootElement != null) visit(rootElement, 0); + return entries; + } - return currentRoute; + /// Name of the topmost current route (innermost navigator wins). Falls + /// back to the page key when the route has no name. + static String? _getCurrentRoute() { + Map? current; + for (final entry in _routeEntries()) { + if (entry['isCurrent'] == true) current = entry; + } + if (current == null) return null; + return (current['name'] ?? current['key']) as String?; } static bool _goBack() { @@ -3162,16 +3469,50 @@ class FlutterSkillBinding { static List _getNavigationStack() { final routes = []; - final context = _findNavigatorContext(); - if (context == null) return routes; + for (final entry in _routeEntries()) { + final label = (entry['name'] ?? entry['key'] ?? entry['type']) as String; + routes.add(label); + } + return routes; + } - // This is a simplified version - full implementation would need NavigatorState access - final currentRoute = _getCurrentRoute(); - if (currentRoute != null) { - routes.add(currentRoute); + /// Pages every mounted vertical/horizontal scrollable from its start to + /// its end (viewport-sized jumps, one frame each) until [find] returns an + /// element. Returns it, or null when nothing appeared anywhere. + static Future _revealByPaging(Element? Function() find) async { + final positions = []; + void visit(Element e) { + if (e is StatefulElement && e.state is ScrollableState) { + final position = (e.state as ScrollableState).position; + if (position.hasContentDimensions && + position.maxScrollExtent > position.minScrollExtent) { + positions.add(position); + } + } + e.visitChildren(visit); } - return routes; + // ignore: invalid_use_of_protected_member + final rootElement = WidgetsBinding.instance.rootElement; + if (rootElement != null) visit(rootElement); + + for (final position in positions) { + final origin = position.pixels; + final step = position.viewportDimension * 0.8; + var offset = position.minScrollExtent; + while (true) { + position.jumpTo(offset); + await _settle(); + final found = find(); + if (found != null) return found; + if (offset >= position.maxScrollExtent) break; + offset = (offset + step) + .clamp(position.minScrollExtent, position.maxScrollExtent); + } + position.jumpTo(origin); + await _settle(); + } + return null; } static BuildContext? _findNavigatorContext() { @@ -4196,7 +4537,7 @@ class _ParticleEffectPainter extends CustomPainter { const Color(0xFFFF5722), i / 6, )! - .withOpacity(0.8 * (1 - progress)); + .withValues(alpha: (0.8 * (1 - progress)).clamp(0.0, 1.0)); particlePaint.color = color; canvas.drawCircle(Offset(x, y), 3 * (1 - progress), particlePaint); @@ -4906,3 +5247,11 @@ class _ActionHint extends StatelessWidget { ); } } + +/// PNG screenshot payload with its pixel dimensions. +class _Screenshot { + const _Screenshot(this.base64, this.width, this.height); + final String base64; + final int width; + final int height; +} diff --git a/lib/src/cli/server.dart b/lib/src/cli/server.dart index ace40567..df97b9f9 100644 --- a/lib/src/cli/server.dart +++ b/lib/src/cli/server.dart @@ -728,11 +728,17 @@ class FlutterMcpServer { final actions = (args['actions'] ?? args['commands'] ?? []) as List; final stopOnFailure = args['stop_on_failure'] ?? true; + // A step often triggers a route/tab transition; give the next step's + // target time to be built (default 250 ms, tunable per batch). + final stepDelayMs = (args['step_delay_ms'] as num?)?.toInt() ?? 250; final results = >[]; var allSuccess = true; for (var i = 0; i < actions.length; i++) { + if (i > 0 && stepDelayMs > 0) { + await Future.delayed(Duration(milliseconds: stepDelayMs)); + } final action = actions[i] as Map; final actionName = (action['action'] ?? action['tool'] ?? action['name']) as String; diff --git a/lib/src/cli/tool_handlers/bf_batch.dart b/lib/src/cli/tool_handlers/bf_batch.dart index 43ea80cd..3ffcba6c 100644 --- a/lib/src/cli/tool_handlers/bf_batch.dart +++ b/lib/src/cli/tool_handlers/bf_batch.dart @@ -50,8 +50,14 @@ extension _BfBatch on FlutterMcpServer { final fc = _asFlutterClient(client!, 'tap_at'); final x = (args['x'] as num).toDouble(); final y = (args['y'] as num).toDouble(); - await fc.tapAt(x, y); - return {"success": true, "action": "tap_at", "x": x, "y": y}; + final tapped = await fc.tapAt(x, y); + return { + "success": true, + "action": "tap_at", + "x": x, + "y": y, + if (tapped['warning'] != null) "warning": tapped['warning'], + }; case 'long_press_at': if (client is BridgeDriver) { diff --git a/lib/src/cli/tool_handlers/bf_inspection.dart b/lib/src/cli/tool_handlers/bf_inspection.dart index b5833f0d..5dcbc37e 100644 --- a/lib/src/cli/tool_handlers/bf_inspection.dart +++ b/lib/src/cli/tool_handlers/bf_inspection.dart @@ -203,7 +203,13 @@ extension _BfInspection on FlutterMcpServer { "error": "key or element parameter required" }; } - return await fc.getWidgetProperties(wpKey); + final props = await fc.getWidgetProperties(wpKey); + return props ?? + { + "success": false, + "error": "No mounted widget with key '$wpKey' " + "(is it on the current screen / built?)", + }; case 'get_text_content': if (client is BridgeDriver) { final text = await client.getText(); diff --git a/lib/src/cli/tool_handlers/bf_interaction.dart b/lib/src/cli/tool_handlers/bf_interaction.dart index d83d9869..3b756edc 100644 --- a/lib/src/cli/tool_handlers/bf_interaction.dart +++ b/lib/src/cli/tool_handlers/bf_interaction.dart @@ -22,12 +22,13 @@ extension _BfInteraction on FlutterMcpServer { }; } final fc = _asFlutterClient(client!, 'tap (coordinates)'); - await fc.tapAt(x.toDouble(), y.toDouble()); + final tapped = await fc.tapAt(x.toDouble(), y.toDouble()); return { "success": true, "method": "coordinates", "message": "Tapped at ($x, $y)", "position": {"x": x, "y": y}, + if (tapped['warning'] != null) "warning": tapped['warning'], }; } @@ -53,6 +54,7 @@ extension _BfInteraction on FlutterMcpServer { "method": args['key'] != null ? "key" : "text", "message": "Tapped", if (result['position'] != null) "position": result['position'], + if (result['warning'] != null) "warning": result['warning'], }; case 'fill': @@ -136,6 +138,19 @@ extension _BfInteraction on FlutterMcpServer { return success ? "Double tapped" : "Double tap failed"; case 'swipe': final distance = (args['distance'] ?? 300).toDouble(); + if (client is FlutterSkillClient) { + final swiped = await client.swipeWithDetails( + direction: args['direction'], + distance: distance, + key: args['key']); + return { + "success": swiped['success'] == true, + "message": swiped['success'] == true + ? "Swiped ${args['direction']}" + : "Swipe failed", + if (swiped['warning'] != null) "warning": swiped['warning'], + }; + } final success = await client!.swipe( direction: args['direction'], distance: distance, key: args['key']); return success ? "Swiped ${args['direction']}" : "Swipe failed"; @@ -156,20 +171,31 @@ extension _BfInteraction on FlutterMcpServer { return result['success'] == true ? "Dragged" : "Drag failed"; } final fc = _asFlutterClient(client!, 'drag'); - // Support coordinate-based drag via swipeCoordinates for FlutterClient - if (args['start_x'] != null) { + // Clients may send numbers as strings; accept both. + double? num_(Object? v) => + v is num ? v.toDouble() : double.tryParse(v?.toString() ?? ''); + final holdMs = num_(args['hold_ms'])?.toInt() ?? 0; + // Coordinate-based drag shares the swipe engine (with hold) + if (num_(args['start_x']) != null) { final result = await fc.swipeCoordinates( - (args['start_x'] as num).toDouble(), - (args['start_y'] as num).toDouble(), - (args['end_x'] as num).toDouble(), - (args['end_y'] as num).toDouble(), + num_(args['start_x'])!, + num_(args['start_y']) ?? 0, + num_(args['end_x']) ?? 0, + num_(args['end_y']) ?? 0, + duration: num_(args['duration_ms'])?.toInt() ?? 400, + holdMs: holdMs, ); return result; } - final success = await fc.drag( + final dragged = await fc.dragWithDetails( fromKey: args['from_key'] as String? ?? '', - toKey: args['to_key'] as String? ?? ''); - return success ? "Dragged" : "Drag failed"; + toKey: args['to_key'] as String? ?? '', + holdMs: holdMs); + return { + "success": dragged['success'] == true, + "message": dragged['success'] == true ? "Dragged" : "Drag failed", + if (dragged['warning'] != null) "warning": dragged['warning'], + }; // State & Validation case 'get_text_value': @@ -185,7 +211,42 @@ extension _BfInteraction on FlutterMcpServer { .callMethod('get_checkbox_state', {'key': args['key']}); } final fc = _asFlutterClient(client!, 'get_checkbox_state'); - return await fc.getCheckboxState(args['key']); + final checked = await fc.getCheckboxState(args['key']); + return checked == null + ? {"success": false, "error": "No Checkbox/Switch with that key"} + : {"success": true, "checked": checked}; + case 'set_checkbox': + if (client is BridgeDriver) { + return await client.callMethod( + 'set_checkbox', {'key': args['key'], 'checked': args['checked']}); + } + return await _asFlutterClient(client!, 'set_checkbox') + .setCheckbox(args['key'] as String, args['checked'] == true); + case 'type_text': + if (client is BridgeDriver) { + return await client.callMethod('type_text', {'text': args['text']}); + } + return await _asFlutterClient(client!, 'type_text') + .typeText(args['text'] as String? ?? ''); + case 'focus': + if (client is BridgeDriver) { + return await client.callMethod('focus', {'key': args['key']}); + } + return await _asFlutterClient(client!, 'focus') + .focus(args['key'] as String? ?? ''); + case 'blur': + if (client is BridgeDriver) { + return await client.callMethod('blur', {'key': args['key']}); + } + return await _asFlutterClient(client!, 'blur') + .blur(key: args['key'] as String?); + case 'hover': + if (client is BridgeDriver) { + return await client.callMethod('hover', + {'key': args['key'], 'text': args['text'], 'ref': args['ref']}); + } + return await _asFlutterClient(client!, 'hover') + .hover(key: args['key'] as String?, text: args['text'] as String?); case 'get_slider_value': if (client is BridgeDriver) { return await client @@ -231,7 +292,9 @@ extension _BfInteraction on FlutterMcpServer { } // Flutter VM Service — use the registered pressKey extension if (client is FlutterSkillClient) { - final result = await client.pressKey(key); + final modifiers = + (args['modifiers'] as List?)?.map((m) => m.toString()).toList(); + final result = await client.pressKey(key, modifiers: modifiers); if (result['success'] == true) { return {"success": true, "message": "Key pressed: $key"}; } diff --git a/lib/src/cli/tool_handlers/bf_navigation.dart b/lib/src/cli/tool_handlers/bf_navigation.dart index c60bc40c..b0df80c2 100644 --- a/lib/src/cli/tool_handlers/bf_navigation.dart +++ b/lib/src/cli/tool_handlers/bf_navigation.dart @@ -10,7 +10,8 @@ extension _BfNavigation on FlutterMcpServer { return {"route": route}; } final fc = _asFlutterClient(client!, 'get_current_route'); - return await fc.getCurrentRoute(); + // Always a map: a bare null would read as "tool not handled". + return await fc.getCurrentRouteInfo(); case 'go_back': final count = (args['count'] as num?)?.toInt() ?? 1; var steps = 0; diff --git a/lib/src/cli/tool_handlers/bf_screenshot.dart b/lib/src/cli/tool_handlers/bf_screenshot.dart index 2016310b..f3bf17fc 100644 --- a/lib/src/cli/tool_handlers/bf_screenshot.dart +++ b/lib/src/cli/tool_handlers/bf_screenshot.dart @@ -1,6 +1,26 @@ part of '../server.dart'; extension _BfScreenshot on FlutterMcpServer { + /// Image vs logical geometry so an agent can convert pixels it sees in the + /// PNG back to the logical coordinates that tap_at/swipe/drag expect. + Map _screenshotGeometry(Map info) { + final imageWidth = info['imageWidth'] as num?; + final logicalWidth = info['logicalWidth'] as num?; + if (imageWidth == null || logicalWidth == null || logicalWidth == 0) { + return const {}; + } + final scale = imageWidth / logicalWidth; + return { + "image_size": {"width": imageWidth, "height": info['imageHeight']}, + "logical_size": {"width": logicalWidth, "height": info['logicalHeight']}, + "device_pixel_ratio": info['devicePixelRatio'], + "scale": scale, + "coordinates_hint": "Tools take LOGICAL coordinates. To tap what you " + "see at image pixel (px, py): x = px / scale, y = py / scale " + "(scale = ${scale.toStringAsFixed(3)}). Prefer inspect() centers.", + }; + } + Future _handleScreenshotTool( String name, Map args, AppDriver? client) async { switch (name) { @@ -11,8 +31,17 @@ extension _BfScreenshot on FlutterMcpServer { final saveToFile = args['save_to_file'] ?? true; // Save to file by default to avoid token overflow - var imageBase64 = - await client!.takeScreenshot(quality: quality, maxWidth: maxWidth); + Map geometry = const {}; + String? imageBase64; + if (client is FlutterSkillClient) { + final info = await client.takeScreenshotWithInfo( + quality: quality, maxWidth: maxWidth); + imageBase64 = info['image'] as String?; + geometry = _screenshotGeometry(info); + } else { + imageBase64 = await client! + .takeScreenshot(quality: quality, maxWidth: maxWidth); + } // Fallback to native screenshot if bridge returns null // (e.g. React Native returns _needs_native flag) @@ -56,6 +85,7 @@ extension _BfScreenshot on FlutterMcpServer { "quality": quality, "max_width": maxWidth, "format": "png", + ...geometry, "message": "Screenshot saved to ${file.path}" }; } else { @@ -64,6 +94,7 @@ extension _BfScreenshot on FlutterMcpServer { "image": imageBase64, "quality": quality, "max_width": maxWidth, + ...geometry, "warning": "Returning base64 data. Consider using save_to_file=true for large images." }; diff --git a/lib/src/cli/tool_handlers/bridge_flutter_handlers.dart b/lib/src/cli/tool_handlers/bridge_flutter_handlers.dart index 9ef68abc..400a231e 100644 --- a/lib/src/cli/tool_handlers/bridge_flutter_handlers.dart +++ b/lib/src/cli/tool_handlers/bridge_flutter_handlers.dart @@ -29,6 +29,7 @@ extension _BridgeFlutterHandlers on FlutterMcpServer { return await (_client as BridgeDriver).callTool(toolName, toolParams); } - throw Exception("Unknown tool: $name"); + throw Exception("Tool '$name' is not implemented for the " + "${client.runtimeType} driver (or returned no result)"); } } diff --git a/lib/src/cli/tool_handlers/dev_tool_handlers.dart b/lib/src/cli/tool_handlers/dev_tool_handlers.dart index d493422c..f4d01d62 100644 --- a/lib/src/cli/tool_handlers/dev_tool_handlers.dart +++ b/lib/src/cli/tool_handlers/dev_tool_handlers.dart @@ -17,6 +17,7 @@ extension _DevToolHandlers on FlutterMcpServer { if (name == 'hot_reload') { final client = _getClient(args); _requireConnection(client); + if (client is FlutterSkillClient) return await client.hotReload(); await client!.hotReload(); return "Hot reload triggered"; } @@ -25,8 +26,7 @@ extension _DevToolHandlers on FlutterMcpServer { final client = _getClient(args); _requireConnection(client); final fc = _asFlutterClient(client!, 'hot_restart'); - await fc.hotRestart(); - return "Hot restart triggered"; + return await fc.hotRestart(); } if (name == 'enable_test_indicators') { diff --git a/lib/src/cli/tool_handlers/flutter_helpers.dart b/lib/src/cli/tool_handlers/flutter_helpers.dart index 8bdea5e4..b112d032 100644 --- a/lib/src/cli/tool_handlers/flutter_helpers.dart +++ b/lib/src/cli/tool_handlers/flutter_helpers.dart @@ -250,6 +250,9 @@ extension _FlutterHelpers on FlutterMcpServer { ); if (found) { + // Being built (cache extent) is not being visible: bring it into + // the viewport before reporting success. + await client.scrollTo(key: key, text: text); return { "success": true, "found": true, @@ -257,14 +260,21 @@ extension _FlutterHelpers on FlutterMcpServer { }; } - // Scroll + // `direction` is the scroll direction ("down" reveals content further + // down); a swipe is the finger's motion, which is the opposite way. + const fingerFor = { + 'down': 'up', + 'up': 'down', + 'right': 'left', + 'left': 'right', + }; await client.swipe( - direction: direction, + direction: fingerFor[direction] ?? direction, distance: 300, key: scrollableKey, ); - // Wait for scroll animation + // Let the fling settle before looking again await Future.delayed(const Duration(milliseconds: 300)); } diff --git a/lib/src/cli/tool_handlers/tool_definitions.dart b/lib/src/cli/tool_handlers/tool_definitions.dart index c4ede0e4..da1d8f4a 100644 --- a/lib/src/cli/tool_handlers/tool_definitions.dart +++ b/lib/src/cli/tool_handlers/tool_definitions.dart @@ -1724,14 +1724,27 @@ Option 3: Tap a TextField first, then enter_text(text: "value") without key/ref }, { "name": "drag", - "description": "Drag from one element to another", + "description": "Drag from one element (or point) to another. " + "Set hold_ms (e.g. 400) for LongPressDraggable / long-press-to-" + "select lists so the drag is recognised before the pointer moves.", "inputSchema": { "type": "object", "properties": { "from_key": {"type": "string", "description": "Source element key"}, "to_key": {"type": "string", "description": "Target element key"}, + "start_x": {"type": "number", "description": "Start X (logical)"}, + "start_y": {"type": "number", "description": "Start Y (logical)"}, + "end_x": {"type": "number", "description": "End X (logical)"}, + "end_y": {"type": "number", "description": "End Y (logical)"}, + "hold_ms": { + "type": "integer", + "description": "Press-and-hold before moving, in ms (default 0)" + }, + "duration_ms": { + "type": "integer", + "description": "Travel time in ms (default 400)" + }, }, - "required": ["from_key", "to_key"], }, }, diff --git a/lib/src/drivers/flutter_driver.dart b/lib/src/drivers/flutter_driver.dart index a3f2a9bb..59bc174c 100644 --- a/lib/src/drivers/flutter_driver.dart +++ b/lib/src/drivers/flutter_driver.dart @@ -1,3 +1,5 @@ +import 'dart:async'; +import 'dart:io'; import 'package:vm_service/vm_service.dart'; import 'package:vm_service/vm_service_io.dart'; import '../discovery/unified_discovery.dart'; @@ -9,6 +11,14 @@ class FlutterSkillClient implements AppDriver { String? _isolateId; bool _reconnecting = false; + /// Services registered on the VM by external clients (notably the Flutter + /// tool, alias "Flutter Tools"), keyed by service name -> callable method + /// (e.g. `reloadSources` -> `s1.reloadSources`). Populated from the + /// `Service` stream, which replays existing registrations on subscribe. + final Map _registeredServices = {}; + final Map> _serviceWaiters = {}; + StreamSubscription? _serviceEvents; + FlutterSkillClient(this.wsUri); /// Get the VM Service URI this client is connected to @@ -38,7 +48,8 @@ Solution: URI: $wsUri'''); } - _isolateId = isolates.first.id!; + _isolateId = _pickMainIsolate(isolates); + await _watchRegisteredServices(); } catch (e) { // Clean up partially initialized service try { @@ -66,6 +77,10 @@ Error details: $e'''); } Future disconnect() async { + await _serviceEvents?.cancel(); + _serviceEvents = null; + _registeredServices.clear(); + _serviceWaiters.clear(); try { await _service?.dispose(); } catch (_) { @@ -103,7 +118,8 @@ Error details: $e'''); final vm = await _service!.getVM(); final isolates = vm.isolates; if (isolates != null && isolates.isNotEmpty) { - _isolateId = isolates.first.id!; + _isolateId = _pickMainIsolate(isolates); + await _watchRegisteredServices(); print('DEBUG: Reconnected to VM Service successfully'); return true; } @@ -136,8 +152,18 @@ URI: $wsUri'''); ); return response.json ?? {}; } catch (e) { - // Detect SDK-not-integrated errors and surface a clear setup message. final msg = e.toString(); + // The cached isolate no longer exists (e.g. after a hot restart): + // re-resolve the main isolate once and retry. + if (_isStaleIsolateError(e) && await _refreshIsolate()) { + final response = await _service!.callServiceExtension( + method, + isolateId: _isolateId!, + args: args, + ); + return response.json ?? {}; + } + // Detect SDK-not-integrated errors and surface a clear setup message. if (msg.contains('Method not found') || msg.contains('Service extension not found') || msg.contains('ext.flutter.flutter_skill') && msg.contains('-32601')) { @@ -261,20 +287,36 @@ Auto-setup: run diagnose_project() to fix automatically. Future swipe( {required String direction, double distance = 300, String? key}) async { - final result = await _call('ext.flutter.flutter_skill.swipe', { + final result = await swipeWithDetails( + direction: direction, distance: distance, key: key); + return result['success'] == true; + } + + /// Swipe; the map carries `success` and, when frames stalled, `warning`. + Future> swipeWithDetails( + {required String direction, double distance = 300, String? key}) async { + return await _call('ext.flutter.flutter_skill.swipe', { 'direction': direction, 'distance': distance.toString(), if (key != null) 'key': key, }); + } + + Future drag( + {required String fromKey, required String toKey, int holdMs = 0}) async { + final result = + await dragWithDetails(fromKey: fromKey, toKey: toKey, holdMs: holdMs); return result['success'] == true; } - Future drag({required String fromKey, required String toKey}) async { - final result = await _call('ext.flutter.flutter_skill.drag', { + /// Drag; the map carries `success` and, when frames stalled, `warning`. + Future> dragWithDetails( + {required String fromKey, required String toKey, int holdMs = 0}) async { + return await _call('ext.flutter.flutter_skill.drag', { 'fromKey': fromKey, 'toKey': toKey, + 'hold': holdMs.toString(), }); - return result['success'] == true; } Future doubleTap({String? key, String? text}) async { @@ -343,11 +385,18 @@ Auto-setup: run diagnose_project() to fix automatically. // ==================== SCREENSHOT ==================== Future takeScreenshot({double quality = 1.0, int? maxWidth}) async { - final result = await _call('ext.flutter.flutter_skill.screenshot', { + return (await takeScreenshotWithInfo( + quality: quality, maxWidth: maxWidth))['image'] as String?; + } + + /// Screenshot plus its geometry: `image` (base64 PNG), `imageWidth`, + /// `imageHeight`, `logicalWidth`, `logicalHeight`, `devicePixelRatio`. + Future> takeScreenshotWithInfo( + {double quality = 1.0, int? maxWidth}) async { + return await _call('ext.flutter.flutter_skill.screenshot', { 'quality': quality.toString(), if (maxWidth != null) 'maxWidth': maxWidth.toString(), }); - return result['image']; } Future takeRegionScreenshot( @@ -371,10 +420,31 @@ Auto-setup: run diagnose_project() to fix automatically. // ==================== NAVIGATION ==================== Future getCurrentRoute() async { - final result = await _call('ext.flutter.flutter_skill.getCurrentRoute'); - return result['route']; + return (await getCurrentRouteInfo())['route'] as String?; } + /// `route` (name of the topmost current route) and `routes` (full stack). + Future> getCurrentRouteInfo() async { + return await _call('ext.flutter.flutter_skill.getCurrentRoute'); + } + + Future> typeText(String text) async => + await _call('ext.flutter.flutter_skill.typeText', {'text': text}); + + Future> focus(String key) async => + await _call('ext.flutter.flutter_skill.focus', {'key': key}); + + Future> blur({String? key}) async => await _call( + 'ext.flutter.flutter_skill.blur', {if (key != null) 'key': key}); + + Future> setCheckbox(String key, bool checked) async => + await _call('ext.flutter.flutter_skill.setCheckbox', + {'key': key, 'checked': checked.toString()}); + + Future> hover({String? key, String? text}) async => + await _call('ext.flutter.flutter_skill.hover', + {if (key != null) 'key': key, if (text != null) 'text': text}); + Future goBack() async { final result = await _call('ext.flutter.flutter_skill.goBack'); return result['success'] == true; @@ -429,6 +499,7 @@ Auto-setup: run diagnose_project() to fix automatically. double endX, double endY, { int duration = 300, + int holdMs = 0, }) async { return await _call('ext.flutter.flutter_skill.swipeCoordinates', { 'startX': startX.toString(), @@ -436,6 +507,7 @@ Auto-setup: run diagnose_project() to fix automatically. 'endX': endX.toString(), 'endY': endY.toString(), 'duration': duration.toString(), + 'hold': holdMs.toString(), }); } @@ -540,33 +612,220 @@ Auto-setup: run diagnose_project() to fix automatically. // ==================== EXISTING HELPERS ==================== - Future hotReload() async { + // ==================== HOT RELOAD / RESTART ==================== + // + // A Flutter app cannot recompile Dart source on its own: the engine ships + // no kernel compiler, so the VM's built-in `reloadSources` RPC fails with + // "Error while starting Kernel isolate task" (and hot restart has no VM + // RPC at all). The `flutter run` tool that launched the app owns the + // incremental compiler and registers `reloadSources` and `hotRestart` as + // VM *services* (alias "Flutter Tools"); invoking those — exactly what + // DevTools and the IDEs do — recompiles, reloads and reassembles. + + static const _reloadService = 'reloadSources'; + static const _restartService = 'hotRestart'; + + /// Hot reload through the Flutter tool. Returns a short human report. + Future hotReload() async { + _requireService(); + final elapsed = await _invokeFlutterTools(_reloadService); + if (elapsed != null) { + await _settleCompileClock(); + return 'Hot reload performed by Flutter Tools (${elapsed}ms)'; + } + // No Flutter tool attached: the raw VM RPC is the only option. It only + // succeeds when the VM can compile Dart itself (plain `dart run`), so + // check the report instead of silently claiming success. + final report = + await _callWithReconnect(() => _service!.reloadSources(_isolateId!)); + if (report.success == true) return 'Hot reload performed by the VM'; + // `notices` is not modelled by package:vm_service; read the raw JSON. + final notices = report.json?['notices'] as List? ?? const []; + final reasons = notices + .map((n) => (n as Map?)?['message']) + .whereType() + .join('; '); + throw Exception('''❌ Hot reload is not possible on this VM Service. + +The VM cannot recompile Dart sources ($reasons) and no `flutter run` tool +is attached to do it (no `reloadSources` service registered). + +Fix: launch the app with `flutter run` (debug mode) and connect to the VM +Service URI it prints — the tool then registers the reload service that +this command drives.'''); + } + + /// Hot restart through the Flutter tool, then re-bind to the new isolate. + Future hotRestart() async { + _requireService(); + final elapsed = await _invokeFlutterTools(_restartService); + if (elapsed == null) { + throw UnsupportedError( + '''❌ Hot restart is not possible on this VM Service. + +Hot restart is performed by the `flutter run` tool, which registers a +`hotRestart` service on the VM. None is registered here — the app was not +launched by `flutter run` in debug mode (or the tool has exited). + +Fix: launch the app with `flutter run` and connect to the printed URI.'''); + } + // The main isolate is recreated by the restart; rebind before returning + // so the next call does not hit a collected isolate. + if (!await _refreshIsolate()) { + throw Exception('Hot restart completed but the new main isolate did ' + 'not appear; reconnect with connect_app().'); + } + final ready = await _awaitFirstFrame(); + await _settleCompileClock(); + return 'Hot restart performed by Flutter Tools (${elapsed}ms' + '${ready ? '' : ', first frame not yet sent'})'; + } + + /// Invokes a service registered by the Flutter tool on the main isolate. + /// Returns the elapsed milliseconds, or null when the tool has not + /// registered [service] (no `flutter run` attached to this VM). + Future _invokeFlutterTools(String service) async { + final method = await _flutterToolsMethod(service); + if (method == null) return null; + final sw = Stopwatch()..start(); + await _callWithReconnect( + () => _service!.callMethod(method, isolateId: _isolateId)); + return sw.elapsedMilliseconds; + } + + /// Waits until the restarted app has built and sent its first frame, so + /// the widget tree is populated for the caller's next inspection. + /// (`didSendFirstFrameEvent` rather than the *Rasterized* variant: the + /// latter never flips on some desktop embedders, e.g. Windows.) + Future _awaitFirstFrame() async { + const ext = 'ext.flutter.didSendFirstFrameEvent'; + final deadline = DateTime.now().add(const Duration(seconds: 10)); + while (DateTime.now().isBefore(deadline)) { + try { + final response = + await _service!.callServiceExtension(ext, isolateId: _isolateId); + final enabled = response.json?['enabled']; + if (enabled == true || enabled == 'true') return true; + } catch (_) { + // extension not registered yet — the isolate is still booting + } + await Future.delayed(const Duration(milliseconds: 100)); + } + return false; + } + + /// The Flutter tool detects edited files by comparing their mtime with the + /// wall-clock time of its last compile. On Windows, Dart reports mtimes + /// truncated to whole seconds, so a file saved within the same second as + /// that compile is silently treated as unchanged by the next reload or + /// restart ("0 updated files"). Agents edit and reload back-to-back, so + /// return only once the clock has rolled past the second in which the + /// compile happened; every later edit is then guaranteed to be picked up. + Future _settleCompileClock() async { + if (!Platform.isWindows) return; + final now = DateTime.now(); + final nextSecond = DateTime( + now.year, now.month, now.day, now.hour, now.minute, now.second + 1); + await Future.delayed(nextSecond.difference(now)); + } + + void _requireService() { if (_service == null || _isolateId == null) { throw Exception('Not connected'); } + } + + Future _callWithReconnect(Future Function() call) async { try { - await _service!.reloadSources(_isolateId!); + return await call(); } catch (e) { if (!_isConnectionError(e)) rethrow; - if (await _reconnect()) { - await _service!.reloadSources(_isolateId!); - return; - } + if (await _reconnect()) return await call(); throw Exception('❌ VM Service connection lost and reconnection failed.\n' 'URI: $wsUri\n' 'Original error: $e'); } } - Future hotRestart() async { - // The VM Service protocol does not expose a hot-restart RPC that can be - // called externally. A real hot restart requires re-running the Dart - // program from scratch, which is only possible by invoking `flutter run` - // again or using the Flutter tool's stdin protocol. - // Throw UnsupportedError so callers can detect this and fall back to - // hotReload or surface a warning to the user. - throw UnsupportedError('hotRestart is not supported via the VM Service. ' - 'Use hotReload for source updates, or restart the app via flutter run.'); + /// Subscribes to the `Service` stream so registered services (and their + /// callable method names) are tracked live. The VM replays existing + /// registrations right after `streamListen`. + Future _watchRegisteredServices() async { + await _serviceEvents?.cancel(); + _registeredServices.clear(); + final service = _service; + if (service == null) return; + _serviceEvents = service.onServiceEvent.listen((event) { + final name = event.service; + if (name == null) return; + if (event.kind == EventKind.kServiceRegistered && event.method != null) { + _registeredServices[name] = event.method!; + _serviceWaiters.remove(name)?.complete(event.method); + } else if (event.kind == EventKind.kServiceUnregistered) { + _registeredServices.remove(name); + } + }); + try { + await service.streamListen(EventStreams.kService); + } on RPCError catch (e) { + // 103 = stream already subscribed; anything else is non-fatal here. + if (e.code != 103) print('DEBUG: Service stream unavailable: $e'); + } + } + + /// Method name (`sN.`) for a service registered by the Flutter + /// tool. Registrations replayed by the VM arrive asynchronously right after + /// subscribing, so wait for the event (bounded) rather than polling. + Future _flutterToolsMethod(String service) async { + if (_serviceEvents == null) await _watchRegisteredServices(); + final known = _registeredServices[service]; + if (known != null) return known; + final waiter = + _serviceWaiters.putIfAbsent(service, () => Completer()); + return waiter.future.timeout(const Duration(seconds: 2), onTimeout: () { + _serviceWaiters.remove(service); + return null; + }); + } + + static String _pickMainIsolate(List isolates) => isolates + .firstWhere((i) => i.name == 'main', orElse: () => isolates.first) + .id!; + + /// True when a call failed because the target isolate no longer exists + /// (the VM answers with a `Collected`/`Expired` sentinel). + bool _isStaleIsolateError(Object e) => + e is SentinelException || + // Some transports surface the sentinel only as a message. + e.toString().contains('Sentinel kind: Collected'); + + /// Re-binds to the recreated main isolate (hot restart, or a Sentinel from + /// a call). The previous isolate may still be listed while it tears down, + /// so only an isolate with a *different* id that is already runnable is + /// accepted; polls briefly because the new one may not be up yet. + Future _refreshIsolate() async { + final service = _service; + if (service == null) return false; + final previous = _isolateId; + final deadline = DateTime.now().add(const Duration(seconds: 5)); + while (DateTime.now().isBefore(deadline)) { + try { + final candidates = ((await service.getVM()).isolates ?? const []) + .where((i) => i.id != previous) + .toList(); + if (candidates.isNotEmpty) { + final id = _pickMainIsolate(candidates); + if ((await service.getIsolate(id)).runnable == true) { + _isolateId = id; + return true; + } + } + } catch (_) { + // transient during restart + } + await Future.delayed(const Duration(milliseconds: 100)); + } + return false; } Future> getLayoutTree() async { diff --git a/lib/src/engine/tool_registry.dart b/lib/src/engine/tool_registry.dart index 60c4bdd7..0ef86071 100644 --- a/lib/src/engine/tool_registry.dart +++ b/lib/src/engine/tool_registry.dart @@ -1844,14 +1844,28 @@ Option 3: Tap a TextField first, then enter_text(text: "value") without key/ref }, { "name": "drag", - "description": "Drag from one element to another", + "description": "Drag from one element (from_key/to_key) or between " + "logical coordinates (start_x/start_y -> end_x/end_y). Set " + "hold_ms (e.g. 400) for LongPressDraggable / long-press-to-select " + "lists so the drag is recognised before the pointer moves.", "inputSchema": { "type": "object", "properties": { "from_key": {"type": "string", "description": "Source element key"}, - "to_key": {"type": "string", "description": "Target element key"} - }, - "required": ["from_key", "to_key"] + "to_key": {"type": "string", "description": "Target element key"}, + "start_x": {"type": "number", "description": "Start X (logical)"}, + "start_y": {"type": "number", "description": "Start Y (logical)"}, + "end_x": {"type": "number", "description": "End X (logical)"}, + "end_y": {"type": "number", "description": "End Y (logical)"}, + "hold_ms": { + "type": "integer", + "description": "Press-and-hold before moving, in ms (default 0)" + }, + "duration_ms": { + "type": "integer", + "description": "Travel time in ms (default 400)" + } + } } },