From a8db1b2a01bb0f4c7cfcfbfdf7810e196b69770f Mon Sep 17 00:00:00 2001 From: cornebidouil Date: Mon, 17 Aug 2026 10:01:31 +0200 Subject: [PATCH 1/4] fix(hot-reload): drive Flutter Tools' reloadSources/hotRestart VM services hot_reload called the VM's raw reloadSources RPC. A Flutter app embeds no kernel compiler, so the VM answers success:false ("Error while starting Kernel isolate task"); the report was ignored and the tool reported success while nothing recompiled. hot_restart was the same call (or threw). The flutter run process registers reloadSources/hotRestart as VM services (alias "Flutter Tools") - what DevTools and IDEs call. The driver now tracks registrations from the Service stream and invokes sN.reloadSources / sN.hotRestart; without an attached tool it checks the ReloadReport and surfaces a clear error instead of a fake success. Also: - re-bind to the new main isolate after a restart (old id -> Sentinel Collected on every later call) and self-heal on stale isolates - wait for ext.flutter.didSendFirstFrameEvent so the tree is populated - on Windows, hold until the wall-clock second rolls over: Dart truncates mtimes to seconds there, so the tool ignores edits made in the same second as its last compile ("0 updated files") - report what happened: "Hot reload performed by Flutter Tools (330ms)" --- .../cli/tool_handlers/dev_tool_handlers.dart | 4 +- lib/src/drivers/flutter_driver.dart | 249 ++++++++++++++++-- 2 files changed, 233 insertions(+), 20 deletions(-) diff --git a/lib/src/cli/tool_handlers/dev_tool_handlers.dart b/lib/src/cli/tool_handlers/dev_tool_handlers.dart index d493422c..f4d01d62 100644 --- a/lib/src/cli/tool_handlers/dev_tool_handlers.dart +++ b/lib/src/cli/tool_handlers/dev_tool_handlers.dart @@ -17,6 +17,7 @@ extension _DevToolHandlers on FlutterMcpServer { if (name == 'hot_reload') { final client = _getClient(args); _requireConnection(client); + if (client is FlutterSkillClient) return await client.hotReload(); await client!.hotReload(); return "Hot reload triggered"; } @@ -25,8 +26,7 @@ extension _DevToolHandlers on FlutterMcpServer { final client = _getClient(args); _requireConnection(client); final fc = _asFlutterClient(client!, 'hot_restart'); - await fc.hotRestart(); - return "Hot restart triggered"; + return await fc.hotRestart(); } if (name == 'enable_test_indicators') { diff --git a/lib/src/drivers/flutter_driver.dart b/lib/src/drivers/flutter_driver.dart index a3f2a9bb..7a48fe75 100644 --- a/lib/src/drivers/flutter_driver.dart +++ b/lib/src/drivers/flutter_driver.dart @@ -1,3 +1,5 @@ +import 'dart:async'; +import 'dart:io'; import 'package:vm_service/vm_service.dart'; import 'package:vm_service/vm_service_io.dart'; import '../discovery/unified_discovery.dart'; @@ -9,6 +11,14 @@ class FlutterSkillClient implements AppDriver { String? _isolateId; bool _reconnecting = false; + /// Services registered on the VM by external clients (notably the Flutter + /// tool, alias "Flutter Tools"), keyed by service name -> callable method + /// (e.g. `reloadSources` -> `s1.reloadSources`). Populated from the + /// `Service` stream, which replays existing registrations on subscribe. + final Map _registeredServices = {}; + final Map> _serviceWaiters = {}; + StreamSubscription? _serviceEvents; + FlutterSkillClient(this.wsUri); /// Get the VM Service URI this client is connected to @@ -38,7 +48,8 @@ Solution: URI: $wsUri'''); } - _isolateId = isolates.first.id!; + _isolateId = _pickMainIsolate(isolates); + await _watchRegisteredServices(); } catch (e) { // Clean up partially initialized service try { @@ -66,6 +77,10 @@ Error details: $e'''); } Future disconnect() async { + await _serviceEvents?.cancel(); + _serviceEvents = null; + _registeredServices.clear(); + _serviceWaiters.clear(); try { await _service?.dispose(); } catch (_) { @@ -103,7 +118,8 @@ Error details: $e'''); final vm = await _service!.getVM(); final isolates = vm.isolates; if (isolates != null && isolates.isNotEmpty) { - _isolateId = isolates.first.id!; + _isolateId = _pickMainIsolate(isolates); + await _watchRegisteredServices(); print('DEBUG: Reconnected to VM Service successfully'); return true; } @@ -136,8 +152,18 @@ URI: $wsUri'''); ); return response.json ?? {}; } catch (e) { - // Detect SDK-not-integrated errors and surface a clear setup message. final msg = e.toString(); + // The cached isolate no longer exists (e.g. after a hot restart): + // re-resolve the main isolate once and retry. + if (_isStaleIsolateError(e) && await _refreshIsolate()) { + final response = await _service!.callServiceExtension( + method, + isolateId: _isolateId!, + args: args, + ); + return response.json ?? {}; + } + // Detect SDK-not-integrated errors and surface a clear setup message. if (msg.contains('Method not found') || msg.contains('Service extension not found') || msg.contains('ext.flutter.flutter_skill') && msg.contains('-32601')) { @@ -540,33 +566,220 @@ Auto-setup: run diagnose_project() to fix automatically. // ==================== EXISTING HELPERS ==================== - Future hotReload() async { + // ==================== HOT RELOAD / RESTART ==================== + // + // A Flutter app cannot recompile Dart source on its own: the engine ships + // no kernel compiler, so the VM's built-in `reloadSources` RPC fails with + // "Error while starting Kernel isolate task" (and hot restart has no VM + // RPC at all). The `flutter run` tool that launched the app owns the + // incremental compiler and registers `reloadSources` and `hotRestart` as + // VM *services* (alias "Flutter Tools"); invoking those — exactly what + // DevTools and the IDEs do — recompiles, reloads and reassembles. + + static const _reloadService = 'reloadSources'; + static const _restartService = 'hotRestart'; + + /// Hot reload through the Flutter tool. Returns a short human report. + Future hotReload() async { + _requireService(); + final elapsed = await _invokeFlutterTools(_reloadService); + if (elapsed != null) { + await _settleCompileClock(); + return 'Hot reload performed by Flutter Tools (${elapsed}ms)'; + } + // No Flutter tool attached: the raw VM RPC is the only option. It only + // succeeds when the VM can compile Dart itself (plain `dart run`), so + // check the report instead of silently claiming success. + final report = + await _callWithReconnect(() => _service!.reloadSources(_isolateId!)); + if (report.success == true) return 'Hot reload performed by the VM'; + // `notices` is not modelled by package:vm_service; read the raw JSON. + final notices = report.json?['notices'] as List? ?? const []; + final reasons = notices + .map((n) => (n as Map?)?['message']) + .whereType() + .join('; '); + throw Exception('''❌ Hot reload is not possible on this VM Service. + +The VM cannot recompile Dart sources ($reasons) and no `flutter run` tool +is attached to do it (no `reloadSources` service registered). + +Fix: launch the app with `flutter run` (debug mode) and connect to the VM +Service URI it prints — the tool then registers the reload service that +this command drives.'''); + } + + /// Hot restart through the Flutter tool, then re-bind to the new isolate. + Future hotRestart() async { + _requireService(); + final elapsed = await _invokeFlutterTools(_restartService); + if (elapsed == null) { + throw UnsupportedError( + '''❌ Hot restart is not possible on this VM Service. + +Hot restart is performed by the `flutter run` tool, which registers a +`hotRestart` service on the VM. None is registered here — the app was not +launched by `flutter run` in debug mode (or the tool has exited). + +Fix: launch the app with `flutter run` and connect to the printed URI.'''); + } + // The main isolate is recreated by the restart; rebind before returning + // so the next call does not hit a collected isolate. + if (!await _refreshIsolate()) { + throw Exception('Hot restart completed but the new main isolate did ' + 'not appear; reconnect with connect_app().'); + } + final ready = await _awaitFirstFrame(); + await _settleCompileClock(); + return 'Hot restart performed by Flutter Tools (${elapsed}ms' + '${ready ? '' : ', first frame not yet sent'})'; + } + + /// Invokes a service registered by the Flutter tool on the main isolate. + /// Returns the elapsed milliseconds, or null when the tool has not + /// registered [service] (no `flutter run` attached to this VM). + Future _invokeFlutterTools(String service) async { + final method = await _flutterToolsMethod(service); + if (method == null) return null; + final sw = Stopwatch()..start(); + await _callWithReconnect( + () => _service!.callMethod(method, isolateId: _isolateId)); + return sw.elapsedMilliseconds; + } + + /// Waits until the restarted app has built and sent its first frame, so + /// the widget tree is populated for the caller's next inspection. + /// (`didSendFirstFrameEvent` rather than the *Rasterized* variant: the + /// latter never flips on some desktop embedders, e.g. Windows.) + Future _awaitFirstFrame() async { + const ext = 'ext.flutter.didSendFirstFrameEvent'; + final deadline = DateTime.now().add(const Duration(seconds: 10)); + while (DateTime.now().isBefore(deadline)) { + try { + final response = + await _service!.callServiceExtension(ext, isolateId: _isolateId); + final enabled = response.json?['enabled']; + if (enabled == true || enabled == 'true') return true; + } catch (_) { + // extension not registered yet — the isolate is still booting + } + await Future.delayed(const Duration(milliseconds: 100)); + } + return false; + } + + /// The Flutter tool detects edited files by comparing their mtime with the + /// wall-clock time of its last compile. On Windows, Dart reports mtimes + /// truncated to whole seconds, so a file saved within the same second as + /// that compile is silently treated as unchanged by the next reload or + /// restart ("0 updated files"). Agents edit and reload back-to-back, so + /// return only once the clock has rolled past the second in which the + /// compile happened; every later edit is then guaranteed to be picked up. + Future _settleCompileClock() async { + if (!Platform.isWindows) return; + final now = DateTime.now(); + final nextSecond = DateTime( + now.year, now.month, now.day, now.hour, now.minute, now.second + 1); + await Future.delayed(nextSecond.difference(now)); + } + + void _requireService() { if (_service == null || _isolateId == null) { throw Exception('Not connected'); } + } + + Future _callWithReconnect(Future Function() call) async { try { - await _service!.reloadSources(_isolateId!); + return await call(); } catch (e) { if (!_isConnectionError(e)) rethrow; - if (await _reconnect()) { - await _service!.reloadSources(_isolateId!); - return; - } + if (await _reconnect()) return await call(); throw Exception('❌ VM Service connection lost and reconnection failed.\n' 'URI: $wsUri\n' 'Original error: $e'); } } - Future hotRestart() async { - // The VM Service protocol does not expose a hot-restart RPC that can be - // called externally. A real hot restart requires re-running the Dart - // program from scratch, which is only possible by invoking `flutter run` - // again or using the Flutter tool's stdin protocol. - // Throw UnsupportedError so callers can detect this and fall back to - // hotReload or surface a warning to the user. - throw UnsupportedError('hotRestart is not supported via the VM Service. ' - 'Use hotReload for source updates, or restart the app via flutter run.'); + /// Subscribes to the `Service` stream so registered services (and their + /// callable method names) are tracked live. The VM replays existing + /// registrations right after `streamListen`. + Future _watchRegisteredServices() async { + await _serviceEvents?.cancel(); + _registeredServices.clear(); + final service = _service; + if (service == null) return; + _serviceEvents = service.onServiceEvent.listen((event) { + final name = event.service; + if (name == null) return; + if (event.kind == EventKind.kServiceRegistered && event.method != null) { + _registeredServices[name] = event.method!; + _serviceWaiters.remove(name)?.complete(event.method); + } else if (event.kind == EventKind.kServiceUnregistered) { + _registeredServices.remove(name); + } + }); + try { + await service.streamListen(EventStreams.kService); + } on RPCError catch (e) { + // 103 = stream already subscribed; anything else is non-fatal here. + if (e.code != 103) print('DEBUG: Service stream unavailable: $e'); + } + } + + /// Method name (`sN.`) for a service registered by the Flutter + /// tool. Registrations replayed by the VM arrive asynchronously right after + /// subscribing, so wait for the event (bounded) rather than polling. + Future _flutterToolsMethod(String service) async { + if (_serviceEvents == null) await _watchRegisteredServices(); + final known = _registeredServices[service]; + if (known != null) return known; + final waiter = + _serviceWaiters.putIfAbsent(service, () => Completer()); + return waiter.future.timeout(const Duration(seconds: 2), onTimeout: () { + _serviceWaiters.remove(service); + return null; + }); + } + + static String _pickMainIsolate(List isolates) => isolates + .firstWhere((i) => i.name == 'main', orElse: () => isolates.first) + .id!; + + /// True when a call failed because the target isolate no longer exists + /// (the VM answers with a `Collected`/`Expired` sentinel). + bool _isStaleIsolateError(Object e) => + e is SentinelException || + // Some transports surface the sentinel only as a message. + e.toString().contains('Sentinel kind: Collected'); + + /// Re-binds to the recreated main isolate (hot restart, or a Sentinel from + /// a call). The previous isolate may still be listed while it tears down, + /// so only an isolate with a *different* id that is already runnable is + /// accepted; polls briefly because the new one may not be up yet. + Future _refreshIsolate() async { + final service = _service; + if (service == null) return false; + final previous = _isolateId; + final deadline = DateTime.now().add(const Duration(seconds: 5)); + while (DateTime.now().isBefore(deadline)) { + try { + final candidates = ((await service.getVM()).isolates ?? const []) + .where((i) => i.id != previous) + .toList(); + if (candidates.isNotEmpty) { + final id = _pickMainIsolate(candidates); + if ((await service.getIsolate(id)).runnable == true) { + _isolateId = id; + return true; + } + } + } catch (_) { + // transient during restart + } + await Future.delayed(const Duration(milliseconds: 100)); + } + return false; } Future> getLayoutTree() async { From b187307fad0ae68c40d6d065012647ad5cbfd66d Mon Sep 17 00:00:00 2001 From: cornebidouil Date: Mon, 17 Aug 2026 11:51:35 +0200 Subject: [PATCH 2/4] fix(input): reliable synthetic pointer engine; single activation; drag hold; screenshot geometry; hittable flag All emulated input now goes through one engine that stamps every event with a monotonically increasing timeStamp (Flutter's velocity tracker and scroll physics need them - without them a swipe on a PageView/TabBarView snaps back or is left mid-transition), always releases the pointer, and returns only after the frame that reflects the input has rendered (state queries right after an action are no longer stale; fixed sleeps removed). Frame waits are bounded: a minimized/hidden window renders no frames on Windows, which used to hang actions and screenshots for minutes; actions now return promptly and carry a 'warning' explaining the stalled frames. tap(key/text/ref) dispatched the pointer AND invoked the widget callback directly, activating twice (toggles cancelled out). The direct call is now only used when the element is not reachable by hit test at its center. inspect / inspect_interactive report 'hittable' (hit test at the element's center): 'visible' only meant 'has a size' and was true for cards hidden under a bottom bar or scrolled off screen. drag / swipe_coordinates accept hold_ms so LongPressDraggable and long-press-to-select lists recognise the drag before the pointer moves; drag also takes start/end coordinates (numbers or numeric strings). screenshot returns image_size, logical_size, device_pixel_ratio, scale and a hint: tools take logical coordinates, not image pixels. Indicator particle painter clamped its opacity (asserted every frame). --- lib/flutter_skill.dart | 297 +++++++++++------- lib/src/cli/tool_handlers/bf_batch.dart | 10 +- lib/src/cli/tool_handlers/bf_interaction.dart | 46 ++- lib/src/cli/tool_handlers/bf_screenshot.dart | 35 ++- .../cli/tool_handlers/tool_definitions.dart | 17 +- lib/src/drivers/flutter_driver.dart | 37 ++- lib/src/engine/tool_registry.dart | 22 +- 7 files changed, 322 insertions(+), 142 deletions(-) diff --git a/lib/flutter_skill.dart b/lib/flutter_skill.dart index a9b63aa7..c825c50b 100644 --- a/lib/flutter_skill.dart +++ b/lib/flutter_skill.dart @@ -7,6 +7,7 @@ import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; +import 'package:flutter/scheduler.dart'; import 'package:flutter/services.dart'; import 'flutter_skill_semantic_refs.dart'; @@ -409,10 +410,10 @@ class FlutterSkillBinding { final success = await _performSwipe( direction: direction, distance: distance, key: key); return developer.ServiceExtensionResponse.result( - jsonEncode({ + jsonEncode(_withFrameStatus({ 'success': success, 'message': success ? 'Swipe successful' : 'Swipe failed' - }), + })), ); } catch (e, stack) { return _errorResponse(e, stack); @@ -425,12 +426,14 @@ class FlutterSkillBinding { try { final fromKey = parameters['fromKey']; final toKey = parameters['toKey']; - final success = await _performDrag(fromKey: fromKey, toKey: toKey); + final holdMs = int.tryParse(parameters['hold'] ?? '0') ?? 0; + final success = + await _performDrag(fromKey: fromKey, toKey: toKey, holdMs: holdMs); return developer.ServiceExtensionResponse.result( - jsonEncode({ + jsonEncode(_withFrameStatus({ 'success': success, 'message': success ? 'Drag successful' : 'Drag failed' - }), + })), ); } catch (e, stack) { return _errorResponse(e, stack); @@ -561,10 +564,20 @@ class FlutterSkillBinding { try { final quality = double.tryParse(parameters['quality'] ?? '1.0') ?? 1.0; final maxWidth = int.tryParse(parameters['maxWidth'] ?? ''); - final base64Image = + final shot = await _takeScreenshot(quality: quality, maxWidth: maxWidth); - return developer.ServiceExtensionResponse.result( - jsonEncode({'image': base64Image})); + // Logical size + DPR let callers map image pixels back to the + // logical coordinates that tap_at/swipe expect. + final view = WidgetsBinding.instance.platformDispatcher.views.first; + final logical = view.physicalSize / view.devicePixelRatio; + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'image': shot?.base64, + 'imageWidth': shot?.width, + 'imageHeight': shot?.height, + 'logicalWidth': logical.width, + 'logicalHeight': logical.height, + 'devicePixelRatio': view.devicePixelRatio, + })); } catch (e, stack) { return _errorResponse(e, stack); } @@ -879,9 +892,9 @@ class FlutterSkillBinding { final x = double.tryParse(parameters['x'] ?? '0') ?? 0; final y = double.tryParse(parameters['y'] ?? '0') ?? 0; await _performTapAt(x, y); - return developer.ServiceExtensionResponse.result( - jsonEncode({'success': true, 'message': 'Tapped at ($x, $y)'}), - ); + return developer.ServiceExtensionResponse.result(jsonEncode( + _withFrameStatus( + {'success': true, 'message': 'Tapped at ($x, $y)'}))); } catch (e, stack) { return _errorResponse(e, stack); } @@ -912,13 +925,14 @@ class FlutterSkillBinding { final endX = double.tryParse(parameters['endX'] ?? '0') ?? 0; final endY = double.tryParse(parameters['endY'] ?? '0') ?? 0; final duration = int.tryParse(parameters['duration'] ?? '300') ?? 300; + final holdMs = int.tryParse(parameters['hold'] ?? '0') ?? 0; await _performSwipeCoordinates(startX, startY, endX, endY, - duration: duration); + duration: duration, holdMs: holdMs); return developer.ServiceExtensionResponse.result( - jsonEncode({ + jsonEncode(_withFrameStatus({ 'success': true, 'message': 'Swiped from ($startX, $startY) to ($endX, $endY)' - }), + })), ); } catch (e, stack) { return _errorResponse(e, stack); @@ -1507,27 +1521,28 @@ class FlutterSkillBinding { _indicatorOverlay!.showTap(center, hint: "Tapping '$targetText'"); } - await _dispatchTap(center); - - // Fallback: directly invoke the callback if the widget supports it - _tryInvokeCallback(element); - - // Wait for frame to pump after callback invocation - await Future.delayed(const Duration(milliseconds: 300)); + // Exactly one activation: drive the tap through the pointer pipeline + // when the element is really hittable at its center; otherwise (covered + // by an overlay, pointer ignored) invoke the widget's callback directly. + if (_isHittableAt(element, center)) { + await _dispatchTap(center); + } else { + _tryInvokeCallback(element); + await _settle(); + } _log('Tap completed on (key: $key, text: $text)'); - return { + return _withFrameStatus({ 'success': true, 'message': 'Tap successful', 'target': {'key': key, 'text': text}, 'position': {'x': center.dx.round(), 'y': center.dy.round()}, - }; + }); } - /// Try to directly invoke the onPressed/onTap callback of a widget. - /// This is a fallback for when pointer dispatch doesn't trigger the callback - /// (e.g., when an overlay intercepts events or hit testing fails). + /// Directly invoke the onPressed/onTap callback of a widget — used only + /// when the element is not reachable by a pointer (see [_isHittableAt]). static void _tryInvokeCallback(Element element) { if (!element.mounted) return; final widget = element.widget; @@ -1643,10 +1658,104 @@ class FlutterSkillBinding { return scored.map((e) => e.key).toList(); } - static Future _dispatchTap(Offset position) async { + // ==================== SYNTHETIC POINTER ENGINE ==================== + // + // Every emulated input goes through [_emitGesture]. Flutter's gesture + // recognizers and scroll physics derive velocities from event + // timestamps, so each event carries a monotonically increasing + // [PointerEvent.timeStamp]; the pointer is always released, even if a + // handler throws; and the call returns only after the framework has + // rendered the frame that reflects the input, so a state query issued + // right afterwards observes the result. + + static final Stopwatch _pointerClock = Stopwatch()..start(); + + static Future _emitGesture({ + required Offset from, + Offset? to, + Duration hold = Duration.zero, + Duration travel = const Duration(milliseconds: 300), + }) async { final binding = WidgetsBinding.instance; final pointer = _pointerCounter++; + final end = to ?? from; + Duration now() => _pointerClock.elapsed; + + binding.handlePointerEvent( + PointerDownEvent(position: from, pointer: pointer, timeStamp: now())); + try { + if (hold > Duration.zero) await Future.delayed(hold); + if (end != from) { + // About one move per frame; enough steps to clear the touch slop + // early and give the velocity tracker a smooth sample history. + final steps = (travel.inMilliseconds / 16).round().clamp(4, 60); + final stepDelay = travel ~/ steps; + var last = from; + for (var i = 1; i <= steps; i++) { + await Future.delayed(stepDelay); + final position = Offset.lerp(from, end, i / steps)!; + binding.handlePointerEvent(PointerMoveEvent( + position: position, + delta: position - last, + pointer: pointer, + timeStamp: now(), + )); + last = position; + } + } + } finally { + binding.handlePointerEvent( + PointerUpEvent(position: end, pointer: pointer, timeStamp: now())); + } + await _settle(); + } + /// Completes once two frames have rendered: the one that runs the input's + /// callbacks and the one that paints the resulting state. Each wait is + /// bounded: a minimized/occluded window produces no frames on some + /// platforms (Windows), and an action must never hang on that. + static Future _settle() async { + _framesStalled = !await _nextFrame() && !await _nextFrame(); + } + + /// True when the last action saw no frame at all — the app is most likely + /// minimized or hidden, so nothing it did is observable yet. + static bool _framesStalled = false; + + static const _stalledWarning = + 'No frame was rendered after this action; the app window is probably ' + 'minimized or hidden. Restore it before reading state or taking a ' + 'screenshot.'; + + /// Adds the stalled-frames warning to an action response when relevant. + static Map _withFrameStatus(Map response) => + _framesStalled ? {...response, 'warning': _stalledWarning} : response; + + static Future _nextFrame() => SchedulerBinding.instance.endOfFrame + .then((_) => true) + .timeout(const Duration(milliseconds: 500), onTimeout: () => false); + + /// Whether a pointer at [position] would reach [element] (or a descendant), + /// i.e. nothing opaque covers it and it takes part in hit testing. + static bool _isHittableAt(Element element, Offset position) { + final target = element.renderObject; + if (target == null) return false; + final binding = WidgetsBinding.instance; + final result = HitTestResult(); + binding.hitTestInView( + result, position, binding.platformDispatcher.views.first.viewId); + for (final entry in result.path) { + RenderObject? node = + entry.target is RenderObject ? entry.target as RenderObject : null; + while (node != null) { + if (identical(node, target)) return true; + node = node.parent; + } + } + return false; + } + + static Future _dispatchTap(Offset position) async { // Show animated character walking to position and tapping if (_indicatorsEnabled && _indicatorOverlay != null) { // Use last position as starting point for walking animation @@ -1661,18 +1770,12 @@ class FlutterSkillBinding { await Future.delayed(const Duration(milliseconds: 200)); } - binding.handlePointerEvent( - PointerDownEvent(position: position, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 50)); - binding.handlePointerEvent( - PointerUpEvent(position: position, pointer: pointer)); + await _emitGesture(from: position, hold: const Duration(milliseconds: 50)); // Show tap ripple after character if (_indicatorsEnabled && _indicatorOverlay != null) { _indicatorOverlay!.showTap(position); } - - await Future.delayed(const Duration(milliseconds: 100)); } // ignore: unused_element @@ -1914,15 +2017,7 @@ class FlutterSkillBinding { final center = renderObject.localToGlobal(renderObject.size.center(Offset.zero)); - final binding = WidgetsBinding.instance; - final pointer = _pointerCounter++; - - binding.handlePointerEvent( - PointerDownEvent(position: center, pointer: pointer)); - await Future.delayed(Duration(milliseconds: duration)); - binding - .handlePointerEvent(PointerUpEvent(position: center, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + await _emitGesture(from: center, hold: Duration(milliseconds: duration)); _log('Long press completed (key: $key, text: $text)'); return true; @@ -1964,31 +2059,14 @@ class FlutterSkillBinding { return false; } - final pointer = _pointerCounter++; - final end = start + delta; - - binding.handlePointerEvent( - PointerDownEvent(position: start, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 16)); - - const steps = 10; - for (int i = 1; i <= steps; i++) { - final current = Offset.lerp(start, end, i / steps)!; - binding.handlePointerEvent(PointerMoveEvent( - position: current, - pointer: pointer, - delta: delta / steps.toDouble())); - await Future.delayed(const Duration(milliseconds: 16)); - } - - binding.handlePointerEvent(PointerUpEvent(position: end, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + await _emitGesture(from: start, to: start + delta); _log('Swipe $direction completed'); return true; } - static Future _performDrag({String? fromKey, String? toKey}) async { + static Future _performDrag( + {String? fromKey, String? toKey, int holdMs = 0}) async { if (fromKey == null || toKey == null) return false; final fromElement = _findElementByKey(fromKey); @@ -2002,23 +2080,14 @@ class FlutterSkillBinding { final start = fromRender.localToGlobal(fromRender.size.center(Offset.zero)); final end = toRender.localToGlobal(toRender.size.center(Offset.zero)); - final binding = WidgetsBinding.instance; - final pointer = _pointerCounter++; - - binding.handlePointerEvent( - PointerDownEvent(position: start, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); - - const steps = 20; - for (int i = 1; i <= steps; i++) { - final current = Offset.lerp(start, end, i / steps)!; - binding.handlePointerEvent( - PointerMoveEvent(position: current, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 16)); - } - - binding.handlePointerEvent(PointerUpEvent(position: end, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + // `hold` lets LongPressDraggable (and long-press-to-select lists) + // recognise the drag before the pointer starts moving. + await _emitGesture( + from: start, + to: end, + hold: Duration(milliseconds: holdMs), + travel: const Duration(milliseconds: 400), + ); _log('Drag from $fromKey to $toKey completed'); return true; @@ -2045,16 +2114,13 @@ class FlutterSkillBinding { // ==================== COORDINATE-BASED ACTIONS ==================== static Future _performTapAt(double x, double y) async { - final position = Offset(x, y); - await _dispatchTap(position); + await _dispatchTap(Offset(x, y)); _log('Tap at coordinates ($x, $y) completed'); } static Future _performLongPressAt(double x, double y, {int duration = 500}) async { final position = Offset(x, y); - final binding = WidgetsBinding.instance; - final pointer = _pointerCounter++; // Show animated character holding if (_indicatorsEnabled && _indicatorOverlay != null) { @@ -2067,12 +2133,7 @@ class FlutterSkillBinding { ); } - binding.handlePointerEvent( - PointerDownEvent(position: position, pointer: pointer)); - await Future.delayed(Duration(milliseconds: duration)); - binding.handlePointerEvent( - PointerUpEvent(position: position, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + await _emitGesture(from: position, hold: Duration(milliseconds: duration)); _log('Long press at coordinates ($x, $y) completed'); } @@ -2083,10 +2144,8 @@ class FlutterSkillBinding { double endX, double endY, { int duration = 300, + int holdMs = 0, }) async { - final binding = WidgetsBinding.instance; - final pointer = _pointerCounter++; - final start = Offset(startX, startY); final end = Offset(endX, endY); final delta = end - start; @@ -2100,25 +2159,12 @@ class FlutterSkillBinding { _indicatorOverlay!.showSwipe(start, end); } - binding.handlePointerEvent( - PointerDownEvent(position: start, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 16)); - - final steps = (duration / 16).round().clamp(5, 30); - final stepDuration = duration ~/ steps; - - for (int i = 1; i <= steps; i++) { - final current = Offset.lerp(start, end, i / steps)!; - binding.handlePointerEvent(PointerMoveEvent( - position: current, - pointer: pointer, - delta: delta / steps.toDouble(), - )); - await Future.delayed(Duration(milliseconds: stepDuration)); - } - - binding.handlePointerEvent(PointerUpEvent(position: end, pointer: pointer)); - await Future.delayed(const Duration(milliseconds: 100)); + await _emitGesture( + from: start, + to: end, + hold: Duration(milliseconds: holdMs), + travel: Duration(milliseconds: duration), + ); _log('Swipe from ($startX, $startY) to ($endX, $endY) completed'); } @@ -2352,6 +2398,12 @@ class FlutterSkillBinding { }; entry['visible'] = hasValidSize; entry['coordinatesReliable'] = coordinatesReliable; + // `visible` only says the box has a size; `hittable` says a pointer + // at its center actually reaches it (not offstage, not covered by + // a bar/overlay) — the flag to trust before tapping/dragging. + entry['hittable'] = coordinatesReliable && + _isHittableAt( + element, offset + renderObject.size.center(Offset.zero)); // Add warning for unreliable coordinates if (!coordinatesReliable && isAtOrigin) { @@ -2573,6 +2625,8 @@ class FlutterSkillBinding { offset.dx.isFinite && offset.dy.isFinite; state['visible'] = isVisible; + state['hittable'] = isVisible && + _isHittableAt(element, offset + size.center(Offset.zero)); // Cache element for performance optimization if (isVisible && refId.isNotEmpty) { @@ -2582,6 +2636,7 @@ class FlutterSkillBinding { // Element has no render box or size - set default bounds elementEntry['bounds'] = {'x': 0, 'y': 0, 'w': 0, 'h': 0}; state['visible'] = false; + state['hittable'] = false; } // Merge state information @@ -2931,7 +2986,8 @@ class FlutterSkillBinding { /// avoiding the bug where DFS finds a RepaintBoundary from an old route. /// Returns the image in logical pixel dimensions scaled by [pixelRatio]. static Future _captureFullScene({double pixelRatio = 1.0}) async { - await WidgetsBinding.instance.endOfFrame; + // Bounded: a hidden window may never deliver a frame (see _nextFrame). + await _nextFrame(); final binding = WidgetsBinding.instance; // ignore: invalid_use_of_protected_member @@ -2982,7 +3038,7 @@ class FlutterSkillBinding { return boundary!.toImage(pixelRatio: pixelRatio); } - static Future _takeScreenshot( + static Future<_Screenshot?> _takeScreenshot( {double quality = 1.0, int? maxWidth}) async { try { // Use quality as pixel ratio (lower = smaller image) @@ -3014,7 +3070,8 @@ class FlutterSkillBinding { final byteData = await image.toByteData(format: ui.ImageByteFormat.png); if (byteData == null) return null; - return base64Encode(byteData.buffer.asUint8List()); + return _Screenshot(base64Encode(byteData.buffer.asUint8List()), + image.width, image.height); } catch (e) { _log('Screenshot failed: $e'); return null; @@ -4196,7 +4253,7 @@ class _ParticleEffectPainter extends CustomPainter { const Color(0xFFFF5722), i / 6, )! - .withOpacity(0.8 * (1 - progress)); + .withValues(alpha: (0.8 * (1 - progress)).clamp(0.0, 1.0)); particlePaint.color = color; canvas.drawCircle(Offset(x, y), 3 * (1 - progress), particlePaint); @@ -4906,3 +4963,11 @@ class _ActionHint extends StatelessWidget { ); } } + +/// PNG screenshot payload with its pixel dimensions. +class _Screenshot { + const _Screenshot(this.base64, this.width, this.height); + final String base64; + final int width; + final int height; +} diff --git a/lib/src/cli/tool_handlers/bf_batch.dart b/lib/src/cli/tool_handlers/bf_batch.dart index 43ea80cd..3ffcba6c 100644 --- a/lib/src/cli/tool_handlers/bf_batch.dart +++ b/lib/src/cli/tool_handlers/bf_batch.dart @@ -50,8 +50,14 @@ extension _BfBatch on FlutterMcpServer { final fc = _asFlutterClient(client!, 'tap_at'); final x = (args['x'] as num).toDouble(); final y = (args['y'] as num).toDouble(); - await fc.tapAt(x, y); - return {"success": true, "action": "tap_at", "x": x, "y": y}; + final tapped = await fc.tapAt(x, y); + return { + "success": true, + "action": "tap_at", + "x": x, + "y": y, + if (tapped['warning'] != null) "warning": tapped['warning'], + }; case 'long_press_at': if (client is BridgeDriver) { diff --git a/lib/src/cli/tool_handlers/bf_interaction.dart b/lib/src/cli/tool_handlers/bf_interaction.dart index d83d9869..3f1b5687 100644 --- a/lib/src/cli/tool_handlers/bf_interaction.dart +++ b/lib/src/cli/tool_handlers/bf_interaction.dart @@ -22,12 +22,13 @@ extension _BfInteraction on FlutterMcpServer { }; } final fc = _asFlutterClient(client!, 'tap (coordinates)'); - await fc.tapAt(x.toDouble(), y.toDouble()); + final tapped = await fc.tapAt(x.toDouble(), y.toDouble()); return { "success": true, "method": "coordinates", "message": "Tapped at ($x, $y)", "position": {"x": x, "y": y}, + if (tapped['warning'] != null) "warning": tapped['warning'], }; } @@ -53,6 +54,7 @@ extension _BfInteraction on FlutterMcpServer { "method": args['key'] != null ? "key" : "text", "message": "Tapped", if (result['position'] != null) "position": result['position'], + if (result['warning'] != null) "warning": result['warning'], }; case 'fill': @@ -136,6 +138,19 @@ extension _BfInteraction on FlutterMcpServer { return success ? "Double tapped" : "Double tap failed"; case 'swipe': final distance = (args['distance'] ?? 300).toDouble(); + if (client is FlutterSkillClient) { + final swiped = await client.swipeWithDetails( + direction: args['direction'], + distance: distance, + key: args['key']); + return { + "success": swiped['success'] == true, + "message": swiped['success'] == true + ? "Swiped ${args['direction']}" + : "Swipe failed", + if (swiped['warning'] != null) "warning": swiped['warning'], + }; + } final success = await client!.swipe( direction: args['direction'], distance: distance, key: args['key']); return success ? "Swiped ${args['direction']}" : "Swipe failed"; @@ -156,20 +171,31 @@ extension _BfInteraction on FlutterMcpServer { return result['success'] == true ? "Dragged" : "Drag failed"; } final fc = _asFlutterClient(client!, 'drag'); - // Support coordinate-based drag via swipeCoordinates for FlutterClient - if (args['start_x'] != null) { + // Clients may send numbers as strings; accept both. + double? num_(Object? v) => + v is num ? v.toDouble() : double.tryParse(v?.toString() ?? ''); + final holdMs = num_(args['hold_ms'])?.toInt() ?? 0; + // Coordinate-based drag shares the swipe engine (with hold) + if (num_(args['start_x']) != null) { final result = await fc.swipeCoordinates( - (args['start_x'] as num).toDouble(), - (args['start_y'] as num).toDouble(), - (args['end_x'] as num).toDouble(), - (args['end_y'] as num).toDouble(), + num_(args['start_x'])!, + num_(args['start_y']) ?? 0, + num_(args['end_x']) ?? 0, + num_(args['end_y']) ?? 0, + duration: num_(args['duration_ms'])?.toInt() ?? 400, + holdMs: holdMs, ); return result; } - final success = await fc.drag( + final dragged = await fc.dragWithDetails( fromKey: args['from_key'] as String? ?? '', - toKey: args['to_key'] as String? ?? ''); - return success ? "Dragged" : "Drag failed"; + toKey: args['to_key'] as String? ?? '', + holdMs: holdMs); + return { + "success": dragged['success'] == true, + "message": dragged['success'] == true ? "Dragged" : "Drag failed", + if (dragged['warning'] != null) "warning": dragged['warning'], + }; // State & Validation case 'get_text_value': diff --git a/lib/src/cli/tool_handlers/bf_screenshot.dart b/lib/src/cli/tool_handlers/bf_screenshot.dart index 2016310b..f3bf17fc 100644 --- a/lib/src/cli/tool_handlers/bf_screenshot.dart +++ b/lib/src/cli/tool_handlers/bf_screenshot.dart @@ -1,6 +1,26 @@ part of '../server.dart'; extension _BfScreenshot on FlutterMcpServer { + /// Image vs logical geometry so an agent can convert pixels it sees in the + /// PNG back to the logical coordinates that tap_at/swipe/drag expect. + Map _screenshotGeometry(Map info) { + final imageWidth = info['imageWidth'] as num?; + final logicalWidth = info['logicalWidth'] as num?; + if (imageWidth == null || logicalWidth == null || logicalWidth == 0) { + return const {}; + } + final scale = imageWidth / logicalWidth; + return { + "image_size": {"width": imageWidth, "height": info['imageHeight']}, + "logical_size": {"width": logicalWidth, "height": info['logicalHeight']}, + "device_pixel_ratio": info['devicePixelRatio'], + "scale": scale, + "coordinates_hint": "Tools take LOGICAL coordinates. To tap what you " + "see at image pixel (px, py): x = px / scale, y = py / scale " + "(scale = ${scale.toStringAsFixed(3)}). Prefer inspect() centers.", + }; + } + Future _handleScreenshotTool( String name, Map args, AppDriver? client) async { switch (name) { @@ -11,8 +31,17 @@ extension _BfScreenshot on FlutterMcpServer { final saveToFile = args['save_to_file'] ?? true; // Save to file by default to avoid token overflow - var imageBase64 = - await client!.takeScreenshot(quality: quality, maxWidth: maxWidth); + Map geometry = const {}; + String? imageBase64; + if (client is FlutterSkillClient) { + final info = await client.takeScreenshotWithInfo( + quality: quality, maxWidth: maxWidth); + imageBase64 = info['image'] as String?; + geometry = _screenshotGeometry(info); + } else { + imageBase64 = await client! + .takeScreenshot(quality: quality, maxWidth: maxWidth); + } // Fallback to native screenshot if bridge returns null // (e.g. React Native returns _needs_native flag) @@ -56,6 +85,7 @@ extension _BfScreenshot on FlutterMcpServer { "quality": quality, "max_width": maxWidth, "format": "png", + ...geometry, "message": "Screenshot saved to ${file.path}" }; } else { @@ -64,6 +94,7 @@ extension _BfScreenshot on FlutterMcpServer { "image": imageBase64, "quality": quality, "max_width": maxWidth, + ...geometry, "warning": "Returning base64 data. Consider using save_to_file=true for large images." }; diff --git a/lib/src/cli/tool_handlers/tool_definitions.dart b/lib/src/cli/tool_handlers/tool_definitions.dart index c4ede0e4..da1d8f4a 100644 --- a/lib/src/cli/tool_handlers/tool_definitions.dart +++ b/lib/src/cli/tool_handlers/tool_definitions.dart @@ -1724,14 +1724,27 @@ Option 3: Tap a TextField first, then enter_text(text: "value") without key/ref }, { "name": "drag", - "description": "Drag from one element to another", + "description": "Drag from one element (or point) to another. " + "Set hold_ms (e.g. 400) for LongPressDraggable / long-press-to-" + "select lists so the drag is recognised before the pointer moves.", "inputSchema": { "type": "object", "properties": { "from_key": {"type": "string", "description": "Source element key"}, "to_key": {"type": "string", "description": "Target element key"}, + "start_x": {"type": "number", "description": "Start X (logical)"}, + "start_y": {"type": "number", "description": "Start Y (logical)"}, + "end_x": {"type": "number", "description": "End X (logical)"}, + "end_y": {"type": "number", "description": "End Y (logical)"}, + "hold_ms": { + "type": "integer", + "description": "Press-and-hold before moving, in ms (default 0)" + }, + "duration_ms": { + "type": "integer", + "description": "Travel time in ms (default 400)" + }, }, - "required": ["from_key", "to_key"], }, }, diff --git a/lib/src/drivers/flutter_driver.dart b/lib/src/drivers/flutter_driver.dart index 7a48fe75..349da5ba 100644 --- a/lib/src/drivers/flutter_driver.dart +++ b/lib/src/drivers/flutter_driver.dart @@ -287,20 +287,36 @@ Auto-setup: run diagnose_project() to fix automatically. Future swipe( {required String direction, double distance = 300, String? key}) async { - final result = await _call('ext.flutter.flutter_skill.swipe', { + final result = await swipeWithDetails( + direction: direction, distance: distance, key: key); + return result['success'] == true; + } + + /// Swipe; the map carries `success` and, when frames stalled, `warning`. + Future> swipeWithDetails( + {required String direction, double distance = 300, String? key}) async { + return await _call('ext.flutter.flutter_skill.swipe', { 'direction': direction, 'distance': distance.toString(), if (key != null) 'key': key, }); + } + + Future drag( + {required String fromKey, required String toKey, int holdMs = 0}) async { + final result = + await dragWithDetails(fromKey: fromKey, toKey: toKey, holdMs: holdMs); return result['success'] == true; } - Future drag({required String fromKey, required String toKey}) async { - final result = await _call('ext.flutter.flutter_skill.drag', { + /// Drag; the map carries `success` and, when frames stalled, `warning`. + Future> dragWithDetails( + {required String fromKey, required String toKey, int holdMs = 0}) async { + return await _call('ext.flutter.flutter_skill.drag', { 'fromKey': fromKey, 'toKey': toKey, + 'hold': holdMs.toString(), }); - return result['success'] == true; } Future doubleTap({String? key, String? text}) async { @@ -369,11 +385,18 @@ Auto-setup: run diagnose_project() to fix automatically. // ==================== SCREENSHOT ==================== Future takeScreenshot({double quality = 1.0, int? maxWidth}) async { - final result = await _call('ext.flutter.flutter_skill.screenshot', { + return (await takeScreenshotWithInfo( + quality: quality, maxWidth: maxWidth))['image'] as String?; + } + + /// Screenshot plus its geometry: `image` (base64 PNG), `imageWidth`, + /// `imageHeight`, `logicalWidth`, `logicalHeight`, `devicePixelRatio`. + Future> takeScreenshotWithInfo( + {double quality = 1.0, int? maxWidth}) async { + return await _call('ext.flutter.flutter_skill.screenshot', { 'quality': quality.toString(), if (maxWidth != null) 'maxWidth': maxWidth.toString(), }); - return result['image']; } Future takeRegionScreenshot( @@ -455,6 +478,7 @@ Auto-setup: run diagnose_project() to fix automatically. double endX, double endY, { int duration = 300, + int holdMs = 0, }) async { return await _call('ext.flutter.flutter_skill.swipeCoordinates', { 'startX': startX.toString(), @@ -462,6 +486,7 @@ Auto-setup: run diagnose_project() to fix automatically. 'endX': endX.toString(), 'endY': endY.toString(), 'duration': duration.toString(), + 'hold': holdMs.toString(), }); } diff --git a/lib/src/engine/tool_registry.dart b/lib/src/engine/tool_registry.dart index 60c4bdd7..0ef86071 100644 --- a/lib/src/engine/tool_registry.dart +++ b/lib/src/engine/tool_registry.dart @@ -1844,14 +1844,28 @@ Option 3: Tap a TextField first, then enter_text(text: "value") without key/ref }, { "name": "drag", - "description": "Drag from one element to another", + "description": "Drag from one element (from_key/to_key) or between " + "logical coordinates (start_x/start_y -> end_x/end_y). Set " + "hold_ms (e.g. 400) for LongPressDraggable / long-press-to-select " + "lists so the drag is recognised before the pointer moves.", "inputSchema": { "type": "object", "properties": { "from_key": {"type": "string", "description": "Source element key"}, - "to_key": {"type": "string", "description": "Target element key"} - }, - "required": ["from_key", "to_key"] + "to_key": {"type": "string", "description": "Target element key"}, + "start_x": {"type": "number", "description": "Start X (logical)"}, + "start_y": {"type": "number", "description": "Start Y (logical)"}, + "end_x": {"type": "number", "description": "End X (logical)"}, + "end_y": {"type": "number", "description": "End Y (logical)"}, + "hold_ms": { + "type": "integer", + "description": "Press-and-hold before moving, in ms (default 0)" + }, + "duration_ms": { + "type": "integer", + "description": "Travel time in ms (default 400)" + } + } } }, From 7e911fe6f72822a20e132550b13d606fee9cb62a Mon Sep 17 00:00:00 2001 From: cornebidouil Date: Mon, 17 Aug 2026 12:37:01 +0200 Subject: [PATCH 3/4] fix(tools): routes, hardware keys, focus/type_text/set_checkbox/hover, lazy scroll_to, scroll direction, batch settle Sweep of the remaining tool families against a probe app (54 checks, tools/harness_sweep.py in the linked repro): - get_current_route/get_navigation_stack: the SDK tested `element.widget is ModalRoute` (a Route is never a Widget) so it always returned null, and the server treated the null as "tool not handled" ("Unknown tool"). Routes are now collected from each route's _ModalScopeStatus scope in stack order, with name/key/isCurrent/depth; the current route is the topmost current one (nested navigators win). - press_key: fed KeyData to KeyEventManager, which on desktop waits for the paired raw-key message that never comes, and mapped every physical key to Enter. Keys now go through HardwareKeyboard + the KeyMessage handler (Focus -> Shortcuts/Actions), with correct USB-HID physical keys and held modifiers; Enter on a focused field also performs the "done" action. - New Flutter implementations for tools that were CDP-only: type_text (IME-style insert at the selection), focus, blur, set_checkbox (also Checkbox/SwitchListTile), hover (one synthetic mouse device). - scroll_to: pages through every scrollable when the target is not built yet (ListView.builder), then ensureVisible. - scroll_until_visible: 'direction' is the scroll direction, so the finger swipe is the opposite way; finishes with ensureVisible (being in the cache extent is not being visible). - execute_batch: 250 ms settle between steps (step_delay_ms). - Null results from handlers no longer surface as "Unknown tool". --- lib/flutter_skill.dart | 438 ++++++++++++++---- lib/src/cli/server.dart | 6 + lib/src/cli/tool_handlers/bf_inspection.dart | 8 +- lib/src/cli/tool_handlers/bf_interaction.dart | 37 +- lib/src/cli/tool_handlers/bf_navigation.dart | 3 +- .../bridge_flutter_handlers.dart | 3 +- .../cli/tool_handlers/flutter_helpers.dart | 16 +- lib/src/drivers/flutter_driver.dart | 25 +- 8 files changed, 449 insertions(+), 87 deletions(-) diff --git a/lib/flutter_skill.dart b/lib/flutter_skill.dart index c825c50b..49d50d2d 100644 --- a/lib/flutter_skill.dart +++ b/lib/flutter_skill.dart @@ -5,6 +5,7 @@ import 'dart:math'; import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; +import 'package:flutter/gestures.dart' show PointerAddedEvent; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter/scheduler.dart'; @@ -626,7 +627,7 @@ class FlutterSkillBinding { try { final route = _getCurrentRoute(); return developer.ServiceExtensionResponse.result( - jsonEncode({'route': route})); + jsonEncode({'route': route, 'routes': _routeEntries()})); } catch (e, stack) { return _errorResponse(e, stack); } @@ -726,6 +727,144 @@ class FlutterSkillBinding { } }); + // ==================== FOCUS / TYPING / TOGGLES / HOVER ==================== + + // type_text: insert into the focused field the way an IME commits text + // (replaces the current selection), so onChanged fires as for a user. + developer.registerExtension('ext.flutter.flutter_skill.typeText', + (method, parameters) async { + try { + final text = parameters['text'] ?? ''; + final field = _findFocusedTextField(); + if (field == null) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'No focused text field. Use focus(key) or tap it first.', + })); + } + final value = field.textEditingValue; + final sel = value.selection.isValid + ? value.selection + : TextSelection.collapsed(offset: value.text.length); + final newText = value.text.replaceRange(sel.start, sel.end, text); + field.updateEditingValue(TextEditingValue( + text: newText, + selection: TextSelection.collapsed(offset: sel.start + text.length), + )); + await _settle(); + return developer.ServiceExtensionResponse.result(jsonEncode( + {'success': true, 'message': 'Typed "$text"', 'value': newText})); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // focus / blur: drive the field's own FocusNode. + developer.registerExtension('ext.flutter.flutter_skill.focus', + (method, parameters) async { + try { + final key = parameters['key']; + final field = key == null ? null : _findEditableText(key); + if (field == null) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'No text field with key "$key"', + })); + } + field.widget.focusNode.requestFocus(); + await _settle(); + return developer.ServiceExtensionResponse.result( + jsonEncode({'success': true, 'message': 'Focused "$key"'})); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + developer.registerExtension('ext.flutter.flutter_skill.blur', + (method, parameters) async { + try { + final key = parameters['key']; + final field = + key == null ? _findFocusedTextField() : _findEditableText(key); + if (field == null) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'No focused text field to blur', + })); + } + field.widget.focusNode.unfocus(); + await _settle(); + return developer.ServiceExtensionResponse.result( + jsonEncode({'success': true, 'message': 'Blurred'})); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // set_checkbox: tap the toggle until it shows the requested state. + developer.registerExtension('ext.flutter.flutter_skill.setCheckbox', + (method, parameters) async { + try { + final key = parameters['key'] ?? ''; + final wanted = parameters['checked'] == 'true'; + final current = _getCheckboxState(key); + if (current == null) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'No Checkbox/Switch (or *ListTile) with key "$key"', + })); + } + if (current != wanted) { + await _performTapWithDetails(key: key); + } + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': _getCheckboxState(key) == wanted, + 'checked': _getCheckboxState(key), + })); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + + // hover: move a mouse pointer over the element (tooltips, hover styles). + developer.registerExtension('ext.flutter.flutter_skill.hover', + (method, parameters) async { + try { + final element = + _findElement(key: parameters['key'], text: parameters['text']); + final box = element?.renderObject; + if (element == null || box is! RenderBox || !box.hasSize) { + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': false, + 'error': 'Element not found or not laid out', + })); + } + final position = box.localToGlobal(box.size.center(Offset.zero)); + final binding = WidgetsBinding.instance; + // One synthetic mouse device, added once: MouseTracker asserts on a + // repeated PointerAddedEvent for a device that was never removed. + if (!_syntheticMouseAdded) { + binding.handlePointerEvent(PointerAddedEvent( + position: position, + kind: ui.PointerDeviceKind.mouse, + device: _syntheticMouseDevice, + timeStamp: _pointerClock.elapsed)); + _syntheticMouseAdded = true; + } + binding.handlePointerEvent(PointerHoverEvent( + position: position, + kind: ui.PointerDeviceKind.mouse, + device: _syntheticMouseDevice, + timeStamp: _pointerClock.elapsed)); + await _settle(); + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': true, + 'position': {'x': position.dx.round(), 'y': position.dy.round()}, + })); + } catch (e, stack) { + return _errorResponse(e, stack); + } + }); + // ==================== PRESS KEY ==================== developer.registerExtension('ext.flutter.flutter_skill.pressKey', @@ -778,55 +917,21 @@ class FlutterSkillBinding { // ignore: unused_local_variable final isMeta = modifiers.contains('meta'); - // Simulate key press through the focus system - final focusNode = FocusManager.instance.primaryFocus; - if (focusNode != null) { - // Use HardwareKeyboard simulation - // ignore: unused_local_variable - final binding = WidgetsBinding.instance; - // ignore: unused_local_variable - final pointer = _pointerCounter++; - - // Create a RawKeyDownEvent and dispatch through the focus system - // ignore: unused_local_variable - final keyDown = KeyDownEvent( - physicalKey: PhysicalKeyboardKey.findKeyByCode(logicalKey.keyId) ?? - PhysicalKeyboardKey.enter, - logicalKey: logicalKey, - timeStamp: - Duration(milliseconds: DateTime.now().millisecondsSinceEpoch), - ); - - // Dispatch through ServicesBinding - await ServicesBinding.instance.keyEventManager - .handleKeyData(ui.KeyData( - type: ui.KeyEventType.down, - timeStamp: - Duration(milliseconds: DateTime.now().millisecondsSinceEpoch), - physical: (PhysicalKeyboardKey.findKeyByCode(logicalKey.keyId) ?? - PhysicalKeyboardKey.enter) - .usbHidUsage, - logical: logicalKey.keyId, - character: key.length == 1 ? key : null, - synthesized: false, - )); - - await Future.delayed(const Duration(milliseconds: 50)); - - await ServicesBinding.instance.keyEventManager - .handleKeyData(ui.KeyData( - type: ui.KeyEventType.up, - timeStamp: - Duration(milliseconds: DateTime.now().millisecondsSinceEpoch), - physical: (PhysicalKeyboardKey.findKeyByCode(logicalKey.keyId) ?? - PhysicalKeyboardKey.enter) - .usbHidUsage, - logical: logicalKey.keyId, - character: null, - synthesized: false, - )); + // Dispatch as a real hardware key: down (with modifiers held), up. + // Enter on a focused text field also performs the platform's "done" + // action, which is what fires onSubmitted on desktop. + final handled = await _pressHardwareKey(logicalKey, modifiers, + character: key.length == 1 ? key : null); + if (logicalKey == LogicalKeyboardKey.enter) { + _findFocusedTextField()?.performAction(TextInputAction.done); } - + await _settle(); + return developer.ServiceExtensionResponse.result(jsonEncode({ + 'success': true, + 'message': 'Key pressed: $key', + 'handled': handled, + })); + // ignore: dead_code return developer.ServiceExtensionResponse.result( jsonEncode({'success': true, 'message': 'Key pressed: $key'}), ); @@ -1658,6 +1763,97 @@ class FlutterSkillBinding { return scored.map((e) => e.key).toList(); } + // ==================== SYNTHETIC KEYBOARD ==================== + + /// USB HID usage codes for the physical keys we can synthesize; the + /// framework requires physical/logical pairs to be consistent. + static const Map _physicalUsageByLogicalId = { + 0x100000008: 0x0007002a, // backspace + 0x100000009: 0x0007002b, // tab + 0x10000000d: 0x00070028, // enter + 0x10000001b: 0x00070029, // escape + 0x10000007f: 0x0007004c, // delete + 0x100000301: 0x00070051, // arrowDown + 0x100000302: 0x00070050, // arrowLeft + 0x100000303: 0x0007004f, // arrowRight + 0x100000304: 0x00070052, // arrowUp + 0x100000305: 0x0007004d, // end + 0x100000306: 0x0007004a, // home + 0x100000307: 0x0007004e, // pageDown + 0x100000308: 0x0007004b, // pageUp + 0x20: 0x0007002c, // space + }; + + /// Device id of the synthetic mouse used by `hover` (added on first use). + static const int _syntheticMouseDevice = 0x5ca1; + static bool _syntheticMouseAdded = false; + + static PhysicalKeyboardKey _physicalFor(LogicalKeyboardKey logical) { + final id = logical.keyId; + final mapped = _physicalUsageByLogicalId[id]; + if (mapped != null) return PhysicalKeyboardKey(mapped); + // Unicode letters/digits: keyA..keyZ = 0x70004.., digit1..9,0 = 0x7001e.. + if (id >= 0x61 && id <= 0x7a) + return PhysicalKeyboardKey(0x00070004 + id - 0x61); + if (id >= 0x31 && id <= 0x39) + return PhysicalKeyboardKey(0x0007001e + id - 0x31); + if (id == 0x30) return const PhysicalKeyboardKey(0x00070027); + return PhysicalKeyboardKey.findKeyByCode(id) ?? PhysicalKeyboardKey.enter; + } + + static const Map + _modifierKeys = { + 'shift': (LogicalKeyboardKey.shiftLeft, PhysicalKeyboardKey.shiftLeft), + 'ctrl': (LogicalKeyboardKey.controlLeft, PhysicalKeyboardKey.controlLeft), + 'control': ( + LogicalKeyboardKey.controlLeft, + PhysicalKeyboardKey.controlLeft + ), + 'alt': (LogicalKeyboardKey.altLeft, PhysicalKeyboardKey.altLeft), + 'meta': (LogicalKeyboardKey.metaLeft, PhysicalKeyboardKey.metaLeft), + }; + + /// Delivers one synthetic [KeyEvent] the way the engine's key path does + /// once a platform message is decoded: [HardwareKeyboard] records the + /// pressed state, then the [KeyMessage] goes to the global handler + /// (FocusManager → Shortcuts/Actions → text-editing shortcuts). + /// Returns whether a handler claimed it. + static bool _deliverKey(KeyEvent event) { + HardwareKeyboard.instance.handleKeyEvent(event); + final handler = ServicesBinding.instance.keyEventManager.keyMessageHandler; + return handler?.call(KeyMessage([event], null)) ?? false; + } + + /// Sends a physical key press (down, up) with [modifiers] held around it. + static Future _pressHardwareKey( + LogicalKeyboardKey logical, List modifiers, + {String? character}) async { + Duration now() => _pointerClock.elapsed; + final held = <(LogicalKeyboardKey, PhysicalKeyboardKey)>[]; + for (final m in modifiers) { + final pair = _modifierKeys[m.toLowerCase()]; + if (pair == null) continue; + _deliverKey(KeyDownEvent( + physicalKey: pair.$2, logicalKey: pair.$1, timeStamp: now())); + held.add(pair); + } + final physical = _physicalFor(logical); + final handled = _deliverKey(KeyDownEvent( + physicalKey: physical, + logicalKey: logical, + character: character, + timeStamp: now(), + )); + await Future.delayed(const Duration(milliseconds: 40)); + _deliverKey(KeyUpEvent( + physicalKey: physical, logicalKey: logical, timeStamp: now())); + for (final pair in held.reversed) { + _deliverKey(KeyUpEvent( + physicalKey: pair.$2, logicalKey: pair.$1, timeStamp: now())); + } + return handled; + } + // ==================== SYNTHETIC POINTER ENGINE ==================== // // Every emulated input goes through [_emitGesture]. Flutter's gesture @@ -1969,11 +2165,16 @@ class FlutterSkillBinding { static Future> _performScroll( {String? key, String? text}) async { - final element = _findElement(key: key, text: text); + var element = _findElement(key: key, text: text); + // Lazily built lists (ListView.builder, slivers) only mount the rows in + // view, so an off-screen target has no element yet. Page through every + // scrollable until it appears, then let ensureVisible do the fine work. + element ??= await _revealByPaging(() => _findElement(key: key, text: text)); if (element == null) { return { 'success': false, - 'error': 'Element not found (key: $key, text: $text)' + 'error': 'Element not found (key: $key, text: $text), ' + 'also after paging through all scrollables', }; } @@ -2933,15 +3134,32 @@ class FlutterSkillBinding { if (element == null) return null; final widget = element.widget; - if (widget is Checkbox) { - return widget.value; - } - if (widget is Switch) { - return widget.value; - } + if (widget is Checkbox) return widget.value; + if (widget is Switch) return widget.value; + if (widget is CheckboxListTile) return widget.value; + if (widget is SwitchListTile) return widget.value; return null; } + /// The EditableText inside the widget with [key] (TextField, TextFormField + /// or a wrapper), or null. + static EditableTextState? _findEditableText(String key) { + final element = _findElementByKey(key); + if (element == null) return null; + EditableTextState? found; + void visit(Element e) { + if (found != null) return; + if (e is StatefulElement && e.state is EditableTextState) { + found = e.state as EditableTextState; + return; + } + e.visitChildren(visit); + } + + visit(element); + return found; + } + static double? _getSliderValue(String key) { final element = _findElementByKey(key); if (element == null) return null; @@ -3183,26 +3401,56 @@ class FlutterSkillBinding { // ==================== NAVIGATION ==================== - static String? _getCurrentRoute() { - String? currentRoute; - final binding = WidgetsBinding.instance; - - void visit(Element element) { - if (element.widget is ModalRoute) { - final route = ModalRoute.of(element); - if (route != null) { - currentRoute = route.settings.name; + /// Every route currently mounted, in stack order (outer navigator first, + /// each navigator bottom → top, nested navigators after the route that + /// hosts them). Each entry: `name` (RouteSettings.name — go_router sets + /// the route name or path), `key` (Page key), `isCurrent`, `depth` + /// (navigator nesting), `type`. + /// + /// Routes are found through the `_ModalScopeStatus` inherited widget that + /// every ModalRoute installs above its content: walking the element tree + /// meets each route exactly once and does not register dependencies (as + /// `ModalRoute.of` would). Field access is dynamic because the class is + /// private to the framework; its members are public and stable. + static List> _routeEntries() { + final entries = >[]; + void visit(Element element, int depth) { + var next = depth; + final widget = element.widget; + if (widget.runtimeType.toString() == '_ModalScopeStatus') { + try { + final route = (widget as dynamic).route as Route; + final settings = route.settings; + entries.add({ + 'name': settings.name, + 'key': settings is Page ? settings.key?.toString() : null, + 'isCurrent': route.isCurrent, + 'depth': depth, + 'type': route.runtimeType.toString(), + }); + next = depth + 1; + } catch (_) { + // framework internals changed shape; skip silently } } - element.visitChildren(visit); + element.visitChildren((child) => visit(child, next)); } // ignore: invalid_use_of_protected_member - if (binding.rootElement != null) { - visit(binding.rootElement!); - } + final rootElement = WidgetsBinding.instance.rootElement; + if (rootElement != null) visit(rootElement, 0); + return entries; + } - return currentRoute; + /// Name of the topmost current route (innermost navigator wins). Falls + /// back to the page key when the route has no name. + static String? _getCurrentRoute() { + Map? current; + for (final entry in _routeEntries()) { + if (entry['isCurrent'] == true) current = entry; + } + if (current == null) return null; + return (current['name'] ?? current['key']) as String?; } static bool _goBack() { @@ -3219,16 +3467,50 @@ class FlutterSkillBinding { static List _getNavigationStack() { final routes = []; - final context = _findNavigatorContext(); - if (context == null) return routes; + for (final entry in _routeEntries()) { + final label = (entry['name'] ?? entry['key'] ?? entry['type']) as String; + routes.add(label); + } + return routes; + } - // This is a simplified version - full implementation would need NavigatorState access - final currentRoute = _getCurrentRoute(); - if (currentRoute != null) { - routes.add(currentRoute); + /// Pages every mounted vertical/horizontal scrollable from its start to + /// its end (viewport-sized jumps, one frame each) until [find] returns an + /// element. Returns it, or null when nothing appeared anywhere. + static Future _revealByPaging(Element? Function() find) async { + final positions = []; + void visit(Element e) { + if (e is StatefulElement && e.state is ScrollableState) { + final position = (e.state as ScrollableState).position; + if (position.hasContentDimensions && + position.maxScrollExtent > position.minScrollExtent) { + positions.add(position); + } + } + e.visitChildren(visit); } - return routes; + // ignore: invalid_use_of_protected_member + final rootElement = WidgetsBinding.instance.rootElement; + if (rootElement != null) visit(rootElement); + + for (final position in positions) { + final origin = position.pixels; + final step = position.viewportDimension * 0.8; + var offset = position.minScrollExtent; + while (true) { + position.jumpTo(offset); + await _settle(); + final found = find(); + if (found != null) return found; + if (offset >= position.maxScrollExtent) break; + offset = (offset + step) + .clamp(position.minScrollExtent, position.maxScrollExtent); + } + position.jumpTo(origin); + await _settle(); + } + return null; } static BuildContext? _findNavigatorContext() { diff --git a/lib/src/cli/server.dart b/lib/src/cli/server.dart index ace40567..df97b9f9 100644 --- a/lib/src/cli/server.dart +++ b/lib/src/cli/server.dart @@ -728,11 +728,17 @@ class FlutterMcpServer { final actions = (args['actions'] ?? args['commands'] ?? []) as List; final stopOnFailure = args['stop_on_failure'] ?? true; + // A step often triggers a route/tab transition; give the next step's + // target time to be built (default 250 ms, tunable per batch). + final stepDelayMs = (args['step_delay_ms'] as num?)?.toInt() ?? 250; final results = >[]; var allSuccess = true; for (var i = 0; i < actions.length; i++) { + if (i > 0 && stepDelayMs > 0) { + await Future.delayed(Duration(milliseconds: stepDelayMs)); + } final action = actions[i] as Map; final actionName = (action['action'] ?? action['tool'] ?? action['name']) as String; diff --git a/lib/src/cli/tool_handlers/bf_inspection.dart b/lib/src/cli/tool_handlers/bf_inspection.dart index b5833f0d..5dcbc37e 100644 --- a/lib/src/cli/tool_handlers/bf_inspection.dart +++ b/lib/src/cli/tool_handlers/bf_inspection.dart @@ -203,7 +203,13 @@ extension _BfInspection on FlutterMcpServer { "error": "key or element parameter required" }; } - return await fc.getWidgetProperties(wpKey); + final props = await fc.getWidgetProperties(wpKey); + return props ?? + { + "success": false, + "error": "No mounted widget with key '$wpKey' " + "(is it on the current screen / built?)", + }; case 'get_text_content': if (client is BridgeDriver) { final text = await client.getText(); diff --git a/lib/src/cli/tool_handlers/bf_interaction.dart b/lib/src/cli/tool_handlers/bf_interaction.dart index 3f1b5687..95d07f93 100644 --- a/lib/src/cli/tool_handlers/bf_interaction.dart +++ b/lib/src/cli/tool_handlers/bf_interaction.dart @@ -211,7 +211,42 @@ extension _BfInteraction on FlutterMcpServer { .callMethod('get_checkbox_state', {'key': args['key']}); } final fc = _asFlutterClient(client!, 'get_checkbox_state'); - return await fc.getCheckboxState(args['key']); + final checked = await fc.getCheckboxState(args['key']); + return checked == null + ? {"success": false, "error": "No Checkbox/Switch with that key"} + : {"success": true, "checked": checked}; + case 'set_checkbox': + if (client is BridgeDriver) { + return await client.callMethod( + 'set_checkbox', {'key': args['key'], 'checked': args['checked']}); + } + return await _asFlutterClient(client!, 'set_checkbox') + .setCheckbox(args['key'] as String, args['checked'] == true); + case 'type_text': + if (client is BridgeDriver) { + return await client.callMethod('type_text', {'text': args['text']}); + } + return await _asFlutterClient(client!, 'type_text') + .typeText(args['text'] as String? ?? ''); + case 'focus': + if (client is BridgeDriver) { + return await client.callMethod('focus', {'key': args['key']}); + } + return await _asFlutterClient(client!, 'focus') + .focus(args['key'] as String? ?? ''); + case 'blur': + if (client is BridgeDriver) { + return await client.callMethod('blur', {'key': args['key']}); + } + return await _asFlutterClient(client!, 'blur') + .blur(key: args['key'] as String?); + case 'hover': + if (client is BridgeDriver) { + return await client.callMethod('hover', + {'key': args['key'], 'text': args['text'], 'ref': args['ref']}); + } + return await _asFlutterClient(client!, 'hover') + .hover(key: args['key'] as String?, text: args['text'] as String?); case 'get_slider_value': if (client is BridgeDriver) { return await client diff --git a/lib/src/cli/tool_handlers/bf_navigation.dart b/lib/src/cli/tool_handlers/bf_navigation.dart index c60bc40c..b0df80c2 100644 --- a/lib/src/cli/tool_handlers/bf_navigation.dart +++ b/lib/src/cli/tool_handlers/bf_navigation.dart @@ -10,7 +10,8 @@ extension _BfNavigation on FlutterMcpServer { return {"route": route}; } final fc = _asFlutterClient(client!, 'get_current_route'); - return await fc.getCurrentRoute(); + // Always a map: a bare null would read as "tool not handled". + return await fc.getCurrentRouteInfo(); case 'go_back': final count = (args['count'] as num?)?.toInt() ?? 1; var steps = 0; diff --git a/lib/src/cli/tool_handlers/bridge_flutter_handlers.dart b/lib/src/cli/tool_handlers/bridge_flutter_handlers.dart index 9ef68abc..400a231e 100644 --- a/lib/src/cli/tool_handlers/bridge_flutter_handlers.dart +++ b/lib/src/cli/tool_handlers/bridge_flutter_handlers.dart @@ -29,6 +29,7 @@ extension _BridgeFlutterHandlers on FlutterMcpServer { return await (_client as BridgeDriver).callTool(toolName, toolParams); } - throw Exception("Unknown tool: $name"); + throw Exception("Tool '$name' is not implemented for the " + "${client.runtimeType} driver (or returned no result)"); } } diff --git a/lib/src/cli/tool_handlers/flutter_helpers.dart b/lib/src/cli/tool_handlers/flutter_helpers.dart index 8bdea5e4..b112d032 100644 --- a/lib/src/cli/tool_handlers/flutter_helpers.dart +++ b/lib/src/cli/tool_handlers/flutter_helpers.dart @@ -250,6 +250,9 @@ extension _FlutterHelpers on FlutterMcpServer { ); if (found) { + // Being built (cache extent) is not being visible: bring it into + // the viewport before reporting success. + await client.scrollTo(key: key, text: text); return { "success": true, "found": true, @@ -257,14 +260,21 @@ extension _FlutterHelpers on FlutterMcpServer { }; } - // Scroll + // `direction` is the scroll direction ("down" reveals content further + // down); a swipe is the finger's motion, which is the opposite way. + const fingerFor = { + 'down': 'up', + 'up': 'down', + 'right': 'left', + 'left': 'right', + }; await client.swipe( - direction: direction, + direction: fingerFor[direction] ?? direction, distance: 300, key: scrollableKey, ); - // Wait for scroll animation + // Let the fling settle before looking again await Future.delayed(const Duration(milliseconds: 300)); } diff --git a/lib/src/drivers/flutter_driver.dart b/lib/src/drivers/flutter_driver.dart index 349da5ba..59bc174c 100644 --- a/lib/src/drivers/flutter_driver.dart +++ b/lib/src/drivers/flutter_driver.dart @@ -420,10 +420,31 @@ Auto-setup: run diagnose_project() to fix automatically. // ==================== NAVIGATION ==================== Future getCurrentRoute() async { - final result = await _call('ext.flutter.flutter_skill.getCurrentRoute'); - return result['route']; + return (await getCurrentRouteInfo())['route'] as String?; } + /// `route` (name of the topmost current route) and `routes` (full stack). + Future> getCurrentRouteInfo() async { + return await _call('ext.flutter.flutter_skill.getCurrentRoute'); + } + + Future> typeText(String text) async => + await _call('ext.flutter.flutter_skill.typeText', {'text': text}); + + Future> focus(String key) async => + await _call('ext.flutter.flutter_skill.focus', {'key': key}); + + Future> blur({String? key}) async => await _call( + 'ext.flutter.flutter_skill.blur', {if (key != null) 'key': key}); + + Future> setCheckbox(String key, bool checked) async => + await _call('ext.flutter.flutter_skill.setCheckbox', + {'key': key, 'checked': checked.toString()}); + + Future> hover({String? key, String? text}) async => + await _call('ext.flutter.flutter_skill.hover', + {if (key != null) 'key': key, if (text != null) 'text': text}); + Future goBack() async { final result = await _call('ext.flutter.flutter_skill.goBack'); return result['success'] == true; From 6d137eb38528c833f2796c38843111f499192d8d Mon Sep 17 00:00:00 2001 From: cornebidouil Date: Mon, 17 Aug 2026 13:20:31 +0200 Subject: [PATCH 4/4] fix(press_key): pass modifiers through the server; no character while a modifier is held MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tool handler dropped 'modifiers', so Ctrl+A arrived as a plain 'a' (and was inserted as text). Modifiers now reach the SDK, and while one is held the key event carries no character, as real platforms do — Ctrl+A selects all, and type_text then replaces the selection. --- lib/flutter_skill.dart | 4 +++- lib/src/cli/tool_handlers/bf_interaction.dart | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/flutter_skill.dart b/lib/flutter_skill.dart index 49d50d2d..69264625 100644 --- a/lib/flutter_skill.dart +++ b/lib/flutter_skill.dart @@ -1838,10 +1838,12 @@ class FlutterSkillBinding { held.add(pair); } final physical = _physicalFor(logical); + // Platforms report no character while a modifier is held (Ctrl+A is a + // shortcut, not text input). final handled = _deliverKey(KeyDownEvent( physicalKey: physical, logicalKey: logical, - character: character, + character: held.isEmpty ? character : null, timeStamp: now(), )); await Future.delayed(const Duration(milliseconds: 40)); diff --git a/lib/src/cli/tool_handlers/bf_interaction.dart b/lib/src/cli/tool_handlers/bf_interaction.dart index 95d07f93..3b756edc 100644 --- a/lib/src/cli/tool_handlers/bf_interaction.dart +++ b/lib/src/cli/tool_handlers/bf_interaction.dart @@ -292,7 +292,9 @@ extension _BfInteraction on FlutterMcpServer { } // Flutter VM Service — use the registered pressKey extension if (client is FlutterSkillClient) { - final result = await client.pressKey(key); + final modifiers = + (args['modifiers'] as List?)?.map((m) => m.toString()).toList(); + final result = await client.pressKey(key, modifiers: modifiers); if (result['success'] == true) { return {"success": true, "message": "Key pressed: $key"}; }