diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 836970d..961a1e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -180,7 +180,7 @@ jobs: } ``` - See [README](https://github.com/shanselman/FlaUI-MCP#readme) for full documentation. + See [README](https://github.com/TabularEditor/FlaUI-MCP#readme) for full documentation. files: artifacts/*.zip draft: ${{ steps.release.outputs.draft }} prerelease: ${{ steps.release.outputs.prerelease }} diff --git a/.gitignore b/.gitignore index 4e2c401..6d892d6 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ obj/ # Build results [Dd]ebug/ [Rr]elease/ +publish/ x64/ x86/ build/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 97aee43..806cd66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- The server now keeps the display awake while tools are actively being called, so Windows does not turn off the screen or show the lock screen in the middle of a long automation run. Implemented with a Windows power availability request (`PowerCreateRequest`/`PowerSetRequest` with `PowerRequestDisplayRequired` + `PowerRequestSystemRequired`) — the same mechanism video players and conferencing apps use, visible in `powercfg /requests`. The request is released after 5 minutes without a tool call; configure the idle period (or disable with `0`) via the `FLAUI_MCP_KEEP_AWAKE_SECONDS` environment variable. + +### Fixed +- `windows_click` no longer hangs (and then times out) when the clicked element's handler opens a modal dialog. UIA pattern calls (Invoke/Toggle/Select) are synchronous cross-process calls: a WinForms/DevExpress handler that calls `ShowDialog()` does not return until the dialog closes, blocking the target app's entire UIA provider. The click now runs on a background thread while non-blocking Win32 APIs watch for the modal signature (new top-level window, or owner window disabled) and returns immediately with the dialog's title and interaction guidance. +- While an app's UIA provider is blocked by such a pending call, `windows_snapshot`, `windows_get_text`, `windows_click`, `windows_type`, `windows_fill`, `windows_send_keys` (ref-based) and `windows_batch` actions targeting that app now fail fast with guidance (use `windows_screenshot` / `windows_send_keys` without ref) instead of hanging until the 30s global timeout. +- `windows_list_windows` now enumerates windows via Win32 instead of walking the UIA desktop tree, so it keeps working even while some app's UIA provider is blocked. Window handles are also stable across calls now (previously every call registered new handles for the same windows). +- `windows_focus` and `windows_close` (by handle) now use Win32 (`SetForegroundWindow` / `WM_CLOSE`) and work while a provider is blocked. +- `windows_screenshot` with a window handle falls back to a Win32 window-bounds capture while the app's provider is blocked. + ## [0.2.0] - 2026-07-08 ### Fixed diff --git a/README.md b/README.md index 2adf296..678709e 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,27 @@ It can also save screenshots with `savePath`, which must be an absolute local Tool calls have a 30-second timeout so a blocked UI Automation provider or modal dialog returns an actionable error instead of hanging the MCP server forever. +### Keeping the Screen Awake + +Long automation runs generate no keyboard or mouse input, so Windows would +normally turn off the display and show the lock screen mid-run — which breaks +screenshots and can freeze rendering. While tools are actively being called, +FlaUI-MCP holds a Windows *power availability request* (the same signal video +players and conferencing apps send) that keeps the display on and suppresses +the idle lock. The request appears in `powercfg /requests` (run as admin) with +the reason "FlaUI-MCP is driving Windows UI automation". + +The request is released after **5 minutes** without a tool call, so an idle MCP +server does not keep your screen on. Configure via the +`FLAUI_MCP_KEEP_AWAKE_SECONDS` environment variable: a positive value changes +the idle period, `0` disables keep-awake entirely. + +Note: this covers the common idle-lock paths (display timeout, screensaver, +sleep). A domain group policy that enforces a hard machine inactivity limit +("Interactive logon: Machine inactivity limit") locks based on input idle time +and is not suppressed by availability requests — no application can override +that policy. + ### Tool Examples Send a keyboard chord to a target element: @@ -215,6 +236,18 @@ This comes from **Windows UI Automation** - the same API screen readers use. Eac FlaUI-MCP uses accessibility because it's what screen readers use - it's designed for programmatic UI interaction. +### Modal Dialogs + +UI Automation pattern calls (Invoke, Toggle, Select) are synchronous cross-process calls. When a click handler opens a **modal dialog** (`ShowDialog()` in WinForms/WPF), the handler — and therefore the pattern call and the app's entire UIA provider — stays blocked until the dialog closes. + +FlaUI-MCP handles this instead of hanging: + +- `windows_click` runs the pattern call on a background thread and watches the app's top-level windows via non-blocking Win32 APIs. If a modal appears, the tool returns immediately with the dialog's title. +- While the call is pending, UIA-based tools targeting that app (`windows_snapshot`, `windows_get_text`, ref-based typing/clicking) **fail fast** with guidance instead of timing out. +- Tools that don't need UIA keep working throughout: `windows_screenshot`, `windows_send_keys` / `windows_type` *without a ref* (pure keyboard input), `windows_list_windows`, `windows_focus`, and `windows_close`. + +Typical flow: click a button → "modal dialog opened" → screenshot to see it → send keys (e.g. `Enter` or `Tab`+`Enter`) to dismiss it → snapshot works again. + ## Building from Source ```powershell diff --git a/src/FlaUI.Mcp/Core/ElementRegistry.cs b/src/FlaUI.Mcp/Core/ElementRegistry.cs index a96a0ef..f86b9ef 100644 --- a/src/FlaUI.Mcp/Core/ElementRegistry.cs +++ b/src/FlaUI.Mcp/Core/ElementRegistry.cs @@ -10,6 +10,7 @@ public class ElementRegistry { private readonly Dictionary _elements = new(); private readonly Dictionary _windowCounters = new(); + private readonly Dictionary _windowProcessIds = new(); /// /// Clear all elements for a window (called before new snapshot) @@ -55,4 +56,26 @@ public bool HasElement(string refId) { return _elements.ContainsKey(refId); } + + /// + /// Record the process id owning a window's elements. Called during snapshot + /// building (when the provider is known to be responsive) so tools can later + /// check for a blocked provider without touching UI Automation. + /// + public void SetWindowProcessId(string windowHandle, int processId) + { + _windowProcessIds[windowHandle] = processId; + } + + /// + /// Get the process id for an element ref (e.g. "w1e5" -> pid of window "w1"). + /// Returns 0 if unknown. + /// + public int GetProcessIdForRef(string refId) + { + var separator = refId.LastIndexOf('e'); + if (separator <= 0) return 0; + var windowHandle = refId[..separator]; + return _windowProcessIds.TryGetValue(windowHandle, out var pid) ? pid : 0; + } } diff --git a/src/FlaUI.Mcp/Core/KeepAwake.cs b/src/FlaUI.Mcp/Core/KeepAwake.cs new file mode 100644 index 0000000..9782f1a --- /dev/null +++ b/src/FlaUI.Mcp/Core/KeepAwake.cs @@ -0,0 +1,234 @@ +using System.Runtime.InteropServices; + +namespace PlaywrightWindows.Mcp.Core; + +/// +/// Holds a resource (typically a Windows power availability request) while tool +/// calls are actively arriving, releasing it after a sliding idle period. +/// +/// acquires the resource on first call and extends the hold +/// on every subsequent call; when no poke arrives for the hold duration, the +/// resource is released so the machine returns to its normal power/lock policy. +/// +public sealed class KeepAwake : IDisposable +{ + private readonly object _lock = new(); + private readonly TimeSpan _holdDuration; + private readonly Action _acquire; + private readonly Action _release; + private readonly IDisposable? _ownedResource; + private readonly System.Threading.Timer _timer; + private long _deadlineTicks; + private bool _active; + private bool _disposed; + + public KeepAwake(TimeSpan holdDuration, Action acquire, Action release, IDisposable? ownedResource = null) + { + if (holdDuration <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(holdDuration), "Hold duration must be greater than zero."); + } + + _holdDuration = holdDuration; + _acquire = acquire; + _release = release; + _ownedResource = ownedResource; + _timer = new System.Threading.Timer(_ => OnTimerFired(), null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + } + + /// + /// Create a KeepAwake that tells Windows the display is in use (the same + /// signal video players and conferencing apps send), preventing display + /// timeout, sleep and the idle-triggered lock screen while tools are active. + /// Returns null if the power request could not be created. + /// + public static KeepAwake? CreateDisplayKeepAwake(TimeSpan holdDuration, string reason) + { + var request = PowerAvailabilityRequest.Create(reason); + if (request == null) + { + return null; + } + return new KeepAwake(holdDuration, request.Set, request.Clear, request); + } + + public bool IsActive + { + get + { + lock (_lock) + { + return _active; + } + } + } + + /// + /// Signal activity: acquire the resource if not already held, and extend + /// the hold so it is released only after the idle period elapses. + /// + public void Poke() + { + lock (_lock) + { + if (_disposed) + { + return; + } + + if (!_active) + { + _acquire(); + _active = true; + } + + _deadlineTicks = Environment.TickCount64 + (long)_holdDuration.TotalMilliseconds; + _timer.Change(_holdDuration, Timeout.InfiniteTimeSpan); + } + } + + private void OnTimerFired() + { + lock (_lock) + { + if (_disposed || !_active) + { + return; + } + + // A Poke may have raced with this callback; only release once the + // most recently extended deadline has actually passed. + var remainingMs = _deadlineTicks - Environment.TickCount64; + if (remainingMs > 0) + { + _timer.Change(TimeSpan.FromMilliseconds(remainingMs), Timeout.InfiniteTimeSpan); + return; + } + + _release(); + _active = false; + } + } + + public void Dispose() + { + lock (_lock) + { + if (_disposed) + { + return; + } + _disposed = true; + _timer.Dispose(); + if (_active) + { + try { _release(); } catch { } + _active = false; + } + _ownedResource?.Dispose(); + } + } +} + +/// +/// Wraps a Windows power availability request (PowerCreateRequest) that, while +/// set, tells the OS the display is required. This suppresses display timeout, +/// automatic sleep and the inactivity lock screen - the same mechanism browsers +/// use during video playback. The request is visible in `powercfg /requests` +/// together with the reason string. +/// +public sealed class PowerAvailabilityRequest : IDisposable +{ + private readonly nint _handle; + private bool _disposed; + + private PowerAvailabilityRequest(nint handle) + { + _handle = handle; + } + + /// + /// Create a power request with the given diagnostic reason, or null on failure. + /// + public static PowerAvailabilityRequest? Create(string reason) + { + var reasonPtr = Marshal.StringToHGlobalUni(reason); + try + { + var context = new REASON_CONTEXT + { + Version = POWER_REQUEST_CONTEXT_VERSION, + Flags = POWER_REQUEST_CONTEXT_SIMPLE_STRING, + SimpleReasonString = reasonPtr + }; + + var handle = PowerCreateRequest(ref context); + if (handle == 0 || handle == INVALID_HANDLE_VALUE) + { + return null; + } + return new PowerAvailabilityRequest(handle); + } + finally + { + Marshal.FreeHGlobal(reasonPtr); + } + } + + /// Activate the request: display must stay on, system must stay awake. + public void Set() + { + PowerSetRequest(_handle, POWER_REQUEST_TYPE.PowerRequestDisplayRequired); + PowerSetRequest(_handle, POWER_REQUEST_TYPE.PowerRequestSystemRequired); + } + + /// Deactivate the request, restoring normal power/lock policy. + public void Clear() + { + PowerClearRequest(_handle, POWER_REQUEST_TYPE.PowerRequestDisplayRequired); + PowerClearRequest(_handle, POWER_REQUEST_TYPE.PowerRequestSystemRequired); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + _disposed = true; + try { Clear(); } catch { } + CloseHandle(_handle); + } + + private const uint POWER_REQUEST_CONTEXT_VERSION = 0; + private const uint POWER_REQUEST_CONTEXT_SIMPLE_STRING = 0x1; + private static readonly nint INVALID_HANDLE_VALUE = -1; + + private enum POWER_REQUEST_TYPE + { + PowerRequestDisplayRequired = 0, + PowerRequestSystemRequired = 1, + PowerRequestAwayModeRequired = 2, + PowerRequestExecutionRequired = 3 + } + + [StructLayout(LayoutKind.Sequential)] + private struct REASON_CONTEXT + { + public uint Version; + public uint Flags; + public nint SimpleReasonString; + } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern nint PowerCreateRequest(ref REASON_CONTEXT context); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool PowerSetRequest(nint powerRequest, POWER_REQUEST_TYPE requestType); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool PowerClearRequest(nint powerRequest, POWER_REQUEST_TYPE requestType); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool CloseHandle(nint handle); +} diff --git a/src/FlaUI.Mcp/Core/ModalAwareInvoker.cs b/src/FlaUI.Mcp/Core/ModalAwareInvoker.cs new file mode 100644 index 0000000..72635bc --- /dev/null +++ b/src/FlaUI.Mcp/Core/ModalAwareInvoker.cs @@ -0,0 +1,152 @@ +using System.Runtime.ExceptionServices; + +namespace PlaywrightWindows.Mcp.Core; + +/// +/// Outcome of a modal-aware pattern call. +/// +public enum PatternCallOutcome +{ + /// The pattern call returned within the grace period. + Completed, + + /// A modal dialog opened; the pattern call is still executing in the background. + ModalDetected, + + /// The pattern call did not return within the grace period and no modal was detected. + StillPending +} + +/// +/// Result of a modal-aware pattern call. +/// +/// What happened within the grace period. +/// Title of the detected modal window, if any. +public sealed record PatternCallResult(PatternCallOutcome Outcome, string? ModalTitle = null); + +/// +/// Executes UI Automation pattern calls (Invoke, Toggle, Select, ...) so that a +/// handler which opens a modal dialog does not hang the calling tool. +/// +/// Background: UIA pattern calls are synchronous cross-process calls. In WinForms +/// and many other frameworks, a button handler that calls ShowDialog() will not +/// return until the dialog closes — so the pattern call, and with it the entire +/// UIA provider of the target process, stays blocked for as long as the dialog +/// is open. This class runs the call on a background thread and watches the +/// target process's top-level windows via non-blocking Win32 APIs. When a new +/// window appears (or an existing window is disabled by a modal), it reports +/// ModalDetected right away so the caller can interact with the dialog instead +/// of timing out. +/// +public static class ModalAwareInvoker +{ + /// How long to wait for the pattern call before giving up on completion. + public static readonly TimeSpan DefaultGracePeriod = TimeSpan.FromSeconds(2); + + private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(50); + + /// + /// Execute a pattern call with modal detection. + /// + /// Target process id (0 if unknown; disables modal detection). + /// Human-readable description, e.g. "Invoke on 'OK'". + /// The synchronous pattern call to execute. + /// Tracker that other tools consult to fail fast while the call is pending. + /// Max time to wait for completion; defaults to . + /// Override for window enumeration (for testing). + /// Rethrows any exception from when it completes within the grace period. + public static PatternCallResult Execute( + int processId, + string description, + Action patternCall, + PendingInvokeTracker tracker, + TimeSpan? gracePeriod = null, + Func>? windowEnumerator = null) + { + var grace = gracePeriod ?? DefaultGracePeriod; + windowEnumerator ??= pid => Win32Desktop.GetTopLevelWindows(pid); + + var windowsBefore = processId != 0 + ? windowEnumerator(processId).ToDictionary(w => w.Hwnd) + : new Dictionary(); + + var info = tracker.Begin(processId, description); + var task = Task.Run(patternCall); + + // Ensure the tracker is cleared whenever the call eventually returns, + // even if we stop waiting for it below. Also observe any exception so + // it does not surface as an unobserved task exception. + _ = task.ContinueWith(t => + { + _ = t.Exception; + tracker.Complete(info); + }, TaskScheduler.Default); + + var deadline = DateTime.UtcNow + grace; + while (true) + { + // WaitAny (unlike Task.Wait) does not throw for faulted tasks, + // letting us rethrow the original exception un-wrapped below + if (Task.WaitAny(new[] { task }, PollInterval) == 0) + { + if (task.IsFaulted && task.Exception != null) + { + var inner = task.Exception.InnerException ?? task.Exception; + ExceptionDispatchInfo.Capture(inner).Throw(); + } + return new PatternCallResult(PatternCallOutcome.Completed); + } + + if (processId != 0) + { + var modalTitle = DetectModal(windowsBefore, windowEnumerator(processId)); + if (modalTitle != null) + { + info.ModalTitle = modalTitle; + return new PatternCallResult(PatternCallOutcome.ModalDetected, modalTitle); + } + } + + if (DateTime.UtcNow >= deadline) + { + return new PatternCallResult(PatternCallOutcome.StillPending); + } + } + } + + /// + /// Detect the modal signature: a new top-level window appeared, or a + /// previously enabled window became disabled (modal dialogs disable their owner). + /// Returns the best available title for the modal, or null if no modal is detected. + /// + private static string? DetectModal( + IReadOnlyDictionary before, + IReadOnlyList current) + { + Win32WindowInfo? newWindow = null; + var ownerDisabled = false; + + foreach (var window in current) + { + if (!before.TryGetValue(window.Hwnd, out var previous)) + { + // Prefer a titled new window over an untitled one + if (newWindow == null || (newWindow.Title.Length == 0 && window.Title.Length > 0)) + { + newWindow = window; + } + } + else if (previous.IsEnabled && !window.IsEnabled) + { + ownerDisabled = true; + } + } + + if (newWindow != null) + { + return newWindow.Title.Length > 0 ? newWindow.Title : "(untitled window)"; + } + + return ownerDisabled ? "(unknown title)" : null; + } +} diff --git a/src/FlaUI.Mcp/Core/PendingInvokeTracker.cs b/src/FlaUI.Mcp/Core/PendingInvokeTracker.cs new file mode 100644 index 0000000..530b586 --- /dev/null +++ b/src/FlaUI.Mcp/Core/PendingInvokeTracker.cs @@ -0,0 +1,92 @@ +namespace PlaywrightWindows.Mcp.Core; + +/// +/// Information about a UI Automation pattern call that has not returned yet. +/// While such a call is in flight, the target process's UIA provider typically +/// cannot answer any other request, so tools should fail fast instead of hanging. +/// +public sealed class PendingInvokeInfo +{ + internal PendingInvokeInfo(int processId, string description) + { + ProcessId = processId; + Description = description; + StartedUtc = DateTime.UtcNow; + } + + /// Process id whose UIA provider is occupied by this call. + public int ProcessId { get; } + + /// Human-readable description of the call, e.g. "Invoke on 'Open...'". + public string Description { get; } + + /// When the call started. + public DateTime StartedUtc { get; } + + /// Title of the modal window detected after the call started, if any. + public string? ModalTitle { get; set; } +} + +/// +/// Tracks UI Automation pattern calls that are still executing per process, so +/// other tools can detect a blocked provider and return a helpful error +/// immediately instead of waiting for the global tool timeout. +/// +public class PendingInvokeTracker +{ + private readonly object _lock = new(); + private readonly List _pending = new(); + + /// + /// Record the start of a pattern call against a process. + /// + public PendingInvokeInfo Begin(int processId, string description) + { + var info = new PendingInvokeInfo(processId, description); + lock (_lock) + { + _pending.Add(info); + } + return info; + } + + /// + /// Record that a pattern call has returned (successfully or not). + /// + public void Complete(PendingInvokeInfo info) + { + lock (_lock) + { + _pending.Remove(info); + } + } + + /// + /// Check whether a process has a pattern call still in flight. + /// A processId of 0 (unknown) never matches. + /// + public bool TryGetPending(int processId, out PendingInvokeInfo info) + { + lock (_lock) + { + var match = processId != 0 ? _pending.FirstOrDefault(p => p.ProcessId == processId) : null; + info = match!; + return match != null; + } + } + + /// + /// Build the standard guidance message for a blocked provider. + /// + public static string DescribeBlocked(PendingInvokeInfo info) + { + var modalPart = info.ModalTitle != null + ? $" — it opened a modal dialog \"{info.ModalTitle}\" that is waiting for input" + : ""; + var elapsed = (int)(DateTime.UtcNow - info.StartedUtc).TotalSeconds; + return $"UI Automation for this app is blocked by a pending '{info.Description}' call " + + $"started {elapsed}s ago{modalPart}. UIA-based tools will hang until it completes. " + + "Interact with the dialog using windows_screenshot (fullScreen: true) to see it, " + + "windows_send_keys (without ref) for keyboard input, or dismiss it; then retry."; + } +} diff --git a/src/FlaUI.Mcp/Core/SessionManager.cs b/src/FlaUI.Mcp/Core/SessionManager.cs index 5bd21f6..989ac4d 100644 --- a/src/FlaUI.Mcp/Core/SessionManager.cs +++ b/src/FlaUI.Mcp/Core/SessionManager.cs @@ -13,6 +13,9 @@ public class SessionManager : IDisposable private readonly UIA3Automation _automation; private readonly Dictionary _applications = new(); private readonly Dictionary _windows = new(); + private readonly Dictionary _windowHwnds = new(); + private readonly Dictionary _windowPids = new(); + private readonly Dictionary _hwndToHandle = new(); private int _windowCounter = 0; public SessionManager() @@ -31,33 +34,33 @@ public SessionManager() Arguments = args != null ? string.Join(" ", args) : "", UseShellExecute = true }; - + var process = System.Diagnostics.Process.Start(psi); if (process == null) { throw new Exception($"Failed to start process: {appPath}"); } - + // Wait for the process to be ready try { process.WaitForInputIdle(5000); } catch { /* Some processes don't support this */ } - + Thread.Sleep(1000); // Extra wait for window to appear - + // Find window by process ID from desktop var desktop = _automation.GetDesktop(); Window? window = null; - + // Try to find by process ID first var element = desktop.FindFirstDescendant(cf => cf.ByProcessId(process.Id)); if (element != null) { window = element.AsWindow(); } - + // If not found, the app might have spawned a different process (common for UWP) // Search by waiting for a new window if (window == null) @@ -66,7 +69,7 @@ public SessionManager() var existingTitles = new HashSet( _windows.Values.Select(w => w.Title).Where(t => !string.IsNullOrEmpty(t)) ); - + // Wait and look for new windows for (int i = 0; i < 10 && window == null; i++) { @@ -89,7 +92,7 @@ public SessionManager() } } } - + if (window == null) { throw new Exception($"Could not find window for {appPath}. Try using windows_list_windows and windows_focus instead."); @@ -103,7 +106,7 @@ public SessionManager() { var desktop = _automation.GetDesktop(); var window = desktop.FindFirstDescendant(cf => cf.ByName(title))?.AsWindow(); - + if (window == null) { throw new Exception($"Window not found: {title}"); @@ -115,45 +118,135 @@ public SessionManager() public string RegisterWindow(Window window) { + // Capture the native handle and process id while the provider is + // responsive, so later operations (focus, close, blocked-provider + // checks) can work without any UI Automation round-trips. + nint hwnd = 0; + var pid = 0; + try + { + hwnd = window.Properties.NativeWindowHandle.ValueOrDefault; + pid = window.Properties.ProcessId.ValueOrDefault; + } + catch { /* best effort */ } + + if (hwnd != 0 && _hwndToHandle.TryGetValue(hwnd, out var existing)) + { + _windows[existing] = window; + return existing; + } + var handle = $"w{++_windowCounter}"; _windows[handle] = window; + if (hwnd != 0) + { + _windowHwnds[handle] = hwnd; + _hwndToHandle[hwnd] = handle; + } + if (pid != 0) + { + _windowPids[handle] = pid; + } + return handle; + } + + /// + /// Register a window by its native handle only, without touching UI Automation. + /// The UIA Window object is created lazily on first use in . + /// + public string RegisterNativeWindow(nint hwnd, int processId) + { + if (_hwndToHandle.TryGetValue(hwnd, out var existing)) + { + _windowPids[existing] = processId; + return existing; + } + + var handle = $"w{++_windowCounter}"; + _windowHwnds[handle] = hwnd; + _hwndToHandle[hwnd] = handle; + _windowPids[handle] = processId; return handle; } public Window? GetWindow(string handle) { - return _windows.TryGetValue(handle, out var window) ? window : null; + if (_windows.TryGetValue(handle, out var window)) + { + return window; + } + + // Lazily attach to windows registered via RegisterNativeWindow + if (_windowHwnds.TryGetValue(handle, out var hwnd)) + { + var attached = _automation.FromHandle(hwnd)?.AsWindow(); + if (attached != null) + { + _windows[handle] = attached; + return attached; + } + } + + return null; + } + + /// + /// Get the cached process id for a window handle, or 0 if unknown. + /// Never touches UI Automation. + /// + public int GetWindowProcessId(string handle) + { + return _windowPids.TryGetValue(handle, out var pid) ? pid : 0; } + /// + /// Get the cached native window handle for a window handle, or 0 if unknown. + /// Never touches UI Automation. + /// + public nint GetWindowHwnd(string handle) + { + return _windowHwnds.TryGetValue(handle, out var hwnd) ? hwnd : 0; + } + + /// + /// List top-level windows using Win32 enumeration only. This never blocks, + /// even when an app's UI Automation provider is busy (e.g., held up by a + /// modal dialog opened from a pending Invoke call). + /// public List<(string handle, string title, string? processName)> ListWindows() { - var desktop = _automation.GetDesktop(); - var windows = desktop.FindAllChildren(cf => cf.ByControlType(FlaUI.Core.Definitions.ControlType.Window)); - var result = new List<(string, string, string?)>(); - foreach (var w in windows) + foreach (var info in Win32Desktop.GetTopLevelWindows()) { - var window = w.AsWindow(); - if (window != null && !string.IsNullOrEmpty(window.Title)) + if (info.Title.Length == 0 || info.IsToolWindow || info.IsCloaked) { - var handle = RegisterWindow(window); - string? processName = null; - try - { - processName = window.Properties.ProcessId.TryGetValue(out var pid) - ? System.Diagnostics.Process.GetProcessById(pid).ProcessName - : null; - } - catch { } - - result.Add((handle, window.Title, processName)); + continue; } + + var handle = RegisterNativeWindow(info.Hwnd, info.ProcessId); + + string? processName = null; + try + { + processName = System.Diagnostics.Process.GetProcessById(info.ProcessId).ProcessName; + } + catch { } + + result.Add((handle, info.Title, processName)); } return result; } public void FocusWindow(string handle) { + // Prefer Win32 focus (never blocks); fall back to UIA for windows + // registered before a native handle was captured. + if (_windowHwnds.TryGetValue(handle, out var hwnd)) + { + Win32Desktop.FocusWindow(hwnd); + return; + } + var window = GetWindow(handle); if (window == null) { @@ -164,13 +257,28 @@ public void FocusWindow(string handle) public void CloseWindow(string handle) { - var window = GetWindow(handle); - if (window == null) + // Prefer a Win32 WM_CLOSE (never blocks); fall back to UIA. + if (_windowHwnds.TryGetValue(handle, out var hwnd)) { - throw new Exception($"Window not found: {handle}"); + Win32Desktop.CloseWindow(hwnd); } - window.Close(); + else + { + var window = GetWindow(handle); + if (window == null) + { + throw new Exception($"Window not found: {handle}"); + } + window.Close(); + } + _windows.Remove(handle); + if (_windowHwnds.TryGetValue(handle, out var removedHwnd)) + { + _hwndToHandle.Remove(removedHwnd); + _windowHwnds.Remove(handle); + } + _windowPids.Remove(handle); } public void Dispose() @@ -181,6 +289,9 @@ public void Dispose() } _applications.Clear(); _windows.Clear(); + _windowHwnds.Clear(); + _windowPids.Clear(); + _hwndToHandle.Clear(); _automation.Dispose(); } } diff --git a/src/FlaUI.Mcp/Core/SnapshotBuilder.cs b/src/FlaUI.Mcp/Core/SnapshotBuilder.cs index c95d551..9eb41e8 100644 --- a/src/FlaUI.Mcp/Core/SnapshotBuilder.cs +++ b/src/FlaUI.Mcp/Core/SnapshotBuilder.cs @@ -23,6 +23,21 @@ public string BuildSnapshot(string windowHandle, AutomationElement root) // Clear previous elements for this window _elementRegistry.ClearWindow(windowHandle); + // Remember the owning process so tools can later detect a blocked + // UIA provider for this window's refs without touching UIA. + try + { + var processId = root.Properties.ProcessId.ValueOrDefault; + if (processId != 0) + { + _elementRegistry.SetWindowProcessId(windowHandle, processId); + } + } + catch + { + // Process id is best-effort; snapshot still works without it + } + var sb = new StringBuilder(); BuildElementSnapshot(sb, windowHandle, root, 0); return sb.ToString(); diff --git a/src/FlaUI.Mcp/Core/Win32Desktop.cs b/src/FlaUI.Mcp/Core/Win32Desktop.cs new file mode 100644 index 0000000..0b643b4 --- /dev/null +++ b/src/FlaUI.Mcp/Core/Win32Desktop.cs @@ -0,0 +1,151 @@ +using System.Runtime.InteropServices; +using System.Text; + +namespace PlaywrightWindows.Mcp.Core; + +/// +/// Lightweight information about a top-level Win32 window, gathered without +/// any UI Automation calls. +/// +/// Native window handle. +/// Window title (may be empty). +/// Owning process id. +/// Whether the window accepts input (modal dialogs disable their owner). +/// Whether the window has WS_EX_TOOLWINDOW (excluded from window lists). +/// Whether the window is DWM-cloaked (e.g., suspended UWP apps). +public sealed record Win32WindowInfo( + nint Hwnd, + string Title, + int ProcessId, + bool IsEnabled, + bool IsToolWindow, + bool IsCloaked); + +/// +/// Win32-based desktop window enumeration and manipulation. +/// All APIs used here read cached window metadata and never block on the target +/// process's message loop, so they remain usable while a UI Automation provider +/// is blocked (e.g., by a modal dialog opened from a pending Invoke call). +/// +public static class Win32Desktop +{ + private const int GWL_EXSTYLE = -20; + private const long WS_EX_TOOLWINDOW = 0x00000080; + private const int DWMWA_CLOAKED = 14; + private const int SW_RESTORE = 9; + private const uint WM_CLOSE = 0x0010; + + private delegate bool EnumWindowsProc(nint hWnd, nint lParam); + + [DllImport("user32.dll")] + private static extern bool EnumWindows(EnumWindowsProc lpEnumFunc, nint lParam); + + [DllImport("user32.dll")] + private static extern bool IsWindowVisible(nint hWnd); + + [DllImport("user32.dll")] + private static extern bool IsWindowEnabled(nint hWnd); + + [DllImport("user32.dll")] + private static extern bool IsIconic(nint hWnd); + + [DllImport("user32.dll", CharSet = CharSet.Unicode)] + private static extern int GetWindowText(nint hWnd, StringBuilder lpString, int nMaxCount); + + [DllImport("user32.dll")] + private static extern uint GetWindowThreadProcessId(nint hWnd, out uint lpdwProcessId); + + [DllImport("user32.dll")] + private static extern nint GetWindowLongPtr(nint hWnd, int nIndex); + + [DllImport("user32.dll")] + private static extern bool SetForegroundWindow(nint hWnd); + + [DllImport("user32.dll")] + private static extern bool ShowWindow(nint hWnd, int nCmdShow); + + [DllImport("user32.dll")] + private static extern bool PostMessage(nint hWnd, uint msg, nint wParam, nint lParam); + + [DllImport("user32.dll")] + private static extern bool GetWindowRect(nint hWnd, out RECT lpRect); + + [DllImport("dwmapi.dll")] + private static extern int DwmGetWindowAttribute(nint hWnd, int dwAttribute, out int pvAttribute, int cbAttribute); + + [StructLayout(LayoutKind.Sequential)] + private struct RECT + { + public int Left; + public int Top; + public int Right; + public int Bottom; + } + + /// + /// Enumerate visible top-level windows, optionally restricted to a single process. + /// + /// If set, only windows owned by this process are returned. + public static IReadOnlyList GetTopLevelWindows(int? processId = null) + { + var result = new List(); + EnumWindows((hwnd, _) => + { + if (!IsWindowVisible(hwnd)) return true; + + GetWindowThreadProcessId(hwnd, out var pid); + if (processId.HasValue && pid != (uint)processId.Value) return true; + + var titleBuilder = new StringBuilder(512); + GetWindowText(hwnd, titleBuilder, titleBuilder.Capacity); + + var exStyle = GetWindowLongPtr(hwnd, GWL_EXSTYLE).ToInt64(); + var isToolWindow = (exStyle & WS_EX_TOOLWINDOW) != 0; + + var isCloaked = false; + if (DwmGetWindowAttribute(hwnd, DWMWA_CLOAKED, out var cloaked, sizeof(int)) == 0) + { + isCloaked = cloaked != 0; + } + + result.Add(new Win32WindowInfo( + hwnd, + titleBuilder.ToString(), + (int)pid, + IsWindowEnabled(hwnd), + isToolWindow, + isCloaked)); + return true; + }, 0); + return result; + } + + /// + /// Bring a window to the foreground, restoring it first if minimized. + /// + public static void FocusWindow(nint hwnd) + { + if (IsIconic(hwnd)) + { + ShowWindow(hwnd, SW_RESTORE); + } + SetForegroundWindow(hwnd); + } + + /// + /// Request a window to close by posting WM_CLOSE (non-blocking). + /// + public static void CloseWindow(nint hwnd) + { + PostMessage(hwnd, WM_CLOSE, 0, 0); + } + + /// + /// Get a window's bounding rectangle in screen coordinates, or null on failure. + /// + public static System.Drawing.Rectangle? GetWindowBounds(nint hwnd) + { + if (!GetWindowRect(hwnd, out var rect)) return null; + return System.Drawing.Rectangle.FromLTRB(rect.Left, rect.Top, rect.Right, rect.Bottom); + } +} diff --git a/src/FlaUI.Mcp/Mcp/ToolRegistry.cs b/src/FlaUI.Mcp/Mcp/ToolRegistry.cs index 507f1a5..cb1fd6d 100644 --- a/src/FlaUI.Mcp/Mcp/ToolRegistry.cs +++ b/src/FlaUI.Mcp/Mcp/ToolRegistry.cs @@ -9,14 +9,16 @@ public class ToolRegistry { private readonly Dictionary _tools = new(); private readonly TimeSpan _toolTimeout; + private readonly Action? _onToolActivity; - public ToolRegistry(TimeSpan? toolTimeout = null) + public ToolRegistry(TimeSpan? toolTimeout = null, Action? onToolActivity = null) { _toolTimeout = toolTimeout ?? TimeSpan.FromSeconds(30); if (_toolTimeout <= TimeSpan.Zero) { throw new ArgumentOutOfRangeException(nameof(toolTimeout), "Tool timeout must be greater than zero."); } + _onToolActivity = onToolActivity; } public void RegisterTool(ITool tool) @@ -45,6 +47,8 @@ public async Task ExecuteToolAsync(string name, JsonElement? argu try { + _onToolActivity?.Invoke(); + var toolTask = Task.Run(() => tool.ExecuteAsync(arguments)); var timeoutTask = Task.Delay(_toolTimeout); diff --git a/src/FlaUI.Mcp/Program.cs b/src/FlaUI.Mcp/Program.cs index 27c532e..10662b2 100644 --- a/src/FlaUI.Mcp/Program.cs +++ b/src/FlaUI.Mcp/Program.cs @@ -7,21 +7,38 @@ // Create shared services var sessionManager = new SessionManager(); var elementRegistry = new ElementRegistry(); +var invokeTracker = new PendingInvokeTracker(); + +// While tools are actively being called, hold a Windows power availability +// request (display required) so the screen does not turn off and the lock +// screen does not interrupt a long automation run. Released after a sliding +// idle period. Set FLAUI_MCP_KEEP_AWAKE_SECONDS to change the idle period, +// or to 0 to disable. +var keepAwakeSeconds = 300; +if (int.TryParse(Environment.GetEnvironmentVariable("FLAUI_MCP_KEEP_AWAKE_SECONDS"), out var configuredSeconds)) +{ + keepAwakeSeconds = configuredSeconds; +} +using var keepAwake = keepAwakeSeconds > 0 + ? KeepAwake.CreateDisplayKeepAwake( + TimeSpan.FromSeconds(keepAwakeSeconds), + "FlaUI-MCP is driving Windows UI automation") + : null; // Register all tools -var toolRegistry = new ToolRegistry(); +var toolRegistry = new ToolRegistry(onToolActivity: keepAwake != null ? keepAwake.Poke : null); toolRegistry.RegisterTool(new LaunchTool(sessionManager)); -toolRegistry.RegisterTool(new SnapshotTool(sessionManager, elementRegistry)); -toolRegistry.RegisterTool(new ClickTool(elementRegistry)); -toolRegistry.RegisterTool(new TypeTool(elementRegistry)); -toolRegistry.RegisterTool(new FillTool(elementRegistry)); -toolRegistry.RegisterTool(new GetTextTool(elementRegistry)); -toolRegistry.RegisterTool(new SendKeysTool(elementRegistry)); -toolRegistry.RegisterTool(new ScreenshotTool(sessionManager, elementRegistry)); +toolRegistry.RegisterTool(new SnapshotTool(sessionManager, elementRegistry, invokeTracker)); +toolRegistry.RegisterTool(new ClickTool(elementRegistry, invokeTracker)); +toolRegistry.RegisterTool(new TypeTool(elementRegistry, invokeTracker)); +toolRegistry.RegisterTool(new FillTool(elementRegistry, invokeTracker)); +toolRegistry.RegisterTool(new GetTextTool(elementRegistry, invokeTracker)); +toolRegistry.RegisterTool(new SendKeysTool(elementRegistry, invokeTracker)); +toolRegistry.RegisterTool(new ScreenshotTool(sessionManager, elementRegistry, invokeTracker)); toolRegistry.RegisterTool(new ListWindowsTool(sessionManager)); toolRegistry.RegisterTool(new FocusWindowTool(sessionManager)); toolRegistry.RegisterTool(new CloseWindowTool(sessionManager)); -toolRegistry.RegisterTool(new BatchTool(sessionManager, elementRegistry)); +toolRegistry.RegisterTool(new BatchTool(sessionManager, elementRegistry, invokeTracker)); // Create and run MCP server var server = new McpServer(toolRegistry); diff --git a/src/FlaUI.Mcp/Tools/BatchTool.cs b/src/FlaUI.Mcp/Tools/BatchTool.cs index 0260f1b..c95613e 100644 --- a/src/FlaUI.Mcp/Tools/BatchTool.cs +++ b/src/FlaUI.Mcp/Tools/BatchTool.cs @@ -14,12 +14,14 @@ public class BatchTool : ToolBase private readonly SessionManager _sessionManager; private readonly ElementRegistry _elementRegistry; private readonly SnapshotBuilder _snapshotBuilder; + private readonly PendingInvokeTracker _invokeTracker; - public BatchTool(SessionManager sessionManager, ElementRegistry elementRegistry) + public BatchTool(SessionManager sessionManager, ElementRegistry elementRegistry, PendingInvokeTracker? invokeTracker = null) { _sessionManager = sessionManager; _elementRegistry = elementRegistry; _snapshotBuilder = new SnapshotBuilder(elementRegistry); + _invokeTracker = invokeTracker ?? new PendingInvokeTracker(); } public override string Name => "windows_batch"; @@ -146,20 +148,37 @@ private string ExecuteClick(JsonElement action) return $"Element not found: {refId}"; } + // Fail fast if this app's UIA provider is blocked by a pending pattern call + var processId = _elementRegistry.GetProcessIdForRef(refId); + if (_invokeTracker.TryGetPending(processId, out var pending)) + { + return PendingInvokeTracker.DescribeBlocked(pending); + } + var elementName = element.Properties.Name.ValueOrDefault ?? refId; // Try Invoke pattern first if (element.Patterns.Invoke.IsSupported) { - element.Patterns.Invoke.Pattern.Invoke(); - return $"Invoked {elementName}"; + var invokePattern = element.Patterns.Invoke.Pattern; + var result = ModalAwareInvoker.Execute( + processId, + $"Invoke on '{elementName}'", + () => invokePattern.Invoke(), + _invokeTracker); + return DescribePatternResult(result, $"Invoked {elementName}"); } // Try Toggle pattern if (element.Patterns.Toggle.IsSupported) { - element.Patterns.Toggle.Pattern.Toggle(); - return $"Toggled {elementName}"; + var togglePattern = element.Patterns.Toggle.Pattern; + var result = ModalAwareInvoker.Execute( + processId, + $"Toggle on '{elementName}'", + () => togglePattern.Toggle(), + _invokeTracker); + return DescribePatternResult(result, $"Toggled {elementName}"); } // Fall back to mouse click @@ -168,6 +187,18 @@ private string ExecuteClick(JsonElement action) return $"Clicked {elementName}"; } + private static string DescribePatternResult(PatternCallResult result, string completedMessage) + { + return result.Outcome switch + { + PatternCallOutcome.Completed => completedMessage, + PatternCallOutcome.ModalDetected => + $"{completedMessage} — a modal dialog \"{result.ModalTitle}\" opened and is waiting for input. " + + "Remaining batch actions that use UIA on this app may fail until the dialog is dismissed.", + _ => $"{completedMessage} — the app's handler is still running in the background.", + }; + } + private string ExecuteType(JsonElement action) { var text = action.TryGetProperty("text", out var textProp) ? textProp.GetString() : null; @@ -184,6 +215,10 @@ private string ExecuteType(JsonElement action) { return $"Element not found: {refId}"; } + if (_invokeTracker.TryGetPending(_elementRegistry.GetProcessIdForRef(refId), out var pending)) + { + return PendingInvokeTracker.DescribeBlocked(pending); + } element.Focus(); Thread.Sleep(30); } @@ -208,6 +243,11 @@ private string ExecuteFill(JsonElement action) return $"Element not found: {refId}"; } + if (_invokeTracker.TryGetPending(_elementRegistry.GetProcessIdForRef(refId), out var pendingFill)) + { + return PendingInvokeTracker.DescribeBlocked(pendingFill); + } + if (element.Patterns.Value.IsSupported) { element.Patterns.Value.Pattern.SetValue(value); @@ -237,6 +277,11 @@ private string ExecuteSnapshot(JsonElement action) Window? window = null; if (!string.IsNullOrEmpty(handle)) { + if (_invokeTracker.TryGetPending(_sessionManager.GetWindowProcessId(handle), out var pendingSnapshot)) + { + return PendingInvokeTracker.DescribeBlocked(pendingSnapshot); + } + window = _sessionManager.GetWindow(handle); if (window == null) { diff --git a/src/FlaUI.Mcp/Tools/ClickTool.cs b/src/FlaUI.Mcp/Tools/ClickTool.cs index 51a838b..abcb81c 100644 --- a/src/FlaUI.Mcp/Tools/ClickTool.cs +++ b/src/FlaUI.Mcp/Tools/ClickTool.cs @@ -1,5 +1,6 @@ using System.Text.Json; using FlaUI.Core.AutomationElements; +using FlaUI.Core.Definitions; using FlaUI.Core.Input; using PlaywrightWindows.Mcp.Core; @@ -11,17 +12,20 @@ namespace PlaywrightWindows.Mcp.Tools; public class ClickTool : ToolBase { private readonly ElementRegistry _elementRegistry; + private readonly PendingInvokeTracker _invokeTracker; - public ClickTool(ElementRegistry elementRegistry) + public ClickTool(ElementRegistry elementRegistry, PendingInvokeTracker? invokeTracker = null) { _elementRegistry = elementRegistry; + _invokeTracker = invokeTracker ?? new PendingInvokeTracker(); } public override string Name => "windows_click"; - public override string Description => + public override string Description => "Click an element by its ref (from windows_snapshot). Prefers Invoke pattern for reliability, " + - "falls back to mouse click if needed."; + "falls back to mouse click if needed. If the click opens a modal dialog, returns immediately " + + "with the dialog title instead of waiting for the dialog to close."; public override object InputSchema => new { @@ -65,6 +69,13 @@ public override Task ExecuteAsync(JsonElement? arguments) return Task.FromResult(ErrorResult($"Element not found: {refId}. Run windows_snapshot to refresh element refs.")); } + // Fail fast if this app's UIA provider is already blocked by an earlier call + var processId = _elementRegistry.GetProcessIdForRef(refId); + if (_invokeTracker.TryGetPending(processId, out var pending)) + { + return Task.FromResult(ErrorResult(PendingInvokeTracker.DescribeBlocked(pending))); + } + try { var elementName = element.Properties.Name.ValueOrDefault ?? refId; @@ -72,28 +83,47 @@ public override Task ExecuteAsync(JsonElement? arguments) // Try Invoke pattern first (most reliable for buttons) if (button == "left" && !doubleClick && element.Patterns.Invoke.IsSupported) { - element.Patterns.Invoke.Pattern.Invoke(); - return Task.FromResult(TextResult($"Invoked {elementName}")); + var invokePattern = element.Patterns.Invoke.Pattern; + var result = ModalAwareInvoker.Execute( + processId, + $"Invoke on '{elementName}'", + () => invokePattern.Invoke(), + _invokeTracker); + return Task.FromResult(PatternResult(result, $"Invoked {elementName}")); } // Try Toggle pattern for checkboxes if (button == "left" && !doubleClick && element.Patterns.Toggle.IsSupported) { - element.Patterns.Toggle.Pattern.Toggle(); - var newState = element.Patterns.Toggle.Pattern.ToggleState.ValueOrDefault; - return Task.FromResult(TextResult($"Toggled {elementName} to {newState}")); + var togglePattern = element.Patterns.Toggle.Pattern; + ToggleState? newState = null; + var result = ModalAwareInvoker.Execute( + processId, + $"Toggle on '{elementName}'", + () => + { + togglePattern.Toggle(); + newState = togglePattern.ToggleState.ValueOrDefault; + }, + _invokeTracker); + return Task.FromResult(PatternResult(result, $"Toggled {elementName} to {newState}")); } // Try SelectionItem pattern for list items if (button == "left" && !doubleClick && element.Patterns.SelectionItem.IsSupported) { - element.Patterns.SelectionItem.Pattern.Select(); - return Task.FromResult(TextResult($"Selected {elementName}")); + var selectionPattern = element.Patterns.SelectionItem.Pattern; + var result = ModalAwareInvoker.Execute( + processId, + $"Select on '{elementName}'", + () => selectionPattern.Select(), + _invokeTracker); + return Task.FromResult(PatternResult(result, $"Selected {elementName}")); } // Fall back to mouse click var clickPoint = element.GetClickablePoint(); - + var mouseButton = button switch { "right" => MouseButton.Right, @@ -117,4 +147,23 @@ public override Task ExecuteAsync(JsonElement? arguments) return Task.FromResult(ErrorResult($"Failed to click {refId}: {ex.Message}")); } } + + /// + /// Map a modal-aware pattern call result to a tool result. + /// + private static McpToolResult PatternResult(PatternCallResult result, string completedMessage) + { + return result.Outcome switch + { + PatternCallOutcome.Completed => TextResult(completedMessage), + PatternCallOutcome.ModalDetected => TextResult( + $"{completedMessage} — a modal dialog \"{result.ModalTitle}\" opened and is waiting for input. " + + "Note: UIA-based tools (windows_snapshot, windows_get_text) on this app will block until the " + + "dialog closes. Use windows_screenshot to see the dialog and windows_send_keys (without ref) " + + "or coordinate clicks to interact with it."), + _ => TextResult( + $"{completedMessage} — the app's handler is still running in the background. " + + "Take a windows_screenshot to check the app's state; UIA-based tools may block until it completes."), + }; + } } diff --git a/src/FlaUI.Mcp/Tools/GetTextTool.cs b/src/FlaUI.Mcp/Tools/GetTextTool.cs index d362212..b53321c 100644 --- a/src/FlaUI.Mcp/Tools/GetTextTool.cs +++ b/src/FlaUI.Mcp/Tools/GetTextTool.cs @@ -9,10 +9,12 @@ namespace PlaywrightWindows.Mcp.Tools; public class GetTextTool : ToolBase { private readonly ElementRegistry _elementRegistry; + private readonly PendingInvokeTracker _invokeTracker; - public GetTextTool(ElementRegistry elementRegistry) + public GetTextTool(ElementRegistry elementRegistry, PendingInvokeTracker? invokeTracker = null) { _elementRegistry = elementRegistry; + _invokeTracker = invokeTracker ?? new PendingInvokeTracker(); } public override string Name => "windows_get_text"; @@ -49,6 +51,12 @@ public override Task ExecuteAsync(JsonElement? arguments) return Task.FromResult(ErrorResult($"Element not found: {refId}. Run windows_snapshot to refresh element refs.")); } + // Fail fast if this app's UIA provider is blocked by a pending pattern call + if (_invokeTracker.TryGetPending(_elementRegistry.GetProcessIdForRef(refId), out var pending)) + { + return Task.FromResult(ErrorResult(PendingInvokeTracker.DescribeBlocked(pending))); + } + try { string? text = null; diff --git a/src/FlaUI.Mcp/Tools/ScreenshotTool.cs b/src/FlaUI.Mcp/Tools/ScreenshotTool.cs index ef882db..5a797d9 100644 --- a/src/FlaUI.Mcp/Tools/ScreenshotTool.cs +++ b/src/FlaUI.Mcp/Tools/ScreenshotTool.cs @@ -11,11 +11,13 @@ public class ScreenshotTool : ToolBase { private readonly SessionManager _sessionManager; private readonly ElementRegistry _elementRegistry; + private readonly PendingInvokeTracker _invokeTracker; - public ScreenshotTool(SessionManager sessionManager, ElementRegistry elementRegistry) + public ScreenshotTool(SessionManager sessionManager, ElementRegistry elementRegistry, PendingInvokeTracker? invokeTracker = null) { _sessionManager = sessionManager; _elementRegistry = elementRegistry; + _invokeTracker = invokeTracker ?? new PendingInvokeTracker(); } public override string Name => "windows_screenshot"; @@ -95,22 +97,50 @@ public override Task ExecuteAsync(JsonElement? arguments) { return Task.FromResult(ErrorResult($"Element not found: {refId}")); } + + // Element capture needs the UIA bounding rectangle, which hangs while + // the app's provider is blocked; suggest fullScreen capture instead. + if (_invokeTracker.TryGetPending(_elementRegistry.GetProcessIdForRef(refId), out var pendingRef)) + { + return Task.FromResult(ErrorResult( + PendingInvokeTracker.DescribeBlocked(pendingRef) + + " For screenshots, use fullScreen: true or a window handle instead of a ref.")); + } + capture = Capture.Element(element); } else if (!string.IsNullOrEmpty(handle)) { - var window = _sessionManager.GetWindow(handle); - if (window == null) + // While the app's UIA provider is blocked (pending pattern call, e.g. an + // open modal dialog), fall back to a pure Win32 capture of the window + // bounds so screenshots keep working. + if (_invokeTracker.TryGetPending(_sessionManager.GetWindowProcessId(handle), out _)) { - return Task.FromResult(ErrorResult($"Window not found: {handle}")); + var hwnd = _sessionManager.GetWindowHwnd(handle); + var bounds = hwnd != 0 ? Win32Desktop.GetWindowBounds(hwnd) : null; + if (bounds == null) + { + return Task.FromResult(ErrorResult( + "This app's UI Automation provider is blocked and its window bounds are unknown. " + + "Use fullScreen: true instead.")); + } + capture = Capture.Rectangle(bounds.Value); } - - if (background && NativeWindowCapture.TryCaptureWindow(window, out var backgroundImage, out _)) + else { - return Task.FromResult(BuildScreenshotResult(backgroundImage, normalizedSavePath, overwrite)); - } + var window = _sessionManager.GetWindow(handle); + if (window == null) + { + return Task.FromResult(ErrorResult($"Window not found: {handle}")); + } - capture = Capture.Element(window); + if (background && NativeWindowCapture.TryCaptureWindow(window, out var backgroundImage, out _)) + { + return Task.FromResult(BuildScreenshotResult(backgroundImage, normalizedSavePath, overwrite)); + } + + capture = Capture.Element(window); + } } else { diff --git a/src/FlaUI.Mcp/Tools/SendKeysTool.cs b/src/FlaUI.Mcp/Tools/SendKeysTool.cs index 3a8b619..df8d459 100644 --- a/src/FlaUI.Mcp/Tools/SendKeysTool.cs +++ b/src/FlaUI.Mcp/Tools/SendKeysTool.cs @@ -115,14 +115,17 @@ public class SendKeysTool : ToolBase }; private readonly ElementRegistry _elementRegistry; + private readonly PendingInvokeTracker _invokeTracker; /// /// Initializes a new instance of the class. /// /// Registry used to resolve element references for focus targeting. - public SendKeysTool(ElementRegistry elementRegistry) + /// Tracker used to fail fast when the target app's UIA provider is blocked. + public SendKeysTool(ElementRegistry elementRegistry, PendingInvokeTracker? invokeTracker = null) { _elementRegistry = elementRegistry; + _invokeTracker = invokeTracker ?? new PendingInvokeTracker(); } /// @@ -200,6 +203,14 @@ public override Task ExecuteAsync(JsonElement? arguments) return Task.FromResult(ErrorResult($"Element not found: {refId}. Run windows_snapshot to refresh element refs.")); } + // Fail fast if this app's UIA provider is blocked (element.Focus() would hang). + // Tip: calling windows_send_keys without a ref sends pure keyboard input to the + // focused element, which works even while the provider is blocked. + if (_invokeTracker.TryGetPending(_elementRegistry.GetProcessIdForRef(refId), out var pending)) + { + return Task.FromResult(ErrorResult(PendingInvokeTracker.DescribeBlocked(pending))); + } + element.Focus(); Thread.Sleep(50); } @@ -313,4 +324,4 @@ private static IEnumerable SplitChord(string? input) .Split('+', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) .Where(part => !string.IsNullOrWhiteSpace(part)); } -} \ No newline at end of file +} diff --git a/src/FlaUI.Mcp/Tools/SnapshotTool.cs b/src/FlaUI.Mcp/Tools/SnapshotTool.cs index 15a17c9..39bda49 100644 --- a/src/FlaUI.Mcp/Tools/SnapshotTool.cs +++ b/src/FlaUI.Mcp/Tools/SnapshotTool.cs @@ -12,12 +12,14 @@ public class SnapshotTool : ToolBase private readonly SessionManager _sessionManager; private readonly ElementRegistry _elementRegistry; private readonly SnapshotBuilder _snapshotBuilder; + private readonly PendingInvokeTracker _invokeTracker; - public SnapshotTool(SessionManager sessionManager, ElementRegistry elementRegistry) + public SnapshotTool(SessionManager sessionManager, ElementRegistry elementRegistry, PendingInvokeTracker? invokeTracker = null) { _sessionManager = sessionManager; _elementRegistry = elementRegistry; _snapshotBuilder = new SnapshotBuilder(elementRegistry); + _invokeTracker = invokeTracker ?? new PendingInvokeTracker(); } public override string Name => "windows_snapshot"; @@ -50,6 +52,15 @@ public override Task ExecuteAsync(JsonElement? arguments) if (!string.IsNullOrEmpty(handle)) { + // Fail fast when this app's UIA provider is blocked by a pending + // pattern call (e.g. a click that opened a modal dialog) — + // walking the UIA tree would hang until the global timeout. + var processId = _sessionManager.GetWindowProcessId(handle); + if (_invokeTracker.TryGetPending(processId, out var pending)) + { + return Task.FromResult(ErrorResult(PendingInvokeTracker.DescribeBlocked(pending))); + } + window = _sessionManager.GetWindow(handle); if (window == null) { diff --git a/src/FlaUI.Mcp/Tools/TypeTools.cs b/src/FlaUI.Mcp/Tools/TypeTools.cs index fb68e27..53ccc51 100644 --- a/src/FlaUI.Mcp/Tools/TypeTools.cs +++ b/src/FlaUI.Mcp/Tools/TypeTools.cs @@ -11,10 +11,12 @@ namespace PlaywrightWindows.Mcp.Tools; public class TypeTool : ToolBase { private readonly ElementRegistry _elementRegistry; + private readonly PendingInvokeTracker _invokeTracker; - public TypeTool(ElementRegistry elementRegistry) + public TypeTool(ElementRegistry elementRegistry, PendingInvokeTracker? invokeTracker = null) { _elementRegistry = elementRegistry; + _invokeTracker = invokeTracker ?? new PendingInvokeTracker(); } public override string Name => "windows_type"; @@ -69,6 +71,14 @@ public override Task ExecuteAsync(JsonElement? arguments) return Task.FromResult(ErrorResult($"Element not found: {refId}. Run windows_snapshot to refresh element refs.")); } + // Fail fast if this app's UIA provider is blocked (element.Focus() would hang). + // Tip: calling windows_type without a ref types into the focused element + // using pure keyboard input, which works even while the provider is blocked. + if (_invokeTracker.TryGetPending(_elementRegistry.GetProcessIdForRef(refId), out var pending)) + { + return Task.FromResult(ErrorResult(PendingInvokeTracker.DescribeBlocked(pending))); + } + element.Focus(); Thread.Sleep(50); // Small delay to ensure focus } @@ -98,10 +108,12 @@ public override Task ExecuteAsync(JsonElement? arguments) public class FillTool : ToolBase { private readonly ElementRegistry _elementRegistry; + private readonly PendingInvokeTracker _invokeTracker; - public FillTool(ElementRegistry elementRegistry) + public FillTool(ElementRegistry elementRegistry, PendingInvokeTracker? invokeTracker = null) { _elementRegistry = elementRegistry; + _invokeTracker = invokeTracker ?? new PendingInvokeTracker(); } public override string Name => "windows_fill"; @@ -148,6 +160,12 @@ public override Task ExecuteAsync(JsonElement? arguments) return Task.FromResult(ErrorResult($"Element not found: {refId}. Run windows_snapshot to refresh element refs.")); } + // Fail fast if this app's UIA provider is blocked by a pending pattern call + if (_invokeTracker.TryGetPending(_elementRegistry.GetProcessIdForRef(refId), out var pending)) + { + return Task.FromResult(ErrorResult(PendingInvokeTracker.DescribeBlocked(pending))); + } + try { var elementName = element.Properties.Name.ValueOrDefault ?? refId; diff --git a/tests/FlaUI.Mcp.IntegrationTests/ModalDialogTests.cs b/tests/FlaUI.Mcp.IntegrationTests/ModalDialogTests.cs new file mode 100644 index 0000000..950cfb9 --- /dev/null +++ b/tests/FlaUI.Mcp.IntegrationTests/ModalDialogTests.cs @@ -0,0 +1,139 @@ +using System.Diagnostics; +using PlaywrightWindows.Mcp.Core; +using PlaywrightWindows.Mcp.Tools; +using Xunit.Abstractions; + +namespace FlaUI.Mcp.IntegrationTests; + +/// +/// Tests for modal-dialog handling: a click whose handler opens a modal dialog +/// must not hang the click tool, and other tools must fail fast (with guidance) +/// instead of hanging while the app's UIA provider is blocked. +/// +[Collection("TestApps")] +public class ModalDialogTests +{ + private readonly TestAppFixture _fixture; + private readonly ITestOutputHelper _output; + + public ModalDialogTests(TestAppFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + } + + [Fact] + public async Task WinForms_ClickOpensModal_ReturnsEarlyAndFailsFastUntilDismissed() + { + var tracker = new PendingInvokeTracker(); + var clickTool = new ClickTool(_fixture.Elements, tracker); + var snapshotTool = new SnapshotTool(_fixture.Session, _fixture.Elements, tracker); + var sendKeysTool = new SendKeysTool(_fixture.Elements, tracker); + + // Capture the pid while the provider is responsive; querying it later, + // while the modal blocks the provider, would itself hang + var processId = GetWinFormsProcessId(); + Assert.NotEqual(0, processId); + + // Navigate to the Dialogs tab + var dialogsTabRef = _fixture.FindRefByName(_fixture.WinFormsHandle, "Dialogs"); + Assert.NotNull(dialogsTabRef); + await _fixture.CallTool(clickTool, new { @ref = dialogsTabRef }); + await Task.Delay(250); + + var modalButtonRef = _fixture.FindRefByName(_fixture.WinFormsHandle, "Open Modal Dialog"); + Assert.NotNull(modalButtonRef); + + try + { + // 1. Clicking the button must return quickly and report the modal + var sw = Stopwatch.StartNew(); + var clickResult = await _fixture.CallTool(clickTool, new { @ref = modalButtonRef }); + sw.Stop(); + _output.WriteLine($"Click result ({sw.ElapsedMilliseconds}ms): {clickResult}"); + + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(10), $"Click took too long: {sw.Elapsed}"); + Assert.Contains("modal dialog", clickResult, StringComparison.OrdinalIgnoreCase); + Assert.Contains("Test Modal Dialog", clickResult); + + // 2. While the dialog is open, snapshot must fail fast with guidance + // instead of hanging until the global timeout + sw.Restart(); + var snapshotResult = await _fixture.CallTool(snapshotTool, new { handle = _fixture.WinFormsHandle }); + sw.Stop(); + _output.WriteLine($"Snapshot result ({sw.ElapsedMilliseconds}ms): {snapshotResult}"); + + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(5), $"Snapshot fail-fast took too long: {sw.Elapsed}"); + Assert.Contains("blocked", snapshotResult, StringComparison.OrdinalIgnoreCase); + Assert.Contains("windows_send_keys", snapshotResult); + } + finally + { + // 3. Dismiss the dialog with pure keyboard input (works while blocked): + // the OK button is the dialog's accept button + await _fixture.CallTool(sendKeysTool, new { chord = "Enter" }); + } + + // 4. Once the dialog closes, the pending invoke completes and tools recover + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (tracker.TryGetPending(processId, out _) && DateTime.UtcNow < deadline) + { + await Task.Delay(100); + } + Assert.False(tracker.TryGetPending(processId, out _), + "Pending invoke did not clear after the dialog was dismissed."); + + var recoveredSnapshot = await _fixture.CallTool(snapshotTool, new { handle = _fixture.WinFormsHandle }); + Assert.Contains("Open Modal Dialog", recoveredSnapshot); + } + + [Fact] + public async Task WinForms_ClickOpensModeless_CompletesNormally() + { + var tracker = new PendingInvokeTracker(); + var clickTool = new ClickTool(_fixture.Elements, tracker); + var sendKeysTool = new SendKeysTool(_fixture.Elements, tracker); + + var processId = GetWinFormsProcessId(); + Assert.NotEqual(0, processId); + + // Navigate to the Dialogs tab + var dialogsTabRef = _fixture.FindRefByName(_fixture.WinFormsHandle, "Dialogs"); + Assert.NotNull(dialogsTabRef); + await _fixture.CallTool(clickTool, new { @ref = dialogsTabRef }); + await Task.Delay(250); + + var modelessButtonRef = _fixture.FindRefByName(_fixture.WinFormsHandle, "Open Modeless Dialog"); + Assert.NotNull(modelessButtonRef); + + try + { + // Show() returns immediately, so the invoke completes; the result may + // legitimately be either a plain completion or (on a slow machine) a + // modal-detection race, but it must never take the full grace period path + var clickResult = await _fixture.CallTool(clickTool, new { @ref = modelessButtonRef }); + _output.WriteLine($"Click result: {clickResult}"); + Assert.Contains("Invoked", clickResult); + + // Provider is not blocked: no pending invoke remains after a moment + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); + while (tracker.TryGetPending(processId, out _) && DateTime.UtcNow < deadline) + { + await Task.Delay(50); + } + Assert.False(tracker.TryGetPending(processId, out _)); + } + finally + { + // Close the modeless dialog (it has focus; Alt+F4 closes it) + await _fixture.CallTool(sendKeysTool, new { chord = "Alt+F4" }); + await Task.Delay(250); + } + } + + private int GetWinFormsProcessId() + { + var window = _fixture.GetWinFormsWindow(); + return window?.Properties.ProcessId.ValueOrDefault ?? 0; + } +} diff --git a/tests/FlaUI.Mcp.Tests/KeepAwakeTests.cs b/tests/FlaUI.Mcp.Tests/KeepAwakeTests.cs new file mode 100644 index 0000000..38ce013 --- /dev/null +++ b/tests/FlaUI.Mcp.Tests/KeepAwakeTests.cs @@ -0,0 +1,143 @@ +using PlaywrightWindows.Mcp.Core; +using Xunit; + +namespace FlaUI.Mcp.Tests; + +public class KeepAwakeTests +{ + [Fact] + public void Poke_FirstCall_AcquiresOnce() + { + var acquired = 0; + var released = 0; + using var keepAwake = new KeepAwake(TimeSpan.FromMinutes(5), () => acquired++, () => released++); + + keepAwake.Poke(); + keepAwake.Poke(); + keepAwake.Poke(); + + Assert.Equal(1, acquired); + Assert.Equal(0, released); + Assert.True(keepAwake.IsActive); + } + + [Fact] + public void Poke_AfterIdlePeriod_Releases() + { + var released = new ManualResetEventSlim(); + using var keepAwake = new KeepAwake(TimeSpan.FromMilliseconds(50), () => { }, released.Set); + + keepAwake.Poke(); + + Assert.True(released.Wait(TimeSpan.FromSeconds(5)), "Release was not called after the hold duration elapsed."); + Assert.False(keepAwake.IsActive); + } + + [Fact] + public void Poke_AfterRelease_AcquiresAgain() + { + var acquired = 0; + var released = new ManualResetEventSlim(); + using var keepAwake = new KeepAwake(TimeSpan.FromMilliseconds(50), () => acquired++, released.Set); + + keepAwake.Poke(); + Assert.True(released.Wait(TimeSpan.FromSeconds(5))); + + keepAwake.Poke(); + + Assert.Equal(2, acquired); + Assert.True(keepAwake.IsActive); + } + + [Fact] + public void Poke_ExtendsHold_NoReleaseWhileActivityContinues() + { + var released = new ManualResetEventSlim(); + using var keepAwake = new KeepAwake(TimeSpan.FromMilliseconds(200), () => { }, released.Set); + + // Keep poking at intervals well within the hold duration. + for (var i = 0; i < 5; i++) + { + keepAwake.Poke(); + Thread.Sleep(50); + Assert.False(released.IsSet, $"Released too early on iteration {i}."); + } + + Assert.True(keepAwake.IsActive); + Assert.True(released.Wait(TimeSpan.FromSeconds(5)), "Release was not called after activity stopped."); + } + + [Fact] + public void Dispose_WhileActive_Releases() + { + var released = 0; + var keepAwake = new KeepAwake(TimeSpan.FromMinutes(5), () => { }, () => released++); + + keepAwake.Poke(); + keepAwake.Dispose(); + + Assert.Equal(1, released); + } + + [Fact] + public void Dispose_WhileInactive_DoesNotRelease() + { + var released = 0; + var keepAwake = new KeepAwake(TimeSpan.FromMinutes(5), () => { }, () => released++); + + keepAwake.Dispose(); + + Assert.Equal(0, released); + } + + [Fact] + public void Poke_AfterDispose_IsIgnored() + { + var acquired = 0; + var keepAwake = new KeepAwake(TimeSpan.FromMinutes(5), () => acquired++, () => { }); + + keepAwake.Dispose(); + keepAwake.Poke(); + + Assert.Equal(0, acquired); + Assert.False(keepAwake.IsActive); + } + + [Fact] + public void Constructor_NonPositiveHoldDuration_Throws() + { + Assert.Throws( + () => new KeepAwake(TimeSpan.Zero, () => { }, () => { })); + } + + [Fact] + public void Dispose_DisposesOwnedResource() + { + var resource = new TrackingDisposable(); + var keepAwake = new KeepAwake(TimeSpan.FromMinutes(5), () => { }, () => { }, resource); + + keepAwake.Dispose(); + + Assert.True(resource.Disposed); + } + + [Fact] + public void CreateDisplayKeepAwake_OnWindows_CreatesWorkingInstance() + { + // Exercises the real PowerCreateRequest/PowerSetRequest/PowerClearRequest + // round-trip. Setting and clearing an availability request has no lasting + // side effects. + using var keepAwake = KeepAwake.CreateDisplayKeepAwake( + TimeSpan.FromMinutes(5), "FlaUI-MCP unit test"); + + Assert.NotNull(keepAwake); + keepAwake!.Poke(); + Assert.True(keepAwake.IsActive); + } + + private sealed class TrackingDisposable : IDisposable + { + public bool Disposed { get; private set; } + public void Dispose() => Disposed = true; + } +} diff --git a/tests/FlaUI.Mcp.Tests/ModalAwareInvokerTests.cs b/tests/FlaUI.Mcp.Tests/ModalAwareInvokerTests.cs new file mode 100644 index 0000000..6da961f --- /dev/null +++ b/tests/FlaUI.Mcp.Tests/ModalAwareInvokerTests.cs @@ -0,0 +1,176 @@ +using System.Diagnostics; +using PlaywrightWindows.Mcp.Core; +using Xunit; + +namespace FlaUI.Mcp.Tests; + +public class ModalAwareInvokerTests +{ + private const int TestProcessId = 1234; + + private static Win32WindowInfo MakeWindow(nint hwnd, string title, bool enabled = true) => + new(hwnd, title, TestProcessId, enabled, IsToolWindow: false, IsCloaked: false); + + [Fact] + public void Execute_CompletedAction_ReturnsCompleted_AndClearsTracker() + { + var tracker = new PendingInvokeTracker(); + var windows = new List { MakeWindow(1, "Main") }; + + var result = ModalAwareInvoker.Execute( + TestProcessId, "Invoke on 'OK'", () => { }, tracker, + windowEnumerator: _ => windows); + + Assert.Equal(PatternCallOutcome.Completed, result.Outcome); + + // The tracker is cleared by a task continuation; allow it a moment + AssertTrackerClearsWithin(tracker, TimeSpan.FromSeconds(2)); + } + + [Fact] + public void Execute_FaultedAction_RethrowsException() + { + var tracker = new PendingInvokeTracker(); + + var ex = Assert.Throws(() => + ModalAwareInvoker.Execute( + TestProcessId, "Invoke on 'OK'", () => throw new InvalidOperationException("boom"), tracker, + windowEnumerator: _ => new List())); + + Assert.Equal("boom", ex.Message); + AssertTrackerClearsWithin(tracker, TimeSpan.FromSeconds(2)); + } + + [Fact] + public void Execute_BlockingActionWithNewWindow_ReturnsModalDetected() + { + var tracker = new PendingInvokeTracker(); + using var release = new ManualResetEventSlim(false); + var callCount = 0; + + // First enumeration: only the main window. Later enumerations: a new + // dialog window appeared while the action is still blocking. + IReadOnlyList Enumerate(int _) + { + var windows = new List { MakeWindow(1, "Main", enabled: callCount == 0) }; + if (Interlocked.Increment(ref callCount) > 1) + { + windows.Add(MakeWindow(2, "Test Modal Dialog")); + } + return windows; + } + + try + { + var sw = Stopwatch.StartNew(); + var result = ModalAwareInvoker.Execute( + TestProcessId, "Invoke on 'Open Modal'", () => release.Wait(), tracker, + gracePeriod: TimeSpan.FromSeconds(10), + windowEnumerator: Enumerate); + sw.Stop(); + + Assert.Equal(PatternCallOutcome.ModalDetected, result.Outcome); + Assert.Equal("Test Modal Dialog", result.ModalTitle); + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(5), $"Modal detection too slow: {sw.Elapsed}"); + + // The call is still pending and the tracker knows about the modal + Assert.True(tracker.TryGetPending(TestProcessId, out var pending)); + Assert.Equal("Test Modal Dialog", pending.ModalTitle); + } + finally + { + release.Set(); + } + + // Once the action returns, the tracker clears + AssertTrackerClearsWithin(tracker, TimeSpan.FromSeconds(2)); + } + + [Fact] + public void Execute_BlockingActionOwnerDisabledOnly_ReturnsModalDetected() + { + var tracker = new PendingInvokeTracker(); + using var release = new ManualResetEventSlim(false); + var callCount = 0; + + // The owner window becomes disabled but no new window is enumerated + // (e.g. the dialog is a tool window filtered elsewhere). + IReadOnlyList Enumerate(int _) + { + var enabled = Interlocked.Increment(ref callCount) == 1; + return new List { MakeWindow(1, "Main", enabled) }; + } + + try + { + var result = ModalAwareInvoker.Execute( + TestProcessId, "Invoke on 'Open Modal'", () => release.Wait(), tracker, + gracePeriod: TimeSpan.FromSeconds(10), + windowEnumerator: Enumerate); + + Assert.Equal(PatternCallOutcome.ModalDetected, result.Outcome); + } + finally + { + release.Set(); + } + } + + [Fact] + public void Execute_BlockingActionNoWindowChange_ReturnsStillPendingAfterGrace() + { + var tracker = new PendingInvokeTracker(); + using var release = new ManualResetEventSlim(false); + var windows = new List { MakeWindow(1, "Main") }; + + try + { + var sw = Stopwatch.StartNew(); + var result = ModalAwareInvoker.Execute( + TestProcessId, "Invoke on 'Slow'", () => release.Wait(), tracker, + gracePeriod: TimeSpan.FromMilliseconds(300), + windowEnumerator: _ => windows); + sw.Stop(); + + Assert.Equal(PatternCallOutcome.StillPending, result.Outcome); + Assert.True(sw.Elapsed >= TimeSpan.FromMilliseconds(250), $"Returned before grace period: {sw.Elapsed}"); + Assert.True(tracker.TryGetPending(TestProcessId, out _)); + } + finally + { + release.Set(); + } + + AssertTrackerClearsWithin(tracker, TimeSpan.FromSeconds(2)); + } + + [Fact] + public void Execute_UnknownProcessId_SkipsModalDetection() + { + var tracker = new PendingInvokeTracker(); + var enumeratorCalled = false; + + var result = ModalAwareInvoker.Execute( + 0, "Invoke on 'OK'", () => Thread.Sleep(100), tracker, + gracePeriod: TimeSpan.FromMilliseconds(600), + windowEnumerator: _ => + { + enumeratorCalled = true; + return new List(); + }); + + Assert.Equal(PatternCallOutcome.Completed, result.Outcome); + Assert.False(enumeratorCalled); + } + + private static void AssertTrackerClearsWithin(PendingInvokeTracker tracker, TimeSpan timeout) + { + var sw = Stopwatch.StartNew(); + while (sw.Elapsed < timeout) + { + if (!tracker.TryGetPending(TestProcessId, out _)) return; + Thread.Sleep(10); + } + Assert.Fail("Tracker still reports a pending invoke after the action completed."); + } +} diff --git a/tests/FlaUI.Mcp.Tests/PendingInvokeTrackerTests.cs b/tests/FlaUI.Mcp.Tests/PendingInvokeTrackerTests.cs new file mode 100644 index 0000000..50388ad --- /dev/null +++ b/tests/FlaUI.Mcp.Tests/PendingInvokeTrackerTests.cs @@ -0,0 +1,58 @@ +using PlaywrightWindows.Mcp.Core; +using Xunit; + +namespace FlaUI.Mcp.Tests; + +public class PendingInvokeTrackerTests +{ + [Fact] + public void TryGetPending_NoPendingCalls_ReturnsFalse() + { + var tracker = new PendingInvokeTracker(); + Assert.False(tracker.TryGetPending(42, out _)); + } + + [Fact] + public void TryGetPending_AfterBegin_ReturnsTrueForThatProcessOnly() + { + var tracker = new PendingInvokeTracker(); + tracker.Begin(42, "Invoke on 'OK'"); + + Assert.True(tracker.TryGetPending(42, out var info)); + Assert.Equal("Invoke on 'OK'", info.Description); + Assert.False(tracker.TryGetPending(43, out _)); + } + + [Fact] + public void TryGetPending_AfterComplete_ReturnsFalse() + { + var tracker = new PendingInvokeTracker(); + var info = tracker.Begin(42, "Invoke on 'OK'"); + tracker.Complete(info); + + Assert.False(tracker.TryGetPending(42, out _)); + } + + [Fact] + public void TryGetPending_ProcessIdZero_NeverMatches() + { + var tracker = new PendingInvokeTracker(); + tracker.Begin(0, "Invoke on unknown process"); + + Assert.False(tracker.TryGetPending(0, out _)); + } + + [Fact] + public void DescribeBlocked_IncludesDescriptionAndModalTitle() + { + var tracker = new PendingInvokeTracker(); + var info = tracker.Begin(42, "Invoke on 'Open...'"); + info.ModalTitle = "Open File"; + + var message = PendingInvokeTracker.DescribeBlocked(info); + + Assert.Contains("Invoke on 'Open...'", message); + Assert.Contains("Open File", message); + Assert.Contains("windows_send_keys", message); + } +} diff --git a/tests/FlaUI.Mcp.Tests/ToolRegistryTimeoutTests.cs b/tests/FlaUI.Mcp.Tests/ToolRegistryTimeoutTests.cs index acd4871..b6ffe26 100644 --- a/tests/FlaUI.Mcp.Tests/ToolRegistryTimeoutTests.cs +++ b/tests/FlaUI.Mcp.Tests/ToolRegistryTimeoutTests.cs @@ -34,6 +34,19 @@ public async Task ExecuteToolAsync_ReturnsSuccessfulToolResult() Assert.Equal("ok", result.Content[0].Text); } + [Fact] + public async Task ExecuteToolAsync_InvokesActivityCallbackForKnownToolsOnly() + { + var activity = 0; + var registry = new ToolRegistry(TimeSpan.FromSeconds(1), onToolActivity: () => activity++); + registry.RegisterTool(new SuccessfulTool()); + + await registry.ExecuteToolAsync("successful", arguments: null); + await registry.ExecuteToolAsync("unknown", arguments: null); + + Assert.Equal(1, activity); + } + private sealed class BlockingTool : ITool { private readonly TimeSpan _delay;