Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions FEATURES.md
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,14 @@ For smaller replies, use `ui_snapshot` with a known `parentElementId`, or use
Snapshot `maxDepth` and `controlTypeFilter` reduce the returned content, but do
not necessarily reduce the work done by the app to provide its controls.

With a positive `timeoutMs` (default: 5,000), retries and scans share one time
budget. No new scan starts at or after the deadline, and scanning stops between
Windows accessibility calls when time runs out. A Windows call already in
progress can still take longer to return. An interrupted scan reports `timeout`
without claiming the target is absent or unique. Set `includeDiagnostics=true`
to see elapsed time and the last scan's element count. `timeoutMs=0` performs
one node-bounded scan without a time deadline.

---

## 🖱️ UI Click (`ui_click`)
Expand Down Expand Up @@ -516,6 +524,10 @@ Wait until a UI condition is met before continuing - no blind sleeps or screensh
- Wait for a specific element to become enabled/visible/toggled
- State mode rejects selectors; appear/disappear reject `elementId` and `desiredState`

Appear/disappear waits use the same shared retry-and-scan time budget as
`ui_find`. An interrupted scan reports `timeout`, not a successful disappearance.
Windows accessibility calls already in progress can still overrun the time budget.

---

## 🧩 UI Batch (`ui_batch`)
Expand Down
14 changes: 12 additions & 2 deletions src/Sbroenne.WindowsMcp/Automation/BoundedSearchTraversal.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ internal static Outcome Walk<T>(
Func<T, int, bool> visit,
int maxNodes,
int maxDepth,
CancellationToken cancellationToken)
CancellationToken cancellationToken,
Action? checkDeadline = null)
where T : class
{
ArgumentNullException.ThrowIfNull(root);
Expand All @@ -23,6 +24,7 @@ internal static Outcome Walk<T>(
ArgumentNullException.ThrowIfNull(visit);
ArgumentOutOfRangeException.ThrowIfNegative(maxNodes);
cancellationToken.ThrowIfCancellationRequested();
checkDeadline?.Invoke();
if (maxDepth <= 0)
{
return new(0, false);
Expand All @@ -35,14 +37,18 @@ internal static Outcome Walk<T>(

var ancestors = new Stack<(T Node, int ParentDepth)>();
var current = firstChild(root);
checkDeadline?.Invoke();
var parentDepth = 0;
var scanned = 0;
while (current is not null)
{
cancellationToken.ThrowIfCancellationRequested();
checkDeadline?.Invoke();
scanned++;
var depth = parentDepth + (countsForDepth(current) ? 1 : 0);
if (visit(current, depth))
var satisfied = visit(current, depth);
checkDeadline?.Invoke();
Comment thread
sbroenne marked this conversation as resolved.
if (satisfied)
{
return new(scanned, false);
}
Expand All @@ -55,7 +61,9 @@ internal static Outcome Walk<T>(
}

cancellationToken.ThrowIfCancellationRequested();
checkDeadline?.Invoke();
var child = depth < maxDepth ? firstChild(current) : null;
checkDeadline?.Invoke();
if (child is not null)
{
ancestors.Push((current, parentDepth));
Expand All @@ -67,7 +75,9 @@ internal static Outcome Walk<T>(
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
checkDeadline?.Invoke();
var sibling = nextSibling(current);
checkDeadline?.Invoke();
if (sibling is not null)
{
current = sibling;
Expand Down
43 changes: 34 additions & 9 deletions src/Sbroenne.WindowsMcp/Automation/ElementIdGenerator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -30,31 +30,45 @@ public static string GenerateId(UIA.IUIAutomationElement element, UIA.IUIAutomat
Generate(element, rootElement, cached: false);

/// <summary>Registers a live element using cached identity properties when available.</summary>
public static string GenerateFastId(UIA.IUIAutomationElement element, UIA.IUIAutomationElement rootElement) =>
Generate(element, rootElement, cached: true);
public static string GenerateFastId(
UIA.IUIAutomationElement element,
UIA.IUIAutomationElement rootElement,
Action? checkDeadline = null) =>
Generate(element, rootElement, cached: true, checkDeadline);

/// <summary>Registers a live element when no property cache was requested.</summary>
public static string GenerateFastIdFromCurrent(UIA.IUIAutomationElement element, UIA.IUIAutomationElement rootElement) =>
Generate(element, rootElement, cached: false);

private static string Generate(UIA.IUIAutomationElement element, UIA.IUIAutomationElement rootElement, bool cached)
public static string GenerateFastIdFromCurrent(
UIA.IUIAutomationElement element,
UIA.IUIAutomationElement rootElement,
Action? checkDeadline = null) =>
Generate(element, rootElement, cached: false, checkDeadline);

private static string Generate(
UIA.IUIAutomationElement element,
UIA.IUIAutomationElement rootElement,
bool cached,
Action? checkDeadline = null)
{
ArgumentNullException.ThrowIfNull(element);
ArgumentNullException.ThrowIfNull(rootElement);
try
{
checkDeadline?.Invoke();
nint handle;
try
{
handle = cached ? rootElement.GetCachedNativeWindowHandle() : rootElement.GetNativeWindowHandle();
checkDeadline?.Invoke();
}
catch (Exception ex) when (COMExceptionHelper.IsExpectedElementFailure(ex))
{
checkDeadline?.Invoke();
handle = rootElement.GetNativeWindowHandle();
checkDeadline?.Invoke();
}
if (handle == nint.Zero)
{
handle = GetTopLevelWindowHandle(element);
handle = GetTopLevelWindowHandle(element, checkDeadline);
}
else
{
Expand All @@ -65,17 +79,23 @@ private static string Generate(UIA.IUIAutomationElement element, UIA.IUIAutomati
int providerProcessId;
try
{
checkDeadline?.Invoke();
runtimeId = cached
? (int[]?)element.GetCachedPropertyValue(UIA3PropertyIds.RuntimeId)
: element.GetRuntimeId();
checkDeadline?.Invoke();
providerProcessId = cached
? (int)element.GetCachedPropertyValue(UIA3PropertyIds.ProcessId)
: element.CurrentProcessId;
checkDeadline?.Invoke();
}
catch (Exception ex) when (COMExceptionHelper.IsExpectedElementFailure(ex))
{
checkDeadline?.Invoke();
runtimeId = element.GetRuntimeId();
checkDeadline?.Invoke();
providerProcessId = element.CurrentProcessId;
checkDeadline?.Invoke();
}

var runtime = runtimeId is { Length: > 0 } ? string.Join(".", runtimeId) : "0";
Expand Down Expand Up @@ -133,19 +153,24 @@ private static string Generate(UIA.IUIAutomationElement element, UIA.IUIAutomati
return null;
}

private static nint GetTopLevelWindowHandle(UIA.IUIAutomationElement element)
private static nint GetTopLevelWindowHandle(UIA.IUIAutomationElement element, Action? checkDeadline = null)
{
var current = element;
checkDeadline?.Invoke();
var desktop = UIA3Automation.Instance.RootElement;
while (current is not null && !current.IsSameElement(desktop))
checkDeadline?.Invoke();
while (current is not null && !current.IsSameElement(desktop, checkDeadline))
{
checkDeadline?.Invoke();
var currentHandle = current.GetNativeWindowHandle();
checkDeadline?.Invoke();
if (currentHandle != nint.Zero)
{
return NativeMethods.GetAncestor(currentHandle, NativeConstants.GA_ROOT);
}

current = current.GetParent();
checkDeadline?.Invoke();
}

return nint.Zero;
Expand Down
5 changes: 4 additions & 1 deletion src/Sbroenne.WindowsMcp/Automation/Tools/UIFindTool.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ public static partial class UIFindTool
/// If unvisited nodes remain at the scan limit, search_incomplete means absence or uniqueness could not be established.
/// Narrow the visited area with a known parentElementId or exactDepth=1, not just more match filters.
/// Increasing timeoutMs does not increase this limit. Use ui_read with a discovered elementId when only its text is needed.
/// Positive timeoutMs limits both retries and scanning. No new scan starts at or after the deadline;
/// an in-progress Windows accessibility call can still overrun it. An interrupted scan reports timeout,
/// not absence or uniqueness. timeoutMs=0 performs one node-bounded scan without a time deadline.
/// </remarks>
/// <param name="windowHandle">Window handle as decimal string (from window_management 'find' or 'list'). REQUIRED.</param>
/// <param name="name">Element name (exact match, case-insensitive). For Electron apps and Chromium browsers, this is often the visible label or ARIA label.</param>
Expand All @@ -52,7 +55,7 @@ public static partial class UIFindTool
/// <param name="scope">Search root: window (default) or active_dialog. Use active_dialog after opening a modal or native file dialog.</param>
/// <param name="requireUnique">Fail with ambiguity details when more than one element matches. Default: false.</param>
/// <param name="enabledOnly">Exclude disabled elements when true.</param>
/// <param name="timeoutMs">Timeout in milliseconds (default: 5000).</param>
/// <param name="timeoutMs">Search time budget in milliseconds (default: 5000). Stops between Windows accessibility calls; an in-progress call can overrun the budget. Zero performs one scan without a time deadline.</param>
/// <param name="includeDiagnostics">Include diagnostics (timing, query, elements scanned) in response. Default: false.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>A call result containing a text content block with the JSON payload listing found elements and their properties (including element IDs). <c>IsError</c> reflects operation success.</returns>
Expand Down
3 changes: 3 additions & 0 deletions src/Sbroenne.WindowsMcp/Automation/Tools/UIWaitTool.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ public static partial class UIWaitTool
/// - 'state': wait until the element with the given elementId reaches desiredState. Provide elementId + desiredState.
/// Uses efficient exponential backoff polling internally. Returns success as soon as the condition holds,
/// or a timeout failure with diagnostics.
/// Appear/disappear waits share one time budget across retries and scanning; no new scan starts
/// at or after the deadline. An in-progress Windows accessibility call can overrun the budget.
/// An interrupted scan cannot establish absence, uniqueness, or disappearance.
/// </remarks>
/// <param name="windowHandle">Window handle as decimal string (from window_management 'find'/'list' or app). Used to scope 'appear'/'disappear'.</param>
/// <param name="mode">Condition to wait for: 'appear', 'disappear', or 'state'. Default: 'appear'.</param>
Expand Down
43 changes: 32 additions & 11 deletions src/Sbroenne.WindowsMcp/Automation/UIA3Extensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,7 @@ public static bool SupportsPattern(this UIA.IUIAutomationElement element, int pa
/// <summary>
/// Gets all supported pattern IDs by checking each known pattern.
/// </summary>
public static int[] GetSupportedPatternIds(this UIA.IUIAutomationElement element)
public static int[] GetSupportedPatternIds(this UIA.IUIAutomationElement element, Action? checkDeadline = null)
{
ArgumentNullException.ThrowIfNull(element);
var supportedPatterns = new List<int>();
Expand Down Expand Up @@ -456,7 +456,10 @@ public static int[] GetSupportedPatternIds(this UIA.IUIAutomationElement element

foreach (var patternId in allPatternIds)
{
if (element.SupportsPattern(patternId))
checkDeadline?.Invoke();
var supported = element.SupportsPattern(patternId);
checkDeadline?.Invoke();
if (supported)
{
supportedPatterns.Add(patternId);
}
Expand All @@ -468,10 +471,10 @@ public static int[] GetSupportedPatternIds(this UIA.IUIAutomationElement element
/// <summary>
/// Gets all supported pattern names.
/// </summary>
public static string[] GetSupportedPatternNames(this UIA.IUIAutomationElement element)
public static string[] GetSupportedPatternNames(this UIA.IUIAutomationElement element, Action? checkDeadline = null)
{
ArgumentNullException.ThrowIfNull(element);
var ids = element.GetSupportedPatternIds();
var ids = element.GetSupportedPatternIds(checkDeadline);
return ids.Select(UIA3PatternIds.ToName).ToArray();
}

Expand Down Expand Up @@ -590,13 +593,17 @@ public static string[] GetSupportedPatternNames(this UIA.IUIAutomationElement el
/// <summary>
/// Tries to get the Value pattern's current value.
/// </summary>
public static string? TryGetValue(this UIA.IUIAutomationElement element)
public static string? TryGetValue(this UIA.IUIAutomationElement element, Action? checkDeadline = null)
{
ArgumentNullException.ThrowIfNull(element);
try
{
checkDeadline?.Invoke();
var pattern = element.GetPattern<UIA.IUIAutomationValuePattern>(UIA3PatternIds.Value);
return pattern?.CurrentValue;
checkDeadline?.Invoke();
var value = pattern?.CurrentValue;
checkDeadline?.Invoke();
return value;
}

catch (COMException ex) when (COMExceptionHelper.IsExpectedElementFailure(ex))
Expand Down Expand Up @@ -677,18 +684,22 @@ public static bool TryToggle(this UIA.IUIAutomationElement element)
/// <summary>
/// Gets the toggle state.
/// </summary>
public static string? GetToggleState(this UIA.IUIAutomationElement element)
public static string? GetToggleState(this UIA.IUIAutomationElement element, Action? checkDeadline = null)
{
ArgumentNullException.ThrowIfNull(element);
try
{
checkDeadline?.Invoke();
var pattern = element.GetPattern<UIA.IUIAutomationTogglePattern>(UIA3PatternIds.Toggle);
checkDeadline?.Invoke();
if (pattern == null)
{
return null;
}

return pattern.CurrentToggleState switch
var state = pattern.CurrentToggleState;
checkDeadline?.Invoke();
return state switch
{
UIA.ToggleState.ToggleState_Off => "Off",
UIA.ToggleState.ToggleState_On => "On",
Expand All @@ -705,13 +716,17 @@ public static bool TryToggle(this UIA.IUIAutomationElement element)
/// <summary>
/// Gets selection state using the SelectionItem pattern.
/// </summary>
public static string? GetSelectionStateName(this UIA.IUIAutomationElement element)
public static string? GetSelectionStateName(this UIA.IUIAutomationElement element, Action? checkDeadline = null)
{
ArgumentNullException.ThrowIfNull(element);
try
{
checkDeadline?.Invoke();
var pattern = element.GetPattern<UIA.IUIAutomationSelectionItemPattern>(UIA3PatternIds.SelectionItem);
return pattern is null ? null : pattern.CurrentIsSelected != 0 ? "On" : "Off";
checkDeadline?.Invoke();
var state = pattern is null ? null : pattern.CurrentIsSelected != 0 ? "On" : "Off";
checkDeadline?.Invoke();
return state;
}
catch (COMException ex) when (COMExceptionHelper.IsExpectedElementFailure(ex))
{
Expand Down Expand Up @@ -850,7 +865,10 @@ public static bool TryScrollIntoView(this UIA.IUIAutomationElement element)
/// <summary>
/// Compares two elements by runtime ID.
/// </summary>
public static bool IsSameElement(this UIA.IUIAutomationElement element, UIA.IUIAutomationElement? other)
public static bool IsSameElement(
this UIA.IUIAutomationElement element,
UIA.IUIAutomationElement? other,
Action? checkDeadline = null)
{
ArgumentNullException.ThrowIfNull(element);
if (other == null)
Expand All @@ -860,8 +878,11 @@ public static bool IsSameElement(this UIA.IUIAutomationElement element, UIA.IUIA

try
{
checkDeadline?.Invoke();
var id1 = element.GetRuntimeId();
checkDeadline?.Invoke();
var id2 = other.GetRuntimeId();
checkDeadline?.Invoke();

if (id1 == null || id2 == null)
{
Expand Down
Loading
Loading