feat(agent): resolve visual targets to semantic refs before coordinate fallback

src/chrome/src/agent/tools.js

@@ -1684,6 +1684,7 @@ ${PLAN_TO_EXECUTION_GUIDANCE}
 Available tools:
 - get_accessibility_tree: PREFERRED read. Flat-text tree of the page with roles, names, and stable ref_ids. Default starting point for almost every turn.
 - inspect_viewport: Read-only visual inspection when appearance or rendered pixels matter.
+- After visual inspection, act on a screenshot-derived point with click({x,y,from_screenshot:true}); WebBrain converts image pixels to CSS pixels mechanically.
 - click_ax: Click a node by its ref_id from the tree. Preferred over click({text/selector}).
 - set_checked: Idempotently set a native checkbox by ref_id and verify checkedBefore/checkedAfter. Use this instead of toggling with click_ax.
 - type_ax: Type into a node by its ref_id from the tree. Preferred over the click-then-type_text pattern.
@@ -2044,6 +2045,7 @@ ${PLAN_TO_EXECUTION_GUIDANCE}
 TOOLS — use only these:
 - get_accessibility_tree: PREFERRED read. Flat-text tree with roles, names, and stable ref_ids. Use filter:"visible" by default.
 - inspect_viewport: Read-only visual inspection for ads, images, canvas, charts, and layout.
+- After inspect_viewport, act on a screenshot-derived point with click({x,y,from_screenshot:true}); WebBrain converts image pixels to CSS pixels mechanically.
 - click_ax({ref_id}) / set_checked({ref_id, checked}) / type_ax({ref_id, text}) / set_field({ref_id, text, submit}): act on nodes by ref_id. set_field is preferred for text fields; set_checked is required for native checkboxes.
 - read_page: prose fallback for long articles. get_window_info: inspect browser window/viewport size. scroll, navigate({url}), go_back()/go_forward(): walk the run tab's history. new_tab({url}) only opens a background reference tab and never retargets the run; promote_iframe({urlFilter}) navigates the current run to one child frame's standalone URL.
 - get_interactive_elements: legacy indexed element list (use when the tree misses elements). click({text}) / type_text({text}) / press_keys({key}): legacy fallbacks. press_keys supports only unmodified Escape/Tab/Enter/arrows or ; (semicolon), never Ctrl/Cmd/Alt/Shift combinations or browser shortcuts.

src/chrome/src/content/accessibility-tree.js

@@ -484,6 +484,61 @@
     return false;
   }
 
+  function composedParent(node) {
+    if (!node) return null;
+    if (node.assignedSlot) return node.assignedSlot;
+    const parent = node.parentNode;
+    if (parent) {
+      return (typeof ShadowRoot !== 'undefined' && parent instanceof ShadowRoot)
+        ? parent.host
+        : parent;
+    }
+    const root = node.getRootNode?.();
+    return (typeof ShadowRoot !== 'undefined' && root instanceof ShadowRoot)
+      ? root.host
+      : null;
+  }
+
+  function deepestOpenShadowHit(x, y) {
+    let hit = document.elementFromPoint(x, y);
+    const seen = new Set();
+    while (hit?.shadowRoot?.mode === 'open' && !seen.has(hit)) {
+      seen.add(hit);
+      const inner = hit.shadowRoot.elementFromPoint(x, y);
+      if (!inner || inner === hit) break;
+      hit = inner;
+    }
+    return hit;
+  }
+
+  function visualTargetEligibility(el) {
+    const tag = el.tagName?.toLowerCase() || '';
+    if (tag === 'button') return 'semantic-button';
+    if (['canvas', 'iframe', 'label', 'input', 'textarea', 'select'].includes(tag)) {
+      return 'coordinate-only';
+    }
+    return isInteractive(el) ? 'coordinate-only' : '';
+  }
+
+  function resolveVisualTargetAtPoint(x, y) {
+    const cssX = Number(x);
+    const cssY = Number(y);
+    if (!Number.isFinite(cssX) || !Number.isFinite(cssY)) return null;
+
+    for (let el = deepestOpenShadowHit(cssX, cssY); el; el = composedParent(el)) {
+      if (el.nodeType !== Node.ELEMENT_NODE) continue;
+      const eligibility = visualTargetEligibility(el);
+      if (!eligibility) continue;
+      return {
+        ref_id: getOrMintRef(el),
+        role: getRole(el),
+        name: (getAccessibleName(el) || '').slice(0, 160),
+        eligibility,
+      };
+    }
+    return null;
+  }
+
   function isLandmark(el) {
     if (LANDMARK_TAGS.has(el.tagName.toLowerCase())) return true;
     return el.getAttribute('role') !== null;
@@ -1170,5 +1225,6 @@
   window.__wb_ax_ref = getOrMintRef;
   window.__wb_ax_name = getAccessibleName;
   window.__wb_ax_role = getRole;
+  window.__wb_ax_resolve_visual_target = resolveVisualTargetAtPoint;
   window.__wb_ax_suggest = suggestNearRefs;
 })();

src/chrome/src/content/content.js

@@ -5379,6 +5379,29 @@
           return { success: false, verified: false, error: error && error.message || String(error) };
         }
       },
+      'resolve_visual_target': () => {
+        try {
+          const x = Number(msg.params?.x);
+          const y = Number(msg.params?.y);
+          if (!Number.isFinite(x) || !Number.isFinite(y)) {
+            return { success: false, error: 'x and y must be finite numbers' };
+          }
+          if (typeof window.__wb_ax_resolve_visual_target !== 'function') {
+            return { success: false, error: 'accessibility-tree.js not injected' };
+          }
+          const semanticTarget = window.__wb_ax_resolve_visual_target(x, y);
+          return semanticTarget
+            ? {
+                success: true,
+                semanticTarget,
+                documentToken: _axDocumentToken(),
+                refScopeUrl: location.href,
+              }
+            : { success: true };
+        } catch (e) {
+          return { success: false, error: e?.message || String(e) };
+        }
+      },
       // ── ref_id → on-screen rect resolver ─────────────────────────────────
       // Helper for the CDP-backed pointer tools (hover, right_click,
       // drag_drop). The agent calls this from background.js to get viewport

src/firefox/src/agent/agent.js

@@ -6001,6 +6001,140 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
     };
   }
 
+  _coordinateReconciliationDiagnostic(point, resolution, clickPath, fallbackReason) {
+    const target = resolution?.success === true ? resolution.semanticTarget : null;
+    const resolved = clickPath === 'semantic';
+    const role = String(target?.role || '').slice(0, 32);
+    const name = String(target?.name || '').replace(/\s+/g, ' ').trim().slice(0, 120);
+    const targetMetadata = {
+      ...(role ? { role } : {}),
+      ...(name ? { name } : {}),
+    };
+    return {
+      canonicalPoint: { x: Number(point.x), y: Number(point.y) },
+      semanticTargetResolved: resolved,
+      ...(Object.keys(targetMetadata).length ? { target: targetMetadata } : {}),
+      clickPath,
+      fallbackReason,
+    };
+  }
+
+  _withCoordinateReconciliation(result, diagnostic) {
+    return diagnostic && result && typeof result === 'object'
+      ? { ...result, coordinateReconciliation: diagnostic }
+      : result;
+  }
+
+  async _resolveCoordinateVisualTarget(tabId, point) {
+    const send = () => browser.tabs.sendMessage(tabId, {
+      target: 'content',
+      action: 'resolve_visual_target',
+      params: { x: Number(point.x), y: Number(point.y) },
+    });
+    try {
+      try {
+        return await this._withIndicatorsHidden(tabId, send);
+      } catch {
+        await this._injectCoreContentScripts(tabId);
+        return await this._withIndicatorsHidden(tabId, send);
+      }
+    } catch {
+      return { success: false };
+    }
+  }
+
+  async _dispatchClickAx(tabId, args, axScope = null, dispatchBinding = null) {
+    let contentArgs = axScope?.documentToken
+      ? {
+          ...args,
+          expectedDocumentToken: axScope.documentToken,
+          ...(axScope.pageUrl ? { expectedPageUrl: axScope.pageUrl } : {}),
+        }
+      : args;
+    if (dispatchBinding?.token) {
+      contentArgs = { ...contentArgs, dispatchBinding };
+    }
+    const messageOptions = dispatchBinding?.token && Number.isInteger(dispatchBinding.frameId)
+      ? { frameId: dispatchBinding.frameId }
+      : undefined;
+    const send = () => browser.tabs.sendMessage(tabId, {
+      target: 'content',
+      action: 'click_ax',
+      params: contentArgs,
+    }, messageOptions);
+    const finish = async (response) => {
+      response = await this._settleContentFilePickerGuard(tabId, response);
+      if (response?.documentToken && (
+        response.documentChanged === true
+        || response.routeChanged === true
+        || response.staleRef === true
+      )) {
+        this._rememberAxScope(tabId, response.documentToken, response.refScopeUrl || '');
+      }
+      this._annotateCredentialField('click_ax', response);
+      this._clearUploadSelectorRecoveryAfterInspection(tabId, 'click_ax', response);
+      return response;
+    };
+
+    try {
+      return await finish(await send());
+    } catch {
+      try {
+        await this._injectCoreContentScripts(tabId);
+        return await finish(await send());
+      } catch (error) {
+        let pageUrl = '';
+        try { pageUrl = (await browser.tabs.get(tabId))?.url || ''; } catch {}
+        const accessFailure = firefoxHostPermissionFailure(pageUrl, error.message);
+        if (accessFailure) return accessFailure;
+        return { error: `Failed to communicate with page: ${error.message}` };
+      }
+    }
+  }
+
+  async _reconcileCoordinateClick(tabId, point) {
+    const resolution = await this._resolveCoordinateVisualTarget(tabId, point);
+    const target = resolution?.semanticTarget;
+    const semanticEligible = resolution?.success === true
+      && target?.eligibility === 'semantic-button'
+      && target?.role === 'button'
+      && typeof target?.ref_id === 'string'
+      && /^ref_\d+$/.test(target.ref_id);
+    if (semanticEligible) {
+      const result = await this._dispatchClickAx(
+        tabId,
+        { ref_id: target.ref_id },
+        { documentToken: resolution.documentToken, pageUrl: resolution.refScopeUrl },
+      );
+      return {
+        result: {
+          ...result,
+          coordinateReconciliation: this._coordinateReconciliationDiagnostic(
+            point,
+            resolution,
+            'semantic',
+            'none',
+          ),
+        },
+        diagnostic: null,
+      };
+    }
+    const fallbackReason = resolution?.success !== true
+      ? 'resolver-error'
+      : target
+        ? 'coordinate-only-target'
+        : 'no-target';
+    return {
+      result: null,
+      diagnostic: this._coordinateReconciliationDiagnostic(
+        point,
+        resolution,
+        'coordinate-fallback',
+        fallbackReason,
+      ),
+    };
+  }
+
   /**
    * Coordinate-system sentence for screenshot notes shown to the model.
    * Captures are CSS-locked (scale:1) but may be downscaled when a viewport
@@ -15273,6 +15407,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
     const dispatchContext = executionContext && typeof executionContext === 'object'
       ? executionContext
       : {};
+    let coordinatePoint = null;
+    let coordinateDiagnostic = null;
     // Canonicalize coordinate clicks before toolbar recovery probes them.
     // The preflight binding and the eventual dispatch must resolve the same
     // CSS-pixel point, especially when the model clicked a downscaled image.
@@ -15287,7 +15423,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
         };
       }
       const mapped = this._screenshotClickCoords(tabId, args);
-      if (mapped?.converted) args = { ...args, x: mapped.x, y: mapped.y };
+      if (mapped && (mapped.converted || args.from_screenshot === true)) {
+        args = { ...args, x: mapped.x, y: mapped.y };
+      }
+      if (args.from_screenshot === true && mapped) {
+        coordinatePoint = { x: mapped.x, y: mapped.y };
+      }
     }
     const richTextToolbarBlock = await this._richTextToolbarToolBlock(
       tabId,
@@ -15297,6 +15438,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
     );
     if (richTextToolbarBlock) return richTextToolbarBlock;
     let dispatchBinding = dispatchContext.dispatchBinding || null;
+    if (coordinatePoint && dispatchBinding?.token) {
+      coordinateDiagnostic = this._coordinateReconciliationDiagnostic(
+        coordinatePoint,
+        null,
+        'coordinate-fallback',
+        'bound-coordinate-target',
+      );
+    }
     if (name === 'load_skill') {
       return this._loadSkillForRun(tabId, args || {});
     }
@@ -17872,6 +18021,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
       }
     } catch { /* tab lookup failures are non-fatal — fall through */ }
 
+    if (name === 'click_ax') {
+      return this._dispatchClickAx(tabId, args, this._lastAxScopes.get(tabId), dispatchBinding);
+    }
+
     if (name === 'click') {
       const duplicateSubmit = await guardRecentSubmitClick(
         this._recentSubmitClicks,
@@ -17883,10 +18036,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
         },
       );
       if (duplicateSubmit) return duplicateSubmit;
+      if (coordinatePoint && !dispatchBinding?.token) {
+        const reconciled = await this._reconcileCoordinateClick(tabId, coordinatePoint);
+        if (reconciled.result) return reconciled.result;
+        coordinateDiagnostic = reconciled.diagnostic;
+      }
     }
 
     const axScope = this._lastAxScopes.get(tabId);
-    let contentArgs = (name === 'click_ax' || name === 'set_checked') && axScope?.documentToken
+    let contentArgs = name === 'set_checked' && axScope?.documentToken
       ? {
           ...args,
           expectedDocumentToken: axScope.documentToken,
@@ -17918,13 +18076,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
       && Number.isInteger(dispatchBinding.frameId)
       ? { frameId: dispatchBinding.frameId }
       : undefined;
+    const sendContentAction = () => browser.tabs.sendMessage(tabId, {
+      target: 'content',
+      action,
+      params: contentArgs,
+    }, messageOptions);
+    const dispatchContentAction = sendContentAction;
     try {
-      let response = await browser.tabs.sendMessage(tabId, {
-        target: 'content',
-        action,
-        params: contentArgs,
-      }, messageOptions);
-      if (name === 'click' || name === 'click_ax') {
+      let response = await dispatchContentAction();
+      if (name === 'click') {
         response = await this._settleContentFilePickerGuard(tabId, response);
       }
       if (response?.documentToken && (
@@ -17944,17 +18104,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
         response = applyReadPageWindow(response, args);
       }
       this._clearUploadSelectorRecoveryAfterInspection(tabId, name, response);
-      return response;
+      return this._withCoordinateReconciliation(response, coordinateDiagnostic);
     } catch (e) {
       // Content script might not be injected — try injecting it
       try {
         await this._injectCoreContentScripts(tabId);
-        let response = await browser.tabs.sendMessage(tabId, {
-          target: 'content',
-          action,
-          params: contentArgs,
-        }, messageOptions);
-        if (name === 'click' || name === 'click_ax') {
+        let response = await dispatchContentAction();
+        if (name === 'click') {
           response = await this._settleContentFilePickerGuard(tabId, response);
         }
         if (response?.documentToken && (
@@ -17974,13 +18130,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
           response = applyReadPageWindow(response, args);
         }
         this._clearUploadSelectorRecoveryAfterInspection(tabId, name, response);
-        return response;
+        return this._withCoordinateReconciliation(response, coordinateDiagnostic);
       } catch (e2) {
         let pageUrl = '';
         try { pageUrl = (await browser.tabs.get(tabId))?.url || ''; } catch {}
         const accessFailure = firefoxHostPermissionFailure(pageUrl, e2.message);
-        if (accessFailure) return accessFailure;
-        return { error: `Failed to communicate with page: ${e2.message}` };
+        if (accessFailure) return this._withCoordinateReconciliation(accessFailure, coordinateDiagnostic);
+        return this._withCoordinateReconciliation(
+          { error: `Failed to communicate with page: ${e2.message}` },
+          coordinateDiagnostic,
+        );
       }
     }
   }