diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index 47e51f538..684d0b56e 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -8360,6 +8360,165 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } + _coordinateReconciliationDiagnostic(point, resolution, clickPath, fallbackReason) { + const target = resolution?.success === true ? resolution.semanticTarget : null; + const resolved = clickPath === 'semantic'; + const role = String(target?.role || '').slice(0, 32); + const name = String(target?.name || '').replace(/\s+/g, ' ').trim().slice(0, 120); + const targetMetadata = { + ...(role ? { role } : {}), + ...(name ? { name } : {}), + }; + return { + canonicalPoint: { x: Number(point.x), y: Number(point.y) }, + semanticTargetResolved: resolved, + ...(Object.keys(targetMetadata).length ? { target: targetMetadata } : {}), + clickPath, + fallbackReason, + }; + } + + _withCoordinateReconciliation(result, diagnostic) { + return diagnostic && result && typeof result === 'object' + ? { ...result, coordinateReconciliation: diagnostic } + : result; + } + + async _resolveCoordinateVisualTarget(tabId, point) { + const send = () => chrome.tabs.sendMessage(tabId, { + target: 'content', + action: 'resolve_visual_target', + params: { x: Number(point.x), y: Number(point.y) }, + }); + try { + try { + return await this._withIndicatorsHidden(tabId, send); + } catch { + await this._injectCoreContentScripts(tabId); + return await this._withIndicatorsHidden(tabId, send); + } + } catch { + return { success: false }; + } + } + + async _dispatchClickAx(tabId, args, axScope = null, dispatchBinding = null) { + const interactionUrl = await this._currentUrl(tabId); + const clickProgressBefore = await this._clickProgressSnapshot(tabId); + const sideEffectWatch = this._beginClickAxSideEffectWatch(tabId); + let baseline = null; + const captureBaseline = async () => { + baseline = await this._captureClickAxObservation( + tabId, + clickProgressBefore, + sideEffectWatch, + Date.now(), + ); + }; + let contentArgs = axScope?.documentToken + ? { + ...args, + expectedDocumentToken: axScope.documentToken, + ...(axScope.pageUrl ? { expectedPageUrl: axScope.pageUrl } : {}), + } + : args; + if (dispatchBinding?.token) { + contentArgs = { ...contentArgs, dispatchBinding }; + } + const messageOptions = dispatchBinding?.token && Number.isInteger(dispatchBinding.frameId) + ? { frameId: dispatchBinding.frameId } + : undefined; + const send = () => chrome.tabs.sendMessage(tabId, { + target: 'content', + action: 'click_ax', + params: contentArgs, + }, messageOptions); + + try { + let response; + try { + await captureBaseline(); + response = await send(); + } catch { + try { + await this._injectCoreContentScripts(tabId); + await captureBaseline(); + response = await send(); + } catch (error) { + return { error: `Failed to communicate with page: ${error.message}` }; + } + } + response = await this._settleContentFilePickerGuard(tabId, response); + if (response?.documentToken && ( + response.documentChanged === true + || response.routeChanged === true + || response.staleRef === true + )) { + this._rememberAxScope(tabId, response.documentToken, response.refScopeUrl || ''); + } + response = await this._maybeFallbackClickAxWithCdp(tabId, args, response, baseline); + const observedAfterSnapshot = response?._clickAxAfterSnapshot || ''; + if (response) delete response._clickAxAfterSnapshot; + await this._annotateClickProgress( + tabId, + 'click_ax', + args, + response, + clickProgressBefore, + { afterSnapshot: observedAfterSnapshot }, + ); + this._recordInteractionRect(tabId, 'click_ax', response, interactionUrl); + this._annotateCredentialField('click_ax', response); + this._clearUploadSelectorRecoveryAfterInspection(tabId, 'click_ax', response); + return response; + } finally { + sideEffectWatch?.stop(); + } + } + + async _reconcileCoordinateClick(tabId, point) { + const resolution = await this._resolveCoordinateVisualTarget(tabId, point); + const target = resolution?.semanticTarget; + const semanticEligible = resolution?.success === true + && target?.eligibility === 'semantic-button' + && target?.role === 'button' + && typeof target?.ref_id === 'string' + && /^ref_\d+$/.test(target.ref_id); + if (semanticEligible) { + const result = await this._dispatchClickAx( + tabId, + { ref_id: target.ref_id }, + { documentToken: resolution.documentToken, pageUrl: resolution.refScopeUrl }, + ); + return { + result: { + ...result, + coordinateReconciliation: this._coordinateReconciliationDiagnostic( + point, + resolution, + 'semantic', + 'none', + ), + }, + diagnostic: null, + }; + } + const fallbackReason = resolution?.success !== true + ? 'resolver-error' + : target + ? 'coordinate-only-target' + : 'no-target'; + return { + result: null, + diagnostic: this._coordinateReconciliationDiagnostic( + point, + resolution, + 'coordinate-fallback', + fallbackReason, + ), + }; + } + /** * Coerce storage / settings values for image budget (issue #311). Rejects * corrupted or out-of-range values so provider payloads never see e.g. @@ -17386,6 +17545,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const dispatchContext = executionContext && typeof executionContext === 'object' ? executionContext : {}; + let coordinatePoint = null; + let coordinateDiagnostic = null; // Canonicalize coordinate clicks before toolbar recovery probes them. // The preflight binding and the eventual dispatch must resolve the same // CSS-pixel point, especially when the model clicked a downscaled image. @@ -17400,7 +17561,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } const mapped = this._screenshotClickCoords(tabId, args); - if (mapped?.converted) args = { ...args, x: mapped.x, y: mapped.y }; + if (mapped && (mapped.converted || args.from_screenshot === true)) { + args = { ...args, x: mapped.x, y: mapped.y }; + } + if (args.from_screenshot === true && mapped) { + coordinatePoint = { x: mapped.x, y: mapped.y }; + } } const richTextToolbarBlock = await this._richTextToolbarToolBlock( tabId, @@ -17410,6 +17576,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ); if (richTextToolbarBlock) return richTextToolbarBlock; const dispatchBinding = dispatchContext.dispatchBinding || null; + if (coordinatePoint && dispatchBinding?.token) { + coordinateDiagnostic = this._coordinateReconciliationDiagnostic( + coordinatePoint, + null, + 'coordinate-fallback', + 'bound-coordinate-target', + ); + } if (name === 'load_skill') { return this._loadSkillForRun(tabId, args || {}); } @@ -20397,8 +20571,6 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (name === 'click' && !dispatchBinding?.token) { try { - await cdpClient.attach(tabId); - const duplicateSubmit = await guardRecentSubmitClick( this._recentSubmitClicks, tabId, @@ -20410,6 +20582,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ); if (duplicateSubmit) return duplicateSubmit; + if (coordinatePoint) { + const reconciled = await this._reconcileCoordinateClick(tabId, coordinatePoint); + if (reconciled.result) return reconciled.result; + coordinateDiagnostic = reconciled.diagnostic; + } + + await cdpClient.attach(tabId); + // ── Global SELECT guard ───────────────────────────────────────── // Inject a capture-phase mousedown+click listener that prevents // native . const coordTagCheck = await cdpClient.evaluate(tabId, ` (() => { @@ -21399,7 +21580,6 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const opts = Array.from(el.options).map(o => o.text.trim()); return { isSelect: true, current: el.options[el.selectedIndex]?.text?.trim() || '', options: opts }; } - // Find the real input target let target = null; @@ -21532,11 +21712,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }); const coordResponse = { success: true, method: 'cdp-coords', x: args.x, y: args.y }; return await this._annotateClickProgress(tabId, 'click', args, coordResponse, progressBeforeCoord); + })(), coordinateDiagnostic); } // index-based: fall through to content-script path which knows the // interactive-elements ordering. } catch (e) { - return { success: false, error: `Click failed: ${e.message}` }; + return this._withCoordinateReconciliation( + { success: false, error: `Click failed: ${e.message}` }, + coordinateDiagnostic, + ); } } @@ -22249,36 +22433,24 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } } catch { /* tab lookup failures are non-fatal — fall through */ } + if (name === 'click_ax') { + return this._dispatchClickAx(tabId, args, this._lastAxScopes.get(tabId), dispatchBinding); + } + const interactionUrl = ( - name === 'click' || name === 'click_ax' || name === 'set_checked' || + name === 'click' || name === 'set_checked' || name === 'type_ax' || name === 'set_field' ) ? await this._currentUrl(tabId) : ''; if (name === 'scroll') { args = await this._augmentScrollArgsWithLastInteraction(tabId, args); } - const clickProgressBefore = (name === 'click' || name === 'click_ax') + const clickProgressBefore = name === 'click' ? await this._clickProgressSnapshot(tabId) : ''; - // Start network/download listeners early so synchronous click work is not - // missed, but stamp the click_ax safety window only immediately before the - // content-script message that actually runs el.click(). Otherwise a slow - // executeScript inject can push the synthetic click outside the 400ms - // attribution window and skip the network veto. - const clickAxSideEffectWatch = name === 'click_ax' ? this._beginClickAxSideEffectWatch(tabId) : null; - let clickAxBaseline = null; - const captureClickAxBaseline = async () => { - if (name !== 'click_ax') return; - clickAxBaseline = await this._captureClickAxObservation( - tabId, - clickProgressBefore, - clickAxSideEffectWatch, - Date.now(), - ); - }; const axScope = this._lastAxScopes.get(tabId); - let contentArgs = (name === 'click_ax' || name === 'set_checked') && axScope?.documentToken + let contentArgs = name === 'set_checked' && axScope?.documentToken ? { ...args, expectedDocumentToken: axScope.documentToken, @@ -22295,19 +22467,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } + const messageOptions = DISPATCH_BINDING_TOOLS.has(name) + && dispatchBinding?.token + && Number.isInteger(dispatchBinding.frameId) + ? { frameId: dispatchBinding.frameId } + : undefined; + const sendContentAction = () => chrome.tabs.sendMessage(tabId, { + target: 'content', + action, + params: contentArgs, + }, messageOptions); + const dispatchContentAction = sendContentAction; + + let response; try { - let response; - try { - await captureClickAxBaseline(); - response = await chrome.tabs.sendMessage(tabId, { - target: 'content', - action, - params: contentArgs, - }, DISPATCH_BINDING_TOOLS.has(name) - && dispatchBinding?.token - && Number.isInteger(dispatchBinding.frameId) - ? { frameId: dispatchBinding.frameId } - : undefined); + response = await dispatchContentAction(); } catch (e) { // Content script might not be injected — try injecting it. // accessibility-tree.js must load first so content.js's @@ -22315,23 +22489,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // window.__generateAccessibilityTree and window.__wb_ax_lookup. try { await this._injectCoreContentScripts(tabId); - // Re-stamp after inject so the safety window does not include the - // injection gap (which can exceed 400ms on cold tabs). - await captureClickAxBaseline(); - response = await chrome.tabs.sendMessage(tabId, { - target: 'content', - action, - params: contentArgs, - }, DISPATCH_BINDING_TOOLS.has(name) - && dispatchBinding?.token - && Number.isInteger(dispatchBinding.frameId) - ? { frameId: dispatchBinding.frameId } - : undefined); + response = await dispatchContentAction(); } catch (e2) { - return { error: `Failed to communicate with page: ${e2.message}` }; + return this._withCoordinateReconciliation( + { error: `Failed to communicate with page: ${e2.message}` }, + coordinateDiagnostic, + ); } } - if (name === 'click' || name === 'click_ax') { + if (name === 'click') { response = await this._settleContentFilePickerGuard(tabId, response); } if (response?.documentToken && ( @@ -22346,24 +22512,18 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d delete response.refScopeUrl; } } - if (name === 'click_ax') { - response = await this._maybeFallbackClickAxWithCdp(tabId, args, response, clickAxBaseline); - } if (name === 'set_checked') { response = await this._completeSetCheckedWithCdp(tabId, args, response, contentArgs); } if (name === 'type_ax' || name === 'set_field') { response = await this._maybeFallbackFieldWithCdp(tabId, name, args, response); } - const observedAfterSnapshot = response?._clickAxAfterSnapshot || ''; - if (response) delete response._clickAxAfterSnapshot; await this._annotateClickProgress( tabId, name, args, response, clickProgressBefore, - { afterSnapshot: observedAfterSnapshot }, ); this._recordInteractionRect(tabId, name, response, interactionUrl); this._annotateCredentialField(name, response); @@ -22371,10 +22531,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d response = applyReadPageWindow(response, args); } this._clearUploadSelectorRecoveryAfterInspection(tabId, name, response); - return response; - } finally { - clickAxSideEffectWatch?.stop(); - } + return this._withCoordinateReconciliation(response, coordinateDiagnostic); } /** diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.js index 765a0d0aa..2b9d49171 100644 --- a/src/chrome/src/agent/tools.js +++ b/src/chrome/src/agent/tools.js @@ -1684,6 +1684,7 @@ ${PLAN_TO_EXECUTION_GUIDANCE} Available tools: - get_accessibility_tree: PREFERRED read. Flat-text tree of the page with roles, names, and stable ref_ids. Default starting point for almost every turn. - inspect_viewport: Read-only visual inspection when appearance or rendered pixels matter. +- After visual inspection, act on a screenshot-derived point with click({x,y,from_screenshot:true}); WebBrain converts image pixels to CSS pixels mechanically. - click_ax: Click a node by its ref_id from the tree. Preferred over click({text/selector}). - set_checked: Idempotently set a native checkbox by ref_id and verify checkedBefore/checkedAfter. Use this instead of toggling with click_ax. - type_ax: Type into a node by its ref_id from the tree. Preferred over the click-then-type_text pattern. @@ -2044,6 +2045,7 @@ ${PLAN_TO_EXECUTION_GUIDANCE} TOOLS — use only these: - get_accessibility_tree: PREFERRED read. Flat-text tree with roles, names, and stable ref_ids. Use filter:"visible" by default. - inspect_viewport: Read-only visual inspection for ads, images, canvas, charts, and layout. +- After inspect_viewport, act on a screenshot-derived point with click({x,y,from_screenshot:true}); WebBrain converts image pixels to CSS pixels mechanically. - click_ax({ref_id}) / set_checked({ref_id, checked}) / type_ax({ref_id, text}) / set_field({ref_id, text, submit}): act on nodes by ref_id. set_field is preferred for text fields; set_checked is required for native checkboxes. - read_page: prose fallback for long articles. get_window_info: inspect browser window/viewport size. scroll, navigate({url}), go_back()/go_forward(): walk the run tab's history. new_tab({url}) only opens a background reference tab and never retargets the run; promote_iframe({urlFilter}) navigates the current run to one child frame's standalone URL. - get_interactive_elements: legacy indexed element list (use when the tree misses elements). click({text}) / type_text({text}) / press_keys({key}): legacy fallbacks. press_keys supports only unmodified Escape/Tab/Enter/arrows or ; (semicolon), never Ctrl/Cmd/Alt/Shift combinations or browser shortcuts. diff --git a/src/chrome/src/content/accessibility-tree.js b/src/chrome/src/content/accessibility-tree.js index 27684faf3..f63dccd58 100644 --- a/src/chrome/src/content/accessibility-tree.js +++ b/src/chrome/src/content/accessibility-tree.js @@ -484,6 +484,61 @@ return false; } + function composedParent(node) { + if (!node) return null; + if (node.assignedSlot) return node.assignedSlot; + const parent = node.parentNode; + if (parent) { + return (typeof ShadowRoot !== 'undefined' && parent instanceof ShadowRoot) + ? parent.host + : parent; + } + const root = node.getRootNode?.(); + return (typeof ShadowRoot !== 'undefined' && root instanceof ShadowRoot) + ? root.host + : null; + } + + function deepestOpenShadowHit(x, y) { + let hit = document.elementFromPoint(x, y); + const seen = new Set(); + while (hit?.shadowRoot?.mode === 'open' && !seen.has(hit)) { + seen.add(hit); + const inner = hit.shadowRoot.elementFromPoint(x, y); + if (!inner || inner === hit) break; + hit = inner; + } + return hit; + } + + function visualTargetEligibility(el) { + const tag = el.tagName?.toLowerCase() || ''; + if (tag === 'button') return 'semantic-button'; + if (['canvas', 'iframe', 'label', 'input', 'textarea', 'select'].includes(tag)) { + return 'coordinate-only'; + } + return isInteractive(el) ? 'coordinate-only' : ''; + } + + function resolveVisualTargetAtPoint(x, y) { + const cssX = Number(x); + const cssY = Number(y); + if (!Number.isFinite(cssX) || !Number.isFinite(cssY)) return null; + + for (let el = deepestOpenShadowHit(cssX, cssY); el; el = composedParent(el)) { + if (el.nodeType !== Node.ELEMENT_NODE) continue; + const eligibility = visualTargetEligibility(el); + if (!eligibility) continue; + return { + ref_id: getOrMintRef(el), + role: getRole(el), + name: (getAccessibleName(el) || '').slice(0, 160), + eligibility, + }; + } + return null; + } + function isLandmark(el) { if (LANDMARK_TAGS.has(el.tagName.toLowerCase())) return true; return el.getAttribute('role') !== null; @@ -1170,5 +1225,6 @@ window.__wb_ax_ref = getOrMintRef; window.__wb_ax_name = getAccessibleName; window.__wb_ax_role = getRole; + window.__wb_ax_resolve_visual_target = resolveVisualTargetAtPoint; window.__wb_ax_suggest = suggestNearRefs; })(); diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js index f08fc89ba..403662dd7 100644 --- a/src/chrome/src/content/content.js +++ b/src/chrome/src/content/content.js @@ -5379,6 +5379,29 @@ return { success: false, verified: false, error: error && error.message || String(error) }; } }, + 'resolve_visual_target': () => { + try { + const x = Number(msg.params?.x); + const y = Number(msg.params?.y); + if (!Number.isFinite(x) || !Number.isFinite(y)) { + return { success: false, error: 'x and y must be finite numbers' }; + } + if (typeof window.__wb_ax_resolve_visual_target !== 'function') { + return { success: false, error: 'accessibility-tree.js not injected' }; + } + const semanticTarget = window.__wb_ax_resolve_visual_target(x, y); + return semanticTarget + ? { + success: true, + semanticTarget, + documentToken: _axDocumentToken(), + refScopeUrl: location.href, + } + : { success: true }; + } catch (e) { + return { success: false, error: e?.message || String(e) }; + } + }, // ── ref_id → on-screen rect resolver ───────────────────────────────── // Helper for the CDP-backed pointer tools (hover, right_click, // drag_drop). The agent calls this from background.js to get viewport diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 010af8fa3..cb6cda382 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -6001,6 +6001,140 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } + _coordinateReconciliationDiagnostic(point, resolution, clickPath, fallbackReason) { + const target = resolution?.success === true ? resolution.semanticTarget : null; + const resolved = clickPath === 'semantic'; + const role = String(target?.role || '').slice(0, 32); + const name = String(target?.name || '').replace(/\s+/g, ' ').trim().slice(0, 120); + const targetMetadata = { + ...(role ? { role } : {}), + ...(name ? { name } : {}), + }; + return { + canonicalPoint: { x: Number(point.x), y: Number(point.y) }, + semanticTargetResolved: resolved, + ...(Object.keys(targetMetadata).length ? { target: targetMetadata } : {}), + clickPath, + fallbackReason, + }; + } + + _withCoordinateReconciliation(result, diagnostic) { + return diagnostic && result && typeof result === 'object' + ? { ...result, coordinateReconciliation: diagnostic } + : result; + } + + async _resolveCoordinateVisualTarget(tabId, point) { + const send = () => browser.tabs.sendMessage(tabId, { + target: 'content', + action: 'resolve_visual_target', + params: { x: Number(point.x), y: Number(point.y) }, + }); + try { + try { + return await this._withIndicatorsHidden(tabId, send); + } catch { + await this._injectCoreContentScripts(tabId); + return await this._withIndicatorsHidden(tabId, send); + } + } catch { + return { success: false }; + } + } + + async _dispatchClickAx(tabId, args, axScope = null, dispatchBinding = null) { + let contentArgs = axScope?.documentToken + ? { + ...args, + expectedDocumentToken: axScope.documentToken, + ...(axScope.pageUrl ? { expectedPageUrl: axScope.pageUrl } : {}), + } + : args; + if (dispatchBinding?.token) { + contentArgs = { ...contentArgs, dispatchBinding }; + } + const messageOptions = dispatchBinding?.token && Number.isInteger(dispatchBinding.frameId) + ? { frameId: dispatchBinding.frameId } + : undefined; + const send = () => browser.tabs.sendMessage(tabId, { + target: 'content', + action: 'click_ax', + params: contentArgs, + }, messageOptions); + const finish = async (response) => { + response = await this._settleContentFilePickerGuard(tabId, response); + if (response?.documentToken && ( + response.documentChanged === true + || response.routeChanged === true + || response.staleRef === true + )) { + this._rememberAxScope(tabId, response.documentToken, response.refScopeUrl || ''); + } + this._annotateCredentialField('click_ax', response); + this._clearUploadSelectorRecoveryAfterInspection(tabId, 'click_ax', response); + return response; + }; + + try { + return await finish(await send()); + } catch { + try { + await this._injectCoreContentScripts(tabId); + return await finish(await send()); + } catch (error) { + let pageUrl = ''; + try { pageUrl = (await browser.tabs.get(tabId))?.url || ''; } catch {} + const accessFailure = firefoxHostPermissionFailure(pageUrl, error.message); + if (accessFailure) return accessFailure; + return { error: `Failed to communicate with page: ${error.message}` }; + } + } + } + + async _reconcileCoordinateClick(tabId, point) { + const resolution = await this._resolveCoordinateVisualTarget(tabId, point); + const target = resolution?.semanticTarget; + const semanticEligible = resolution?.success === true + && target?.eligibility === 'semantic-button' + && target?.role === 'button' + && typeof target?.ref_id === 'string' + && /^ref_\d+$/.test(target.ref_id); + if (semanticEligible) { + const result = await this._dispatchClickAx( + tabId, + { ref_id: target.ref_id }, + { documentToken: resolution.documentToken, pageUrl: resolution.refScopeUrl }, + ); + return { + result: { + ...result, + coordinateReconciliation: this._coordinateReconciliationDiagnostic( + point, + resolution, + 'semantic', + 'none', + ), + }, + diagnostic: null, + }; + } + const fallbackReason = resolution?.success !== true + ? 'resolver-error' + : target + ? 'coordinate-only-target' + : 'no-target'; + return { + result: null, + diagnostic: this._coordinateReconciliationDiagnostic( + point, + resolution, + 'coordinate-fallback', + fallbackReason, + ), + }; + } + /** * Coordinate-system sentence for screenshot notes shown to the model. * Captures are CSS-locked (scale:1) but may be downscaled when a viewport @@ -15273,6 +15407,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const dispatchContext = executionContext && typeof executionContext === 'object' ? executionContext : {}; + let coordinatePoint = null; + let coordinateDiagnostic = null; // Canonicalize coordinate clicks before toolbar recovery probes them. // The preflight binding and the eventual dispatch must resolve the same // CSS-pixel point, especially when the model clicked a downscaled image. @@ -15287,7 +15423,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } const mapped = this._screenshotClickCoords(tabId, args); - if (mapped?.converted) args = { ...args, x: mapped.x, y: mapped.y }; + if (mapped && (mapped.converted || args.from_screenshot === true)) { + args = { ...args, x: mapped.x, y: mapped.y }; + } + if (args.from_screenshot === true && mapped) { + coordinatePoint = { x: mapped.x, y: mapped.y }; + } } const richTextToolbarBlock = await this._richTextToolbarToolBlock( tabId, @@ -15297,6 +15438,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ); if (richTextToolbarBlock) return richTextToolbarBlock; let dispatchBinding = dispatchContext.dispatchBinding || null; + if (coordinatePoint && dispatchBinding?.token) { + coordinateDiagnostic = this._coordinateReconciliationDiagnostic( + coordinatePoint, + null, + 'coordinate-fallback', + 'bound-coordinate-target', + ); + } if (name === 'load_skill') { return this._loadSkillForRun(tabId, args || {}); } @@ -17872,6 +18021,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } } catch { /* tab lookup failures are non-fatal — fall through */ } + if (name === 'click_ax') { + return this._dispatchClickAx(tabId, args, this._lastAxScopes.get(tabId), dispatchBinding); + } + if (name === 'click') { const duplicateSubmit = await guardRecentSubmitClick( this._recentSubmitClicks, @@ -17883,10 +18036,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }, ); if (duplicateSubmit) return duplicateSubmit; + if (coordinatePoint && !dispatchBinding?.token) { + const reconciled = await this._reconcileCoordinateClick(tabId, coordinatePoint); + if (reconciled.result) return reconciled.result; + coordinateDiagnostic = reconciled.diagnostic; + } } const axScope = this._lastAxScopes.get(tabId); - let contentArgs = (name === 'click_ax' || name === 'set_checked') && axScope?.documentToken + let contentArgs = name === 'set_checked' && axScope?.documentToken ? { ...args, expectedDocumentToken: axScope.documentToken, @@ -17918,13 +18076,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d && Number.isInteger(dispatchBinding.frameId) ? { frameId: dispatchBinding.frameId } : undefined; + const sendContentAction = () => browser.tabs.sendMessage(tabId, { + target: 'content', + action, + params: contentArgs, + }, messageOptions); + const dispatchContentAction = sendContentAction; try { - let response = await browser.tabs.sendMessage(tabId, { - target: 'content', - action, - params: contentArgs, - }, messageOptions); - if (name === 'click' || name === 'click_ax') { + let response = await dispatchContentAction(); + if (name === 'click') { response = await this._settleContentFilePickerGuard(tabId, response); } if (response?.documentToken && ( @@ -17944,17 +18104,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d response = applyReadPageWindow(response, args); } this._clearUploadSelectorRecoveryAfterInspection(tabId, name, response); - return response; + return this._withCoordinateReconciliation(response, coordinateDiagnostic); } catch (e) { // Content script might not be injected — try injecting it try { await this._injectCoreContentScripts(tabId); - let response = await browser.tabs.sendMessage(tabId, { - target: 'content', - action, - params: contentArgs, - }, messageOptions); - if (name === 'click' || name === 'click_ax') { + let response = await dispatchContentAction(); + if (name === 'click') { response = await this._settleContentFilePickerGuard(tabId, response); } if (response?.documentToken && ( @@ -17974,13 +18130,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d response = applyReadPageWindow(response, args); } this._clearUploadSelectorRecoveryAfterInspection(tabId, name, response); - return response; + return this._withCoordinateReconciliation(response, coordinateDiagnostic); } catch (e2) { let pageUrl = ''; try { pageUrl = (await browser.tabs.get(tabId))?.url || ''; } catch {} const accessFailure = firefoxHostPermissionFailure(pageUrl, e2.message); - if (accessFailure) return accessFailure; - return { error: `Failed to communicate with page: ${e2.message}` }; + if (accessFailure) return this._withCoordinateReconciliation(accessFailure, coordinateDiagnostic); + return this._withCoordinateReconciliation( + { error: `Failed to communicate with page: ${e2.message}` }, + coordinateDiagnostic, + ); } } } diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.js index 8622969e6..65bacc38c 100644 --- a/src/firefox/src/agent/tools.js +++ b/src/firefox/src/agent/tools.js @@ -1562,6 +1562,7 @@ ${PLAN_TO_EXECUTION_GUIDANCE} Available tools: - inspect_viewport: Read-only visual inspection when appearance or rendered pixels matter. +- After visual inspection, act on a screenshot-derived point with click({x,y,from_screenshot:true}); WebBrain converts image pixels to CSS pixels mechanically. - read_page: Read the current page content - get_window_info / resize_window: Inspect or resize the browser window for recording/layout tasks. - get_interactive_elements: List all clickable/interactive elements @@ -1800,6 +1801,7 @@ ${PLAN_TO_EXECUTION_GUIDANCE} TOOLS — use only these: - get_accessibility_tree: PREFERRED read. Flat-text tree with roles, names, and stable ref_ids. Use filter:"visible" by default. - inspect_viewport: Read-only visual inspection for ads, images, canvas, charts, and layout. +- After inspect_viewport, act on a screenshot-derived point with click({x,y,from_screenshot:true}); WebBrain converts image pixels to CSS pixels mechanically. - click_ax({ref_id}) / set_checked({ref_id, checked}) / type_ax({ref_id, text}) / set_field({ref_id, text, submit}): act on nodes by ref_id. set_field is preferred for text fields; set_checked is required for native checkboxes. - read_page: prose fallback for long articles. get_window_info: inspect browser window/viewport size. scroll, navigate({url}), go_back()/go_forward(): walk the run tab's history. new_tab({url}) only opens a background reference tab and never retargets the run; promote_iframe({urlFilter}) navigates the current run to one child frame's standalone URL. - get_interactive_elements: legacy indexed element list (use when the tree misses elements). click({text}) / type_text({text}) / press_keys({key}): legacy fallbacks. press_keys supports only unmodified Escape/Tab/Enter/arrows or ; (semicolon), never Ctrl/Cmd/Alt/Shift combinations or browser shortcuts. diff --git a/src/firefox/src/content/accessibility-tree.js b/src/firefox/src/content/accessibility-tree.js index 27684faf3..f63dccd58 100644 --- a/src/firefox/src/content/accessibility-tree.js +++ b/src/firefox/src/content/accessibility-tree.js @@ -484,6 +484,61 @@ return false; } + function composedParent(node) { + if (!node) return null; + if (node.assignedSlot) return node.assignedSlot; + const parent = node.parentNode; + if (parent) { + return (typeof ShadowRoot !== 'undefined' && parent instanceof ShadowRoot) + ? parent.host + : parent; + } + const root = node.getRootNode?.(); + return (typeof ShadowRoot !== 'undefined' && root instanceof ShadowRoot) + ? root.host + : null; + } + + function deepestOpenShadowHit(x, y) { + let hit = document.elementFromPoint(x, y); + const seen = new Set(); + while (hit?.shadowRoot?.mode === 'open' && !seen.has(hit)) { + seen.add(hit); + const inner = hit.shadowRoot.elementFromPoint(x, y); + if (!inner || inner === hit) break; + hit = inner; + } + return hit; + } + + function visualTargetEligibility(el) { + const tag = el.tagName?.toLowerCase() || ''; + if (tag === 'button') return 'semantic-button'; + if (['canvas', 'iframe', 'label', 'input', 'textarea', 'select'].includes(tag)) { + return 'coordinate-only'; + } + return isInteractive(el) ? 'coordinate-only' : ''; + } + + function resolveVisualTargetAtPoint(x, y) { + const cssX = Number(x); + const cssY = Number(y); + if (!Number.isFinite(cssX) || !Number.isFinite(cssY)) return null; + + for (let el = deepestOpenShadowHit(cssX, cssY); el; el = composedParent(el)) { + if (el.nodeType !== Node.ELEMENT_NODE) continue; + const eligibility = visualTargetEligibility(el); + if (!eligibility) continue; + return { + ref_id: getOrMintRef(el), + role: getRole(el), + name: (getAccessibleName(el) || '').slice(0, 160), + eligibility, + }; + } + return null; + } + function isLandmark(el) { if (LANDMARK_TAGS.has(el.tagName.toLowerCase())) return true; return el.getAttribute('role') !== null; @@ -1170,5 +1225,6 @@ window.__wb_ax_ref = getOrMintRef; window.__wb_ax_name = getAccessibleName; window.__wb_ax_role = getRole; + window.__wb_ax_resolve_visual_target = resolveVisualTargetAtPoint; window.__wb_ax_suggest = suggestNearRefs; })(); diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js index e32f0e7b1..3b5e4e969 100644 --- a/src/firefox/src/content/content.js +++ b/src/firefox/src/content/content.js @@ -3653,6 +3653,29 @@ return { error: 'Failed to build accessibility tree: ' + (e && e.message || String(e)) }; } }, + 'resolve_visual_target': () => { + try { + const x = Number(msg.params?.x); + const y = Number(msg.params?.y); + if (!Number.isFinite(x) || !Number.isFinite(y)) { + return { success: false, error: 'x and y must be finite numbers' }; + } + if (typeof window.__wb_ax_resolve_visual_target !== 'function') { + return { success: false, error: 'accessibility-tree.js not injected' }; + } + const semanticTarget = window.__wb_ax_resolve_visual_target(x, y); + return semanticTarget + ? { + success: true, + semanticTarget, + documentToken: _axDocumentToken(), + refScopeUrl: location.href, + } + : { success: true }; + } catch (e) { + return { success: false, error: e?.message || String(e) }; + } + }, 'resolve_form_field_refs': () => { try { if (typeof window.__wb_ax_ref !== 'function') { diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjs index 4e0ad6047..634183f98 100644 --- a/test/fixtures/run.mjs +++ b/test/fixtures/run.mjs @@ -2565,6 +2565,118 @@ for (const browserKind of ['chrome', 'firefox']) { throw new Error(`product context bounds regressed: ${JSON.stringify(result.targetContext)}`); } }); + + test(`resolve_visual_target (${browserKind}): nested SVG resolves semantic button`, async (page) => { + await setupContentHtml(page, ` + + + `, browserKind); + const point = await page.locator('#icon circle').evaluate((el) => { + const r = el.getBoundingClientRect(); + return { x: r.left + r.width / 2, y: r.top + r.height / 2 }; + }); + const result = await call(page, 'resolve_visual_target', point); + if ( + !result?.success + || result.semanticTarget?.role !== 'button' + || result.semanticTarget?.name !== 'Add to cart' + || !/^ref_\d+$/.test(result.semanticTarget?.ref_id || '') + || result.semanticTarget?.eligibility !== 'semantic-button' + || Object.hasOwn(result.semanticTarget, 'rect') + ) { + throw new Error(`nested SVG did not resolve to button: ${JSON.stringify(result)}`); + } + const clickResult = await call(page, 'click_ax', { + ref_id: result.semanticTarget.ref_id, + expectedDocumentToken: result.documentToken, + expectedPageUrl: result.refScopeUrl, + }); + if (!clickResult?.success || await page.evaluate(() => window.__nestedSvgClicked) !== true) { + throw new Error(`nested SVG semantic dispatch did not activate its button: ${JSON.stringify(clickResult)}`); + } + }); + + test(`resolve_visual_target (${browserKind}): plain canvas stays coordinate-only`, async (page) => { + await setupContentHtml(page, '', browserKind); + const point = { x: 70, y: 55 }; + const result = await call(page, 'resolve_visual_target', point); + if ( + !result?.success + || result.semanticTarget?.eligibility !== 'coordinate-only' + || !/^ref_\d+$/.test(result.semanticTarget?.ref_id || '') + || Object.hasOwn(result.semanticTarget, 'rect') + ) { + throw new Error(`canvas should stay coordinate-only: ${JSON.stringify(result)}`); + } + }); + + test(`resolve_visual_target (${browserKind}): form controls and iframe boundaries stay coordinate-only`, async (page) => { + await setupContentHtml(page, ` + + + + + + + + `, browserKind); + + for (const selector of ['#frame', '#label', '#text', '#notes', '#select', '#file']) { + const point = await page.locator(selector).evaluate((el) => { + const r = el.getBoundingClientRect(); + return { x: r.left + r.width / 2, y: r.top + r.height / 2 }; + }); + const result = await call(page, 'resolve_visual_target', point); + if ( + !result?.success + || result.semanticTarget?.eligibility !== 'coordinate-only' + || !/^ref_\d+$/.test(result.semanticTarget?.ref_id || '') + || Object.hasOwn(result.semanticTarget, 'rect') + ) { + throw new Error(`${selector} should stay coordinate-only: ${JSON.stringify(result)}`); + } + } + }); + + test(`resolve_visual_target (${browserKind}): open shadow button resolves`, async (page) => { + await setupContentHtml(page, '
', browserKind); + await page.evaluate(() => { + const root = document.querySelector('#host').attachShadow({ mode: 'open' }); + root.innerHTML = ''; + root.querySelector('button').addEventListener('click', () => { window.__shadowClicked = true; }); + }); + const point = await page.locator('#host').evaluate((host) => { + const r = host.shadowRoot.querySelector('button').getBoundingClientRect(); + return { x: r.left + r.width / 2, y: r.top + r.height / 2 }; + }); + const result = await call(page, 'resolve_visual_target', point); + if ( + !result?.success + || result.semanticTarget?.role !== 'button' + || result.semanticTarget?.name !== 'Shadow action' + || !/^ref_\d+$/.test(result.semanticTarget?.ref_id || '') + || result.semanticTarget?.eligibility !== 'semantic-button' + ) { + throw new Error(`open shadow target was not resolved: ${JSON.stringify(result)}`); + } + const clickResult = await call(page, 'click_ax', { + ref_id: result.semanticTarget.ref_id, + expectedDocumentToken: result.documentToken, + expectedPageUrl: result.refScopeUrl, + }); + if (!clickResult?.success || await page.evaluate(() => window.__shadowClicked) !== true) { + throw new Error(`open shadow semantic dispatch did not activate its button: ${JSON.stringify(clickResult)}`); + } + }); } test('set_field (chrome): trusted contenteditable input updates framework state and enables submit', async (page) => { diff --git a/test/run.js b/test/run.js index 03c890a2d..494b242e6 100644 --- a/test/run.js +++ b/test/run.js @@ -10484,6 +10484,711 @@ test('screenshot click scale: from_screenshot converts image px to CSS px', () = } }); +test('visual target resolution stays private across Chrome and Firefox tools and prompts', () => { + for (const [label, getTools, prompts] of [ + ['chrome', getToolsForModeCh, [ + SYSTEM_PROMPT_ASK_CH, + SYSTEM_PROMPT_ACT_CH, + SYSTEM_PROMPT_ACT_MID_CH, + SYSTEM_PROMPT_ACT_COMPACT_CH, + SYSTEM_PROMPT_DEV_APPENDIX_CH, + ]], + ['firefox', getToolsForModeFx, [ + SYSTEM_PROMPT_ASK_FX, + SYSTEM_PROMPT_ACT_FX, + SYSTEM_PROMPT_ACT_MID_FX, + SYSTEM_PROMPT_ACT_COMPACT_FX, + SYSTEM_PROMPT_DEV_APPENDIX_FX, + ]], + ]) { + const namesFor = (mode, tier) => new Set( + getTools(mode, { tier }).map(tool => tool.function.name), + ); + for (const [mode, tiers] of [ + ['ask', ['compact', 'mid', 'full']], + ['act', ['compact', 'mid', 'full']], + ['dev', ['compact', 'mid', 'full']], + ]) { + for (const tier of tiers) { + assert.equal(namesFor(mode, tier).has('resolve_visual_target'), false, `${label}: ${mode}/${tier} must omit visual target resolution`); + } + } + const promptText = prompts.join('\n'); + assert.doesNotMatch(promptText, /resolve_visual_target/, `${label}: prompts must not mention the private resolver`); + assert.match(promptText, /click\(\{x,y,from_screenshot:true\}\)/, `${label}: prompts must route screenshot points through click directly`); + } +}); + +async function runCoordinateSemanticCase({ + label, + AgentClass, + globalKey, + resolverResponse, + clickAxResponse = null, + cachedAxScope = null, + chromeAttachError = null, + dispatchBinding = null, +}) { + const previousChrome = globalThis.chrome; + const previousBrowser = globalThis.browser; + const originalCdpAttach = cdpClientCh.attach; + const resolveParams = []; + const clickAxParams = []; + const fallbackParams = []; + const sendMessage = async (_tabId, message) => { + if (message.type === 'WB_HIDE_FOR_TOOL_USE' || message.type === 'WB_SHOW_AFTER_TOOL_USE') return {}; + if (message.action === 'resolve_visual_target') { + resolveParams.push(message.params); + return resolverResponse; + } + if (message.action === 'click_ax') { + clickAxParams.push(message.params); + return clickAxResponse || { success: true, method: 'click_ax', ref_id: message.params.ref_id }; + } + if (message.action === 'click') { + fallbackParams.push(message.params); + return { success: true, method: 'coordinate-fallback' }; + } + throw new Error(`${label}: unexpected content message ${message.action || message.type}`); + }; + const tabs = { + get: async () => ({ url: 'https://example.test/' }), + sendMessage, + }; + globalThis[globalKey] = globalKey === 'chrome' + ? { ...(previousChrome || {}), tabs: { ...(previousChrome?.tabs || {}), ...tabs } } + : { ...(previousBrowser || {}), tabs: { ...(previousBrowser?.tabs || {}), ...tabs } }; + try { + const agent = new AgentClass({}); + const tabId = label === 'chrome' ? 8811 : 8812; + agent._isPdfTab = async () => false; + agent._richTextToolbarToolBlock = async () => null; + agent._recentSubmitClicks = null; + agent._settleContentFilePickerGuard = async (_tabId, response) => response; + if (cachedAxScope) agent._lastAxScopes.set(tabId, cachedAxScope); + if (label === 'chrome') { + cdpClientCh.attach = async () => { + if (chromeAttachError) throw new Error(chromeAttachError); + }; + agent._currentUrl = async () => 'https://example.test/'; + agent._clickProgressSnapshot = async () => ''; + agent._annotateClickProgress = async (_tabId, _name, _args, response) => response; + agent._beginClickAxSideEffectWatch = () => ({ stop() {} }); + agent._captureClickAxObservation = async () => ({}); + agent._maybeFallbackClickAxWithCdp = async (_tabId, _args, response) => response; + agent._recordInteractionRect = () => {}; + } + const mapScreenshotCoords = agent._screenshotClickCoords.bind(agent); + let mappingCalls = 0; + agent._screenshotClickCoords = (...args) => { + mappingCalls += 1; + return mapScreenshotCoords(...args); + }; + agent._setScreenshotClickScale(tabId, 2560 / 1568, 1440 / 882); + const result = await agent.executeTool(tabId, 'click', { + x: 784, + y: 441, + from_screenshot: true, + }, null, dispatchBinding ? { dispatchBinding } : undefined); + return { result, mappingCalls, resolveParams, clickAxParams, fallbackParams }; + } finally { + cdpClientCh.attach = originalCdpAttach; + if (globalKey === 'chrome') { + if (previousChrome === undefined) delete globalThis.chrome; + else globalThis.chrome = previousChrome; + } else if (previousBrowser === undefined) delete globalThis.browser; + else globalThis.browser = previousBrowser; + } +} + +test('coordinate semantic reconciliation: toolbar-bound coordinate dispatch stays coordinate-only and diagnostic', async () => { + const dispatchBinding = { token: 'toolbar-coordinate-binding', frameId: 7 }; + for (const [label, AgentClass, globalKey] of [ + ['chrome', AgentCh, 'chrome'], + ['firefox', AgentFx, 'browser'], + ]) { + const observed = await runCoordinateSemanticCase({ + label, + AgentClass, + globalKey, + resolverResponse: { success: true }, + dispatchBinding, + }); + assert.equal(observed.mappingCalls, 1, `${label}: bound coordinate mapping must run exactly once`); + assert.deepEqual(observed.resolveParams, [], `${label}: a preserved toolbar binding must not be replaced by resolver targeting`); + assert.equal(observed.fallbackParams.length, 1, `${label}: bound coordinate click must keep content coordinate dispatch`); + assert.deepEqual(observed.result.coordinateReconciliation, { + canonicalPoint: { x: 1280, y: 720 }, + semanticTargetResolved: false, + clickPath: 'coordinate-fallback', + fallbackReason: 'bound-coordinate-target', + }); + } +}); + +test('coordinate semantic reconciliation: screenshot click converts once and routes eligible buttons through click_ax', async () => { + const semanticTarget = { + ref_id: 'ref_1231', + role: 'button', + name: 'SVG parent action', + eligibility: 'semantic-button', + }; + for (const [label, AgentClass, globalKey] of [ + ['chrome', AgentCh, 'chrome'], + ['firefox', AgentFx, 'browser'], + ]) { + const observed = await runCoordinateSemanticCase({ + label, + AgentClass, + globalKey, + resolverResponse: { + success: true, + semanticTarget, + documentToken: 'resolver-document-token', + refScopeUrl: 'https://example.test/current-route', + }, + cachedAxScope: { + documentToken: 'stale-document-token', + pageUrl: 'https://example.test/previous-route', + }, + }); + assert.equal(observed.mappingCalls, 1, `${label}: coordinate mapping must run exactly once`); + assert.deepEqual(observed.resolveParams, [{ x: 1280, y: 720 }], `${label}: resolver must receive the canonical CSS point`); + assert.deepEqual(observed.clickAxParams, [{ + ref_id: 'ref_1231', + expectedDocumentToken: 'resolver-document-token', + expectedPageUrl: 'https://example.test/current-route', + }], `${label}: semantic button must route through click_ax with its resolver scope`); + assert.equal(observed.fallbackParams.length, 0); + assert.deepEqual(observed.result.coordinateReconciliation, { + canonicalPoint: { x: 1280, y: 720 }, + semanticTargetResolved: true, + target: { role: 'button', name: 'SVG parent action' }, + clickPath: 'semantic', + fallbackReason: 'none', + }); + } +}); + +test('coordinate semantic reconciliation: Chrome semantic buttons do not require coordinate CDP attachment', async () => { + const observed = await runCoordinateSemanticCase({ + label: 'chrome', + AgentClass: AgentCh, + globalKey: 'chrome', + resolverResponse: { + success: true, + semanticTarget: { + ref_id: 'ref_904', + role: 'button', + name: 'Semantic action without CDP', + eligibility: 'semantic-button', + }, + documentToken: 'resolver-token', + refScopeUrl: 'https://example.test/', + }, + chromeAttachError: 'synthetic debugger already attached', + }); + + assert.deepEqual(observed.resolveParams, [{ x: 1280, y: 720 }]); + assert.equal(observed.clickAxParams[0].ref_id, 'ref_904'); + assert.equal(observed.result.success, true); + assert.equal(observed.result.method, 'click_ax'); +}); + +test('coordinate semantic reconciliation: canvas miss and resolver error keep the coordinate fallback', async () => { + for (const [resolverResponse, fallbackReason] of [ + [{ success: true }, 'no-target'], + [{ success: false, error: 'synthetic resolver failure', rawDom: '' }, 'resolver-error'], + ]) { + for (const [label, AgentClass, globalKey] of [ + ['chrome', AgentCh, 'chrome'], + ['firefox', AgentFx, 'browser'], + ]) { + const observed = await runCoordinateSemanticCase({ label, AgentClass, globalKey, resolverResponse }); + assert.equal(observed.mappingCalls, 1, `${label}: fallback coordinate mapping must run exactly once`); + assert.deepEqual(observed.resolveParams, [{ x: 1280, y: 720 }]); + assert.deepEqual(observed.clickAxParams, []); + assert.equal(observed.fallbackParams.length, label === 'firefox' ? 1 : 0); + assert.deepEqual(observed.result.coordinateReconciliation, { + canonicalPoint: { x: 1280, y: 720 }, + semanticTargetResolved: false, + clickPath: 'coordinate-fallback', + fallbackReason, + }); + assert.equal(JSON.stringify(observed.result.coordinateReconciliation).includes('synthetic resolver failure'), false); + assert.equal(JSON.stringify(observed.result.coordinateReconciliation).includes('rawDom'), false); + } + } +}); + +test('coordinate semantic reconciliation: coordinate-only semantic targets preserve coordinate behavior', async () => { + const observed = await runCoordinateSemanticCase({ + label: 'firefox', + AgentClass: AgentFx, + globalKey: 'browser', + resolverResponse: { + success: true, + semanticTarget: { + ref_id: 'ref_902', + role: 'textbox', + name: 'Label behavior unchanged'.repeat(40), + eligibility: 'coordinate-only', + }, + }, + }); + assert.deepEqual(observed.clickAxParams, []); + assert.equal(observed.fallbackParams.length, 1); + assert.deepEqual(observed.result.coordinateReconciliation, { + canonicalPoint: { x: 1280, y: 720 }, + semanticTargetResolved: false, + target: { role: 'textbox', name: 'Label behavior unchanged'.repeat(40).slice(0, 120) }, + clickPath: 'coordinate-fallback', + fallbackReason: 'coordinate-only-target', + }); + assert.equal(JSON.stringify(observed.result.coordinateReconciliation).includes('ref_902'), false); +}); + +test('coordinate semantic reconciliation: plain legacy coordinates never invoke the resolver or emit diagnostics', async () => { + const previousChrome = globalThis.chrome; + const previousBrowser = globalThis.browser; + const originalCdpAttach = cdpClientCh.attach; + for (const [label, AgentClass, globalKey] of [ + ['chrome', AgentCh, 'chrome'], + ['firefox', AgentFx, 'browser'], + ]) { + const contentClicks = []; + const tabs = { + get: async () => ({ url: 'https://example.test/' }), + sendMessage: async (_tabId, message) => { + if (message.action === 'click') { + contentClicks.push(message.params); + return { success: true, method: 'legacy-coordinate' }; + } + throw new Error(`${label}: unexpected content message ${message.action || message.type}`); + }, + }; + globalThis[globalKey] = globalKey === 'chrome' + ? { ...(previousChrome || {}), tabs: { ...(previousChrome?.tabs || {}), ...tabs } } + : { ...(previousBrowser || {}), tabs: { ...(previousBrowser?.tabs || {}), ...tabs } }; + try { + const agent = new AgentClass({}); + const tabId = label === 'chrome' ? 8821 : 8822; + agent._isPdfTab = async () => false; + agent._richTextToolbarToolBlock = async () => null; + agent._recentSubmitClicks = null; + agent._settleContentFilePickerGuard = async (_tabId, response) => response; + let resolverCalls = 0; + agent._resolveCoordinateVisualTarget = async () => { + resolverCalls += 1; + return { success: true }; + }; + const mapCoordinates = agent._screenshotClickCoords.bind(agent); + let mappingCalls = 0; + agent._screenshotClickCoords = (...args) => { + mappingCalls += 1; + return mapCoordinates(...args); + }; + agent._setScreenshotClickScale(tabId, 2, 2); + if (label === 'chrome') { + cdpClientCh.attach = async () => { throw new Error('stop after legacy routing check'); }; + } + const result = await agent.executeTool(tabId, 'click', { x: 784, y: 441 }); + assert.equal(mappingCalls, 1, `${label}: legacy coordinate validation should remain single-pass`); + assert.equal(resolverCalls, 0, `${label}: plain coordinate clicks must not enter reconciliation`); + assert.equal(Object.hasOwn(result, 'coordinateReconciliation'), false, `${label}: legacy results must keep their old shape`); + if (label === 'firefox') { + assert.deepEqual(contentClicks, [{ x: 784, y: 441 }], 'Firefox must dispatch the unchanged legacy point'); + } + } finally { + cdpClientCh.attach = originalCdpAttach; + if (globalKey === 'chrome') { + if (previousChrome === undefined) delete globalThis.chrome; + else globalThis.chrome = previousChrome; + } else if (previousBrowser === undefined) delete globalThis.browser; + else globalThis.browser = previousBrowser; + } + } +}); + +test('coordinate semantic reconciliation: stale semantic dispatch never retries by coordinate', async () => { + for (const [label, AgentClass, globalKey] of [ + ['chrome', AgentCh, 'chrome'], + ['firefox', AgentFx, 'browser'], + ]) { + const observed = await runCoordinateSemanticCase({ + label, + AgentClass, + globalKey, + resolverResponse: { + success: true, + semanticTarget: { ref_id: 'ref_905', role: 'button', name: 'Rerendered action', eligibility: 'semantic-button' }, + documentToken: 'resolver-token', + refScopeUrl: 'https://example.test/', + }, + clickAxResponse: { success: false, dispatched: false, noDispatch: true, staleRef: true, error: 'stale ref_id' }, + }); + assert.equal(observed.clickAxParams.length, 1, `${label}: AX dispatch should start once`); + assert.equal(observed.fallbackParams.length, 0, `${label}: coordinate retry is forbidden after AX dispatch begins`); + assert.equal(observed.result.success, false); + assert.equal(observed.result.staleRef, true); + assert.equal(observed.result.coordinateReconciliation.clickPath, 'semantic'); + assert.equal(observed.result.coordinateReconciliation.fallbackReason, 'none'); + } +}); + +test('click_ax preserves rich-text toolbar dispatch bindings after helper extraction', async () => { + const previousChrome = globalThis.chrome; + const previousBrowser = globalThis.browser; + for (const [label, AgentClass, globalKey] of [ + ['chrome', AgentCh, 'chrome'], + ['firefox', AgentFx, 'browser'], + ]) { + const messages = []; + const tabs = { + get: async () => ({ url: 'https://example.test/' }), + sendMessage: async (_tabId, message, options) => { + messages.push({ message, options }); + return { success: true, ref_id: 'ref_906' }; + }, + }; + globalThis[globalKey] = globalKey === 'chrome' + ? { ...(previousChrome || {}), tabs: { ...(previousChrome?.tabs || {}), ...tabs } } + : { ...(previousBrowser || {}), tabs: { ...(previousBrowser?.tabs || {}), ...tabs } }; + try { + const agent = new AgentClass({}); + agent._isPdfTab = async () => false; + agent._richTextToolbarToolBlock = async () => null; + agent._settleContentFilePickerGuard = async (_tabId, response) => response; + if (label === 'chrome') { + agent._currentUrl = async () => 'https://example.test/'; + agent._clickProgressSnapshot = async () => ''; + agent._beginClickAxSideEffectWatch = () => ({ stop() {} }); + agent._captureClickAxObservation = async () => ({}); + agent._maybeFallbackClickAxWithCdp = async (_tabId, _args, response) => response; + agent._annotateClickProgress = async () => {}; + agent._recordInteractionRect = () => {}; + } + const dispatchBinding = { token: 'toolbar-binding', frameId: 7, ref_id: 'ref_906' }; + await agent.executeTool(90, 'click_ax', { ref_id: 'ref_906' }, null, { dispatchBinding }); + const clickMessage = messages.find(entry => entry.message.action === 'click_ax'); + assert.deepEqual(clickMessage?.message.params.dispatchBinding, dispatchBinding, `${label}: click_ax binding was dropped`); + assert.deepEqual(clickMessage?.options, { frameId: 7 }, `${label}: click_ax frame binding was dropped`); + } finally { + if (globalKey === 'chrome') { + if (previousChrome === undefined) delete globalThis.chrome; + else globalThis.chrome = previousChrome; + } else if (previousBrowser === undefined) delete globalThis.browser; + else globalThis.browser = previousBrowser; + } + } +}); + +test('public click_ax preserves the complete pre-extraction dispatch and post-processing contract', async () => { + const previousChrome = globalThis.chrome; + const previousBrowser = globalThis.browser; + for (const [label, AgentClass, globalKey] of [ + ['chrome', AgentCh, 'chrome'], + ['firefox', AgentFx, 'browser'], + ]) { + const events = []; + const response = { + success: true, + ref_id: 'ref_907', + documentToken: 'fresh-document-token', + refScopeUrl: 'https://example.test/fresh-route', + routeChanged: true, + }; + const tabs = { + get: async () => ({ url: 'https://example.test/fresh-route' }), + sendMessage: async (_tabId, message) => { + if (message.action !== 'click_ax') throw new Error(`${label}: unexpected ${message.action}`); + events.push('dispatch'); + assert.equal(message.params.expectedDocumentToken, 'cached-document-token'); + assert.equal(message.params.expectedPageUrl, 'https://example.test/cached-route'); + return response; + }, + }; + globalThis[globalKey] = globalKey === 'chrome' + ? { ...(previousChrome || {}), tabs: { ...(previousChrome?.tabs || {}), ...tabs } } + : { ...(previousBrowser || {}), tabs: { ...(previousBrowser?.tabs || {}), ...tabs } }; + try { + const agent = new AgentClass({}); + const tabId = label === 'chrome' ? 8831 : 8832; + agent._isPdfTab = async () => false; + agent._richTextToolbarToolBlock = async () => null; + agent._lastAxScopes.set(tabId, { + documentToken: 'cached-document-token', + pageUrl: 'https://example.test/cached-route', + }); + agent._settleContentFilePickerGuard = async (_tabId, value) => { + events.push('file-picker-settle'); + return value; + }; + agent._annotateCredentialField = (toolName, value) => { + events.push('credential-annotation'); + assert.equal(toolName, 'click_ax'); + assert.equal(value, response); + }; + agent._clearUploadSelectorRecoveryAfterInspection = (_tabId, toolName, value) => { + events.push('upload-recovery-clear'); + assert.equal(toolName, 'click_ax'); + assert.equal(value, response); + }; + if (label === 'chrome') { + agent._currentUrl = async () => { + events.push('current-url'); + return 'https://example.test/fresh-route'; + }; + agent._clickProgressSnapshot = async () => { + events.push('progress-before'); + return 'before'; + }; + agent._beginClickAxSideEffectWatch = () => ({ + stop() { events.push('side-effect-watch-stop'); }, + }); + agent._captureClickAxObservation = async () => { + events.push('baseline'); + return { snapshot: 'before' }; + }; + agent._maybeFallbackClickAxWithCdp = async (_tabId, clickArgs, value) => { + events.push('cdp-fallback'); + assert.deepEqual(clickArgs, { ref_id: 'ref_907' }); + value._clickAxAfterSnapshot = 'after'; + return value; + }; + agent._annotateClickProgress = async (_tabId, toolName, clickArgs, value, before, options) => { + events.push('progress-verify'); + assert.equal(toolName, 'click_ax'); + assert.deepEqual(clickArgs, { ref_id: 'ref_907' }); + assert.equal(value, response); + assert.equal(before, 'before'); + assert.deepEqual(options, { afterSnapshot: 'after' }); + }; + agent._recordInteractionRect = (_tabId, toolName, value, url) => { + events.push('interaction-rect'); + assert.equal(toolName, 'click_ax'); + assert.equal(value, response); + assert.equal(url, 'https://example.test/fresh-route'); + }; + } + + const result = await agent.executeTool(tabId, 'click_ax', { ref_id: 'ref_907' }); + assert.equal(result.documentToken, 'fresh-document-token', `${label}: click_ax response token must remain public`); + assert.equal(result.refScopeUrl, 'https://example.test/fresh-route', `${label}: click_ax scope URL must remain public`); + assert.deepEqual(agent._lastAxScopes.get(tabId), { + documentToken: 'fresh-document-token', + pageUrl: 'https://example.test/fresh-route', + }); + for (const required of ['dispatch', 'file-picker-settle', 'credential-annotation', 'upload-recovery-clear']) { + assert.equal(events.includes(required), true, `${label}: missing old click_ax stage ${required}`); + } + if (label === 'chrome') { + for (const required of ['baseline', 'cdp-fallback', 'progress-verify', 'interaction-rect', 'side-effect-watch-stop']) { + assert.equal(events.includes(required), true, `chrome: missing old click_ax stage ${required}`); + } + assert.equal(Object.hasOwn(result, '_clickAxAfterSnapshot'), false); + } + } finally { + if (globalKey === 'chrome') { + if (previousChrome === undefined) delete globalThis.chrome; + else globalThis.chrome = previousChrome; + } else if (previousBrowser === undefined) delete globalThis.browser; + else globalThis.browser = previousBrowser; + } + } +}); + +test('coordinate semantic reconciliation: Chrome label fallback keeps the existing input redirect', async () => { + const previousChrome = globalThis.chrome; + const originalCdp = { + attach: cdpClientCh.attach, + evaluate: cdpClientCh.evaluate, + armFileInputClickGuard: cdpClientCh.armFileInputClickGuard, + dispatchMouseEvent: cdpClientCh.dispatchMouseEvent, + consumeFileInputClickGuard: cdpClientCh.consumeFileInputClickGuard, + }; + const clickAxParams = []; + const dispatched = []; + let evaluateCall = 0; + globalThis.chrome = { + ...(previousChrome || {}), + tabs: { + ...(previousChrome?.tabs || {}), + get: async () => ({ url: 'https://example.test/' }), + query: async () => [], + sendMessage: async (_tabId, message) => { + if (message.type === 'WB_HIDE_FOR_TOOL_USE' || message.type === 'WB_SHOW_AFTER_TOOL_USE') return {}; + if (message.action === 'resolve_visual_target') { + return { + success: true, + semanticTarget: { + ref_id: 'ref_903', + role: 'textbox', + name: 'Label behavior unchanged', + eligibility: 'coordinate-only', + }, + }; + } + if (message.action === 'click_ax') { + clickAxParams.push(message.params); + return { success: true }; + } + throw new Error(`chrome: unexpected content message ${message.action || message.type}`); + }, + }, + }; + cdpClientCh.attach = async () => {}; + cdpClientCh.evaluate = async () => { + evaluateCall += 1; + if (evaluateCall === 1) return { result: { value: undefined } }; // select guard injection + if (evaluateCall === 2) return { result: { value: { isSelect: false } } }; // coordinate select probe + if (evaluateCall === 3) return { result: { value: { x: 1390, y: 748, tag: 'INPUT' } } }; // label redirect + if (evaluateCall === 4) return { result: { value: null } }; // post-click select probe + throw new Error(`unexpected CDP evaluate call ${evaluateCall}`); + }; + cdpClientCh.armFileInputClickGuard = async () => {}; + cdpClientCh.dispatchMouseEvent = async (_tabId, type, x, y) => { dispatched.push({ type, x, y }); }; + cdpClientCh.consumeFileInputClickGuard = async () => ({ blocked: false }); + + try { + const agent = new AgentCh({}); + const tabId = 8813; + agent._isPdfTab = async () => false; + agent._richTextToolbarToolBlock = async () => null; + agent._currentUrl = async () => 'https://example.test/'; + agent._clickProgressSnapshot = async () => ''; + agent._annotateClickProgress = async (_tabId, _name, _args, response) => response; + agent._redirectTargetBlankClick = async () => ({ redirected: false }); + agent._showAgentTarget = () => {}; + agent._setScreenshotClickScale(tabId, 2560 / 1568, 1440 / 882); + const result = await agent.executeTool(tabId, 'click', { x: 784, y: 441, from_screenshot: true }); + + assert.equal(result.success, true); + assert.deepEqual(clickAxParams, []); + assert.deepEqual(dispatched, [ + { type: 'mouseMoved', x: 1390, y: 748 }, + { type: 'mousePressed', x: 1390, y: 748 }, + { type: 'mouseReleased', x: 1390, y: 748 }, + ]); + assert.deepEqual(result.coordinateReconciliation, { + canonicalPoint: { x: 1280, y: 720 }, + semanticTargetResolved: false, + target: { role: 'textbox', name: 'Label behavior unchanged' }, + clickPath: 'coordinate-fallback', + fallbackReason: 'coordinate-only-target', + }); + } finally { + Object.assign(cdpClientCh, originalCdp); + if (previousChrome === undefined) delete globalThis.chrome; + else globalThis.chrome = previousChrome; + } +}); + +test('coordinate semantic reconciliation: Chrome canvas fallback preserves the legacy nearby-input heuristic', async () => { + const previousChrome = globalThis.chrome; + const originalCdp = { + attach: cdpClientCh.attach, + evaluate: cdpClientCh.evaluate, + armFileInputClickGuard: cdpClientCh.armFileInputClickGuard, + dispatchMouseEvent: cdpClientCh.dispatchMouseEvent, + consumeFileInputClickGuard: cdpClientCh.consumeFileInputClickGuard, + }; + const dispatched = []; + let inputFocusCalls = 0; + let evaluateCall = 0; + const input = { + tagName: 'INPUT', + focus: () => { inputFocusCalls += 1; }, + scrollIntoView: () => {}, + getBoundingClientRect: () => ({ left: 1300, top: 730, width: 180, height: 36 }), + parentElement: null, + }; + const grid = { + tagName: 'DIV', + querySelector: () => input, + parentElement: null, + }; + const card = { + tagName: 'SECTION', + querySelector: () => null, + parentElement: grid, + }; + const targetArea = { + tagName: 'DIV', + querySelector: () => null, + parentElement: card, + }; + const canvas = { + tagName: 'CANVAS', + closest: () => null, + querySelector: () => null, + parentElement: targetArea, + }; + const document = { elementFromPoint: () => canvas }; + + globalThis.chrome = { + ...(previousChrome || {}), + tabs: { + ...(previousChrome?.tabs || {}), + get: async () => ({ url: 'https://example.test/' }), + query: async () => [], + sendMessage: async (_tabId, message) => { + if (message.type === 'WB_HIDE_FOR_TOOL_USE' || message.type === 'WB_SHOW_AFTER_TOOL_USE') return {}; + if (message.action === 'resolve_visual_target') { + return { success: true, cssPoint: { x: 1280, y: 720 } }; + } + throw new Error(`chrome: unexpected content message ${message.action || message.type}`); + }, + }, + }; + cdpClientCh.attach = async () => {}; + cdpClientCh.evaluate = async (_tabId, expression) => { + evaluateCall += 1; + if (evaluateCall === 1) return { result: { value: undefined } }; // select guard injection + if (evaluateCall === 2) return { result: { value: { isSelect: false } } }; // coordinate select probe + if (evaluateCall === 3) { + const value = Function('document', `return (${expression});`)(document); + return { result: { value } }; + } + if (evaluateCall === 4) return { result: { value: null } }; // post-click select probe + throw new Error(`unexpected CDP evaluate call ${evaluateCall}`); + }; + cdpClientCh.armFileInputClickGuard = async () => {}; + cdpClientCh.dispatchMouseEvent = async (_tabId, type, x, y) => { dispatched.push({ type, x, y }); }; + cdpClientCh.consumeFileInputClickGuard = async () => ({ blocked: false }); + + try { + const agent = new AgentCh({}); + const tabId = 8814; + agent._isPdfTab = async () => false; + agent._richTextToolbarToolBlock = async () => null; + agent._currentUrl = async () => 'https://example.test/'; + agent._clickProgressSnapshot = async () => ''; + agent._annotateClickProgress = async (_tabId, _name, _args, response) => response; + agent._redirectTargetBlankClick = async () => ({ redirected: false }); + agent._showAgentTarget = () => {}; + agent._setScreenshotClickScale(tabId, 2560 / 1568, 1440 / 882); + const result = await agent.executeTool(tabId, 'click', { x: 784, y: 441, from_screenshot: true }); + + assert.equal(inputFocusCalls, 1, 'canvas fallback must preserve the old nearby-input focus heuristic'); + assert.deepEqual(dispatched, [ + { type: 'mouseMoved', x: 1390, y: 748 }, + { type: 'mousePressed', x: 1390, y: 748 }, + { type: 'mouseReleased', x: 1390, y: 748 }, + ]); + assert.deepEqual(result.coordinateReconciliation, { + canonicalPoint: { x: 1280, y: 720 }, + semanticTargetResolved: false, + clickPath: 'coordinate-fallback', + fallbackReason: 'no-target', + }); + } finally { + Object.assign(cdpClientCh, originalCdp); + if (previousChrome === undefined) delete globalThis.chrome; + else globalThis.chrome = previousChrome; + } +}); + test('chrome screenshot tool saves pre-budget data URL when save:true', () => { // Structural: save path must prefer saveDataUrl (full CSS) over budgeted dataUrl. const source = fs.readFileSync(path.join(ROOT, 'src/chrome/src/agent/agent.js'), 'utf8');