From 42fa8e5534c36408df8d96d9fa904bc283d0d138 Mon Sep 17 00:00:00 2001 From: Dara Adedeji <76637177+SunkenInTime@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:09:53 -0400 Subject: [PATCH] Fix strict-CSP screenshots and editor activation --- CHANGELOG.md | 5 ++ extension/firefox-compat.js | 95 ++++++++++++++++++++------- tests/fixtures/README.md | 8 ++- tests/fixtures/cua-ax-functions.json | 4 +- tests/test-firefox-lifecycle-live.mjs | 29 +++++++- 5 files changed, 114 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21b7bec..e88ea5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +- Fix strict-CSP screenshot capture by measuring layout and device-pixel ratio without page eval. +- Support the computer-use scroll and activation fallbacks for covered editors without page eval. + ## 1.4.12 - 2026-09-10 - Fix strict-CSP failures in computer-use element clicks by running the AX hit-test and remote-object cleanup without page eval. diff --git a/extension/firefox-compat.js b/extension/firefox-compat.js index 678503a..3a940e7 100644 --- a/extension/firefox-compat.js +++ b/extension/firefox-compat.js @@ -1160,6 +1160,60 @@ } return {}; } + case "layoutMetrics": { + const width = window.innerWidth; + const height = window.innerHeight; + const contentWidth = Math.max(document.documentElement?.scrollWidth ?? 0, document.body?.scrollWidth ?? 0, width); + const contentHeight = Math.max(document.documentElement?.scrollHeight ?? 0, document.body?.scrollHeight ?? 0, height); + return { + layoutViewport: { pageX: window.scrollX, pageY: window.scrollY, clientWidth: width, clientHeight: height }, + visualViewport: { offsetX: 0, offsetY: 0, pageX: window.scrollX, pageY: window.scrollY, clientWidth: width, clientHeight: height, scale: 1, zoom: 1 }, + contentSize: { x: 0, y: 0, width: contentWidth, height: contentHeight }, + cssLayoutViewport: { pageX: window.scrollX, pageY: window.scrollY, clientWidth: width, clientHeight: height }, + cssVisualViewport: { offsetX: 0, offsetY: 0, pageX: window.scrollX, pageY: window.scrollY, clientWidth: width, clientHeight: height, scale: 1, zoom: 1 }, + cssContentSize: { x: 0, y: 0, width: contentWidth, height: contentHeight }, + }; + } + case "viewportSize": + return { width: window.innerWidth, height: window.innerHeight }; + case "devicePixelRatio": + return window.devicePixelRatio; + case "accessibilityScroll": + case "accessibilityActivate": { + try { + const node = nodeFromPayload(); + const element = node instanceof Element ? node : node?.parentElement; + if (operation === "accessibilityScroll") { + if (element == null) throw new Error("Cannot scroll a detached accessibility element"); + Element.prototype.scrollIntoView.call(element, { behavior: "instant", block: "center", inline: "center" }); + } else { + const control = element?.closest('a[href],button,input,select,textarea,label,[role="button"],[role="link"],[role="checkbox"],[role="radio"],[role="switch"]'); + const associated = control instanceof HTMLLabelElement ? control.control : control; + const target = associated ?? element?.closest('[contenteditable="true"],[contenteditable=""],[contenteditable="plaintext-only"]') ?? element; + if (!target?.isConnected) throw new Error("Cannot interact with a detached element"); + if (!(target instanceof HTMLElement)) throw new Error("Accessibility element does not support programmatic activation"); + if (target.matches(":disabled")) throw new Error("Cannot interact with a disabled element"); + if ((!associated && target.isContentEditable) || target instanceof HTMLTextAreaElement || (target instanceof HTMLInputElement && !["button", "submit", "reset", "checkbox", "radio", "image", "file", "hidden", "color", "range"].includes(target.type))) { + if ((target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement) && target.readOnly) throw new Error("Cannot focus a read-only element"); + target.focus({ preventScroll: true }); + const root = target.getRootNode(); + const focused = root instanceof ShadowRoot ? root.activeElement : target.ownerDocument.activeElement; + if (focused !== target && !focused?.contains(target)) throw new Error("Could not focus the target element"); + } else { + target.click(); + } + } + return { result: { type: "undefined" } }; + } catch (error) { + return { + result: { type: "undefined" }, + exceptionDetails: { + text: error?.message ?? String(error), lineNumber: 0, columnNumber: 0, + exception: { type: "object", subtype: "error", className: error?.name ?? "Error", description: String(error) }, + }, + }; + } + } case "boxModel": { const node = nodeFromPayload(); if (!(node instanceof Element)) throw new Error("DOM element not found"); @@ -2107,6 +2161,9 @@ const returnByValue = params.returnByValue === true; const awaitPromise = params.awaitPromise === true; const objectGroup = typeof params.objectGroup === "string" ? params.objectGroup : null; + if (expression === "window.devicePixelRatio") { + return { result: byValueRemote(await executeCspSafePageOperation(tabId, "devicePixelRatio", {}, { frameId })) }; + } const browserUseBindingName = browserUseBindingNameFromExpression(expression); if (browserUseBindingName != null) { const installed = bindingNamesByTab.get(tabId)?.has(browserUseBindingName) === true; @@ -2239,6 +2296,20 @@ && declaration.includes("viewportSize:")) { return executeCspSafePageOperation(tabId, "accessibilityHitTest", { objectId: params.objectId }, { frameId }); } + if (params.objectId && params.returnByValue === true + && declaration.startsWith("function (argument) { const __name = (target) => target; return (") + && declaration.endsWith(").call(this, argument); }")) { + let operation; + if (declaration.includes("Cannot scroll a detached accessibility element") + && declaration.includes("Element.prototype.scrollIntoView.call")) { + operation = "accessibilityScroll"; + } else if (declaration.includes("Accessibility element does not support programmatic activation") + && declaration.includes("Cannot focus a read-only element") + && declaration.includes("Could not focus the target element")) { + operation = "accessibilityActivate"; + } + if (operation) return executeCspSafePageOperation(tabId, operation, { objectId: params.objectId }, { frameId }); + } const argumentsJson = JSON.stringify(params.arguments ?? []); const returnByValue = params.returnByValue === true; const awaitPromise = params.awaitPromise === true; @@ -2598,31 +2669,11 @@ } async function getLayoutMetrics(tabId, frameId = 0) { - return executeUserScript( - tabId, - operationScript(` - const width = window.innerWidth; - const height = window.innerHeight; - const contentWidth = Math.max(document.documentElement?.scrollWidth ?? 0, document.body?.scrollWidth ?? 0, width); - const contentHeight = Math.max(document.documentElement?.scrollHeight ?? 0, document.body?.scrollHeight ?? 0, height); - return { - layoutViewport: { pageX: window.scrollX, pageY: window.scrollY, clientWidth: width, clientHeight: height }, - visualViewport: { offsetX: 0, offsetY: 0, pageX: window.scrollX, pageY: window.scrollY, clientWidth: width, clientHeight: height, scale: 1, zoom: 1 }, - contentSize: { x: 0, y: 0, width: contentWidth, height: contentHeight }, - cssLayoutViewport: { pageX: window.scrollX, pageY: window.scrollY, clientWidth: width, clientHeight: height }, - cssVisualViewport: { offsetX: 0, offsetY: 0, pageX: window.scrollX, pageY: window.scrollY, clientWidth: width, clientHeight: height, scale: 1, zoom: 1 }, - cssContentSize: { x: 0, y: 0, width: contentWidth, height: contentHeight }, - }; - `), - { frameId }, - ); + return executeCspSafePageOperation(tabId, "layoutMetrics", {}, { frameId }); } async function readViewportSize(tabId) { - return executeUserScript( - tabId, - operationScript(`return { width: window.innerWidth, height: window.innerHeight };`), - ); + return executeCspSafePageOperation(tabId, "viewportSize", {}); } async function resizeWindowForViewport(windowId, tabId, viewport, state) { diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index ac75144..d690d76 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -1,8 +1,8 @@ # Computer-use protocol fixtures -`cua-ax-functions.json` records the node-bound hit-test callback sent by the +`cua-ax-functions.json` records the node-bound hit-test, scrolling, and activation callbacks sent by the installed Browser Use runtime (26.903.71938) during `tab.click(elementIndex)`. -It is passed unchanged to Runtime.callFunctionOn in the real Firefox test. +They are passed unchanged to Runtime.callFunctionOn in the real Firefox test. The static equivalent in firefox-compat.js preserves its hit-test semantics without compiling JavaScript inside the website's CSP realm. @@ -11,3 +11,7 @@ and DOM protocol, runs this callback, dispatches mouse events, inserts text, and checks the input value and foreground-tab preservation. To demonstrate the regression against a prior adapter, set FIREFOX_COMPAT_SOURCE to that adapter's path when running npm run test:live. + +The live test also covers CSP-safe layout metrics, device-pixel ratio, PNG capture, +and multiline replacement in a covered textarea below the viewport. The fixture +loads its CSS from the same origin so its overlay is enforced under strict CSP. diff --git a/tests/fixtures/cua-ax-functions.json b/tests/fixtures/cua-ax-functions.json index 1e97c83..267c0aa 100644 --- a/tests/fixtures/cua-ax-functions.json +++ b/tests/fixtures/cua-ax-functions.json @@ -1,3 +1,5 @@ { - "hitTest": "function (argument) { const __name = (target) => target; return (function nX(){let e=this instanceof Element?this:this.parentElement;if(e==null)throw new Error(\"Cannot hit-test a detached accessibility element\");let t=e.getClientRects()[0]??e.getBoundingClientRect();if(this instanceof Text){let A=document.createRange();A.selectNode(this),t=A.getClientRects()[0]??A.getBoundingClientRect()}let r={x:t.left+t.width/2,y:t.top+t.height/2},n=e.ownerDocument.defaultView;if(n==null)throw new Error(\"Cannot hit-test an accessibility element without a viewport\");let o=n.visualViewport,i=o?.offsetLeft??0,s=o?.offsetTop??0,a=Math.max(t.left,i),l=Math.min(t.right,i+(o?.width??n.innerWidth)),c=Math.max(t.top,s),p=Math.min(t.bottom,s+(o?.height??n.innerHeight)),m=l>a&&p>c?{x:(a+l)/2,y:(c+p)/2}:r,h=e instanceof HTMLInputElement||e instanceof HTMLButtonElement||e instanceof HTMLSelectElement||e instanceof HTMLTextAreaElement?e:void 0,y=[e,...Array.from(h?.labels??[])],w=[\"button\",\"a[href]\",\"input\",\"select\",\"textarea\",'[role=\"button\"]','[role=\"checkbox\"]','[role=\"combobox\"]','[role=\"link\"]','[role=\"listbox\"]','[role=\"menuitem\"]','[role=\"option\"]','[role=\"radio\"]','[role=\"slider\"]','[role=\"spinbutton\"]','[role=\"switch\"]','[role=\"tab\"]','[role=\"textbox\"]','[tabindex]:not([tabindex=\"-1\"])'].join(\",\"),x=(A,L=m.x,$=m.y)=>{let U=A.getRootNode(),X=U instanceof ShadowRoot?U.elementFromPoint(L,$):A.ownerDocument.elementFromPoint(L,$);for(;X?.shadowRoot!=null;){let oe=X.shadowRoot.elementFromPoint(L,$);if(oe==null||oe===X)break;X=oe}return X},S=(A,L,$=!1)=>{let U=!1;for(let X=L;X!=null;){if(X===A)return!$||U;let oe=X instanceof Element||X instanceof Text?X.assignedSlot:null;if(oe!=null)U=!0,X=oe;else if(X.parentNode!=null)X=X.parentNode;else{let me=X.getRootNode(),se=me instanceof ShadowRoot?me.host:null;U||=se!=null,X=se}}return!1},D=(A,L=m.x,$=m.y)=>{let U=x(A,L,$);if(U==null)return!1;let X=U.closest(w),oe=(A instanceof HTMLLabelElement?A.control:null)??A.closest(w);if(oe!=null&&X!=null&&X!==oe&&!S(X,oe))return!1;if(S(A,U)||S(U,A,!0))return!0;let me=A.closest(\"label\")?.control;return me!=null&&(S(U,me)||S(me,U)||A===e&&U.parentElement===me.parentElement&&(X==null||S(X,me)))||oe!=null&&oe===X?!0:getComputedStyle(A).pointerEvents===\"none\"&&S(U,A)},T=y.some(A=>D(A));return{fallbackPoint:(T?null:y.slice(1).flatMap(A=>Array.from(A.getClientRects()).map(L=>({candidate:A,point:{x:L.left+L.width/2,y:L.top+L.height/2},size:{height:L.height,width:L.width}}))).find(({candidate:A,point:L,size:$})=>$.width>1&&$.height>1&&D(A,L.x,L.y))?.point)??null,hitTag:x(e)?.tagName??null,hitsTarget:T,point:m,viewportSize:{height:n.innerHeight,width:n.innerWidth}}}).call(this, argument); }" + "hitTest": "function (argument) { const __name = (target) => target; return (function nX(){let e=this instanceof Element?this:this.parentElement;if(e==null)throw new Error(\"Cannot hit-test a detached accessibility element\");let t=e.getClientRects()[0]??e.getBoundingClientRect();if(this instanceof Text){let A=document.createRange();A.selectNode(this),t=A.getClientRects()[0]??A.getBoundingClientRect()}let r={x:t.left+t.width/2,y:t.top+t.height/2},n=e.ownerDocument.defaultView;if(n==null)throw new Error(\"Cannot hit-test an accessibility element without a viewport\");let o=n.visualViewport,i=o?.offsetLeft??0,s=o?.offsetTop??0,a=Math.max(t.left,i),l=Math.min(t.right,i+(o?.width??n.innerWidth)),c=Math.max(t.top,s),p=Math.min(t.bottom,s+(o?.height??n.innerHeight)),m=l>a&&p>c?{x:(a+l)/2,y:(c+p)/2}:r,h=e instanceof HTMLInputElement||e instanceof HTMLButtonElement||e instanceof HTMLSelectElement||e instanceof HTMLTextAreaElement?e:void 0,y=[e,...Array.from(h?.labels??[])],w=[\"button\",\"a[href]\",\"input\",\"select\",\"textarea\",'[role=\"button\"]','[role=\"checkbox\"]','[role=\"combobox\"]','[role=\"link\"]','[role=\"listbox\"]','[role=\"menuitem\"]','[role=\"option\"]','[role=\"radio\"]','[role=\"slider\"]','[role=\"spinbutton\"]','[role=\"switch\"]','[role=\"tab\"]','[role=\"textbox\"]','[tabindex]:not([tabindex=\"-1\"])'].join(\",\"),x=(A,L=m.x,$=m.y)=>{let U=A.getRootNode(),X=U instanceof ShadowRoot?U.elementFromPoint(L,$):A.ownerDocument.elementFromPoint(L,$);for(;X?.shadowRoot!=null;){let oe=X.shadowRoot.elementFromPoint(L,$);if(oe==null||oe===X)break;X=oe}return X},S=(A,L,$=!1)=>{let U=!1;for(let X=L;X!=null;){if(X===A)return!$||U;let oe=X instanceof Element||X instanceof Text?X.assignedSlot:null;if(oe!=null)U=!0,X=oe;else if(X.parentNode!=null)X=X.parentNode;else{let me=X.getRootNode(),se=me instanceof ShadowRoot?me.host:null;U||=se!=null,X=se}}return!1},D=(A,L=m.x,$=m.y)=>{let U=x(A,L,$);if(U==null)return!1;let X=U.closest(w),oe=(A instanceof HTMLLabelElement?A.control:null)??A.closest(w);if(oe!=null&&X!=null&&X!==oe&&!S(X,oe))return!1;if(S(A,U)||S(U,A,!0))return!0;let me=A.closest(\"label\")?.control;return me!=null&&(S(U,me)||S(me,U)||A===e&&U.parentElement===me.parentElement&&(X==null||S(X,me)))||oe!=null&&oe===X?!0:getComputedStyle(A).pointerEvents===\"none\"&&S(U,A)},T=y.some(A=>D(A));return{fallbackPoint:(T?null:y.slice(1).flatMap(A=>Array.from(A.getClientRects()).map(L=>({candidate:A,point:{x:L.left+L.width/2,y:L.top+L.height/2},size:{height:L.height,width:L.width}}))).find(({candidate:A,point:L,size:$})=>$.width>1&&$.height>1&&D(A,L.x,L.y))?.point)??null,hitTag:x(e)?.tagName??null,hitsTarget:T,point:m,viewportSize:{height:n.innerHeight,width:n.innerWidth}}}).call(this, argument); }", + "scroll": "function (argument) { const __name = (target) => target; return (function eX(){let e=this instanceof Element?this:this.parentElement;if(e==null)throw new Error(\"Cannot scroll a detached accessibility element\");Element.prototype.scrollIntoView.call(e,{behavior:\"instant\",block:\"center\",inline:\"center\"})}).call(this, argument); }", + "activate": "function (argument) { const __name = (target) => target; return (function tX(){let e=this instanceof Element?this:this.parentElement,t=e?.closest('a[href],button,input,select,textarea,label,[role=\"button\"],[role=\"link\"],[role=\"checkbox\"],[role=\"radio\"],[role=\"switch\"]'),r=t instanceof HTMLLabelElement?t.control:t,n=r??e?.closest('[contenteditable=\"true\"],[contenteditable=\"\"],[contenteditable=\"plaintext-only\"]')??e;if(!n?.isConnected)throw new Error(\"Cannot interact with a detached element\");if(!(n instanceof HTMLElement))throw new Error(\"Accessibility element does not support programmatic activation\");if(n.matches(\":disabled\"))throw new Error(\"Cannot interact with a disabled element\");if(!r&&n.isContentEditable||n instanceof HTMLTextAreaElement||n instanceof HTMLInputElement&&![\"button\",\"submit\",\"reset\",\"checkbox\",\"radio\",\"image\",\"file\",\"hidden\",\"color\",\"range\"].includes(n.type)){if((n instanceof HTMLInputElement||n instanceof HTMLTextAreaElement)&&n.readOnly)throw new Error(\"Cannot focus a read-only element\");n.focus({preventScroll:!0});let i=n.getRootNode(),s=i instanceof ShadowRoot?i.activeElement:n.ownerDocument.activeElement;if(s!==n&&!s?.contains(n))throw new Error(\"Could not focus the target element\")}else n.click()}).call(this, argument); }" } diff --git a/tests/test-firefox-lifecycle-live.mjs b/tests/test-firefox-lifecycle-live.mjs index 2e119cf..cec577b 100644 --- a/tests/test-firefox-lifecycle-live.mjs +++ b/tests/test-firefox-lifecycle-live.mjs @@ -45,11 +45,14 @@ const server = http.createServer((req, res) => { res.end("ok"); finish(JSON.parse(body)); }); + } else if (req.url === "/fixture.css") { + res.setHeader("content-type", "text/css"); + res.end(".editor {position:relative;margin-top:200vh}.overlay {position:absolute;inset:0;background:white}"); } else { res.setHeader("content-type", "text/html"); res.setHeader("Content-Security-Policy", "default-src 'self'; script-src 'self'; object-src 'none'"); res.end( - "Lifecycle fixture

isolated lifecycle test

", + "Lifecycle fixture

isolated lifecycle test

Editor overlay
", ); } }); @@ -105,6 +108,28 @@ for(const modifiers of [4,2]){ const cleared=await chrome.debugger.sendCommand(debuggee,'Accessibility.getFullAXTree',{}); if(cleared.nodes.find(n=>n.name?.value==='Repository search')?.value?.value!=='')throw Error('Select-all and Backspace did not clear input (modifiers='+modifiers+')'); } +const metrics=await chrome.debugger.sendCommand(debuggee,'Page.getLayoutMetrics',{}); +if(!(metrics.cssVisualViewport.clientWidth>0&&metrics.cssContentSize.height>metrics.cssVisualViewport.clientHeight))throw Error('Strict-CSP layout metrics missing document dimensions'); +const dpr=await chrome.debugger.sendCommand(debuggee,'Runtime.evaluate',{expression:'window.devicePixelRatio',returnByValue:true}); +if(!(dpr.result.value>0))throw Error('Strict-CSP device pixel ratio missing'); +const screenshot=await chrome.debugger.sendCommand(debuggee,'Page.captureScreenshot',{format:'png'}); +if(!screenshot.data?.startsWith('iVBOR'))throw Error('Strict-CSP screenshot missing PNG data'); +const editor=ax.nodes.find(n=>n.name?.value==='Code editor'); +const editorHandle=await chrome.debugger.sendCommand(debuggee,'DOM.resolveNode',{backendNodeId:editor.backendDOMNodeId}); +for(const functionDeclaration of [${JSON.stringify(axFunctions.scroll)},${JSON.stringify(axFunctions.activate)}]){ + const response=await chrome.debugger.sendCommand(debuggee,'Runtime.callFunctionOn',{objectId:editorHandle.object.objectId,functionDeclaration,arguments:[{value:{}}],returnByValue:true,userGesture:true}); + if(response.exceptionDetails||response.result.type!=='undefined')throw Error('AX editor fallback failed: '+JSON.stringify(response)); +} +for(const type of ['keyDown','keyUp'])await chrome.debugger.sendCommand(debuggee,'Input.dispatchKeyEvent',{type,key:'a',code:'KeyA',modifiers:4}); +await chrome.debugger.sendCommand(debuggee,'Input.insertText',{text:'maintenance: |\\n npm install'}); +const edited=await chrome.debugger.sendCommand(debuggee,'Accessibility.getFullAXTree',{}); +if(edited.nodes.find(n=>n.name?.value==='Code editor')?.value?.value!=='maintenance: |\\n npm install')throw Error('Covered editor replacement failed'); +const readOnly=ax.nodes.find(n=>n.name?.value==='Read-only editor'); +const readOnlyHandle=await chrome.debugger.sendCommand(debuggee,'DOM.resolveNode',{backendNodeId:readOnly.backendDOMNodeId}); +const denied=await chrome.debugger.sendCommand(debuggee,'Runtime.callFunctionOn',{objectId:readOnlyHandle.object.objectId,functionDeclaration:${JSON.stringify(axFunctions.activate)},arguments:[{value:{}}],returnByValue:true}); +if(!denied.exceptionDetails?.text.includes('read-only'))throw Error('Editor fallback must reject read-only controls'); +await chrome.debugger.sendCommand(debuggee,'Runtime.releaseObject',{objectId:readOnlyHandle.object.objectId}); +await chrome.debugger.sendCommand(debuggee,'Runtime.releaseObject',{objectId:editorHandle.object.objectId}); await chrome.debugger.sendCommand(debuggee,'Runtime.releaseObject',{objectId:resolved.object.objectId}); const released=await chrome.debugger.sendCommand(debuggee,'Runtime.callFunctionOn',{objectId:resolved.object.objectId,functionDeclaration:${JSON.stringify(axFunctions.hitTest)},arguments:[{}],returnByValue:true}); if(released.result?.type!=='undefined'||!released.exceptionDetails?.text)throw Error('Released handle must return CDP exceptionDetails'); @@ -114,7 +139,7 @@ const releasedGroup=await chrome.debugger.sendCommand(debuggee,'Runtime.callFunc if(!releasedGroup.exceptionDetails?.text)throw Error('Group cleanup retained the handle'); const activeAfter=(await browser.tabs.query({active:true,currentWindow:true}))[0]; if(activeAfter.id!==foreground.id||activations.includes(target.id))throw Error('CSP click activated background tab'); -await fetch(${JSON.stringify(url + "/result")},{method:'POST',body:JSON.stringify({ok:true,backgroundTabPreserved:true,strictCspAxClickTypeAndClear:true,objectCleanupAndExceptionContract:true,lifecycle:lifecycle.map(e=>e.params.name),loaderId:lastLoad.params.loaderId})}); +await fetch(${JSON.stringify(url + "/result")},{method:'POST',body:JSON.stringify({ok:true,backgroundTabPreserved:true,strictCspAxClickTypeAndClear:true,objectCleanupAndExceptionContract:true,strictCspScreenshotAndEditor:true,lifecycle:lifecycle.map(e=>e.params.name),loaderId:lastLoad.params.loaderId})}); }catch(e){await fetch(${JSON.stringify(url + "/result")},{method:'POST',body:JSON.stringify({ok:false,error:String(e)+' '+e.stack})});}})();`, ); const child = spawn(