diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 9cb41a2..49418f6 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -32,3 +32,20 @@ jobs: with: name: release-candidate path: dist/* + + live-firefox: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: 24 + - uses: browser-actions/setup-firefox@v1 + id: firefox + - name: Real Firefox strict-CSP click, type, and lifecycle regression + env: + FIREFOX_BINARY: ${{ steps.firefox.outputs.firefox-path }} + run: npm run test:live diff --git a/CHANGELOG.md b/CHANGELOG.md index 508ae39..c16ded1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +- Fix strict-CSP failures in computer-use element clicks by running the AX hit-test and remote-object cleanup without page eval. +- Run the real Firefox click-and-type regression in pull-request CI, including background-tab preservation. + This project follows semantic versioning. The Firefox add-on and native companion always share one version and are released from a matching `vMAJOR.MINOR.PATCH` Git tag. diff --git a/PORT_STATUS.md b/PORT_STATUS.md index 8fbe390..0430fd0 100644 --- a/PORT_STATUS.md +++ b/PORT_STATUS.md @@ -87,3 +87,14 @@ The isolated headless Zen smoke test passed on macOS on 2026-09-10: real WebExtension attach, lifecycle notifications, matching loader IDs, and unchanged foreground tab. The signed-in Codex transport has not been retested in the user's profile; the installed add-on remains untouched. + +## Strict-CSP computer-use click follow-up + +A live Devin test exposed an uncovered Runtime.callFunctionOn path: AX element +clicks run a node-bound hit test before dispatching input. That helper and runtime +object cleanup now use static scripting.executeScript operations. No site CSP is +changed. The actual Codex computer-use connection clicked, typed into, and cleared +a strict-CSP fixture in an isolated Zen profile. The live Firefox regression runs +in PR CI and reproduces the original eval-blocked error with the 1.4.11 adapter. +This change addresses AX element clicks; arbitrary dynamic Runtime functions and +unsupported Playwright locator helpers remain outside this static adapter path. diff --git a/extension/firefox-compat.js b/extension/firefox-compat.js index d612547..678503a 100644 --- a/extension/firefox-compat.js +++ b/extension/firefox-compat.js @@ -1007,6 +1007,159 @@ }], }; } + // Static equivalent of the Browser Use AX hit-test callback. The callback + // must run through executeScript, never eval in the page's CSP realm. + case "accessibilityHitTest": { + function hitTestAccessibilityNode() { + let e = this instanceof Element ? this : this.parentElement; + if (e == null) + throw new Error("Cannot hit-test a detached accessibility element"); + let t = e.getClientRects()[0] ?? e.getBoundingClientRect(); + if (this instanceof Text) { + let A = document.createRange(); + (A.selectNode(this), + (t = A.getClientRects()[0] ?? A.getBoundingClientRect())); + } + let r = { x: t.left + t.width / 2, y: t.top + t.height / 2 }, + n = e.ownerDocument.defaultView; + if (n == null) + throw new Error( + "Cannot hit-test an accessibility element without a viewport", + ); + let o = n.visualViewport, + i = o?.offsetLeft ?? 0, + s = o?.offsetTop ?? 0, + a = Math.max(t.left, i), + l = Math.min(t.right, i + (o?.width ?? n.innerWidth)), + c = Math.max(t.top, s), + p = Math.min(t.bottom, s + (o?.height ?? n.innerHeight)), + m = l > a && p > c ? { x: (a + l) / 2, y: (c + p) / 2 } : r, + h = + e instanceof HTMLInputElement || + e instanceof HTMLButtonElement || + e instanceof HTMLSelectElement || + e instanceof HTMLTextAreaElement + ? e + : void 0, + y = [e, ...Array.from(h?.labels ?? [])], + w = [ + "button", + "a[href]", + "input", + "select", + "textarea", + '[role="button"]', + '[role="checkbox"]', + '[role="combobox"]', + '[role="link"]', + '[role="listbox"]', + '[role="menuitem"]', + '[role="option"]', + '[role="radio"]', + '[role="slider"]', + '[role="spinbutton"]', + '[role="switch"]', + '[role="tab"]', + '[role="textbox"]', + '[tabindex]:not([tabindex="-1"])', + ].join(","), + x = (A, L = m.x, $ = m.y) => { + let U = A.getRootNode(), + X = + U instanceof ShadowRoot + ? U.elementFromPoint(L, $) + : A.ownerDocument.elementFromPoint(L, $); + for (; X?.shadowRoot != null;) { + let oe = X.shadowRoot.elementFromPoint(L, $); + if (oe == null || oe === X) break; + X = oe; + } + return X; + }, + S = (A, L, $ = !1) => { + let U = !1; + for (let X = L; X != null;) { + if (X === A) return !$ || U; + let oe = + X instanceof Element || X instanceof Text ? X.assignedSlot : null; + if (oe != null) ((U = !0), (X = oe)); + else if (X.parentNode != null) X = X.parentNode; + else { + let me = X.getRootNode(), + se = me instanceof ShadowRoot ? me.host : null; + ((U ||= se != null), (X = se)); + } + } + return !1; + }, + D = (A, L = m.x, $ = m.y) => { + let U = x(A, L, $); + if (U == null) return !1; + let X = U.closest(w), + oe = (A instanceof HTMLLabelElement ? A.control : null) ?? A.closest(w); + if (oe != null && X != null && X !== oe && !S(X, oe)) return !1; + if (S(A, U) || S(U, A, !0)) return !0; + let me = A.closest("label")?.control; + return (me != null && + (S(U, me) || + S(me, U) || + (A === e && + U.parentElement === me.parentElement && + (X == null || S(X, me))))) || + (oe != null && oe === X) + ? !0 + : getComputedStyle(A).pointerEvents === "none" && S(U, A); + }, + T = y.some((A) => D(A)); + return { + fallbackPoint: + (T + ? null + : y + .slice(1) + .flatMap((A) => + Array.from(A.getClientRects()).map((L) => ({ + candidate: A, + point: { x: L.left + L.width / 2, y: L.top + L.height / 2 }, + size: { height: L.height, width: L.width }, + })), + ) + .find( + ({ candidate: A, point: L, size: $ }) => + $.width > 1 && $.height > 1 && D(A, L.x, L.y), + )?.point) ?? null, + hitTag: x(e)?.tagName ?? null, + hitsTarget: T, + point: m, + viewportSize: { height: n.innerHeight, width: n.innerWidth }, + }; + } + try { + return { result: { type: "object", value: hitTestAccessibilityNode.call(nodeFromPayload()) } }; + } catch (error) { + return { + result: { type: "undefined" }, + exceptionDetails: { + text: error?.message ?? String(error), lineNumber: 0, columnNumber: 0, + exception: { type: "object", subtype: "error", className: error?.name ?? "Error", description: String(error) }, + }, + }; + } + } + case "releaseObject": { + state.objects.delete(payload.objectId); + state.objectGroups.delete(payload.objectId); + return {}; + } + case "releaseObjectGroup": { + for (const [id, group] of state.objectGroups) { + if (group === payload.objectGroup) { + state.objects.delete(id); + state.objectGroups.delete(id); + } + } + return {}; + } case "boxModel": { const node = nodeFromPayload(); if (!(node instanceof Element)) throw new Error("DOM element not found"); @@ -2075,6 +2228,17 @@ async function callRuntimeFunction(tabId, params, frameId = 0) { const declaration = stripSourceUrl(params.functionDeclaration ?? "function () {}"); + // Browser Use's AX click first calls a read-only, node-bound hit test. + // Recognize that helper before the generic dynamic-function fallback. + if (params.objectId && params.returnByValue === true + && declaration.startsWith("function (argument) { const __name = (target) => target; return (") + && declaration.endsWith(").call(this, argument); }") + && declaration.includes("Cannot hit-test a detached accessibility element") + && declaration.includes("Cannot hit-test an accessibility element without a viewport") + && declaration.includes("fallbackPoint:") && declaration.includes("hitsTarget:") + && declaration.includes("viewportSize:")) { + return executeCspSafePageOperation(tabId, "accessibilityHitTest", { objectId: params.objectId }, { frameId }); + } const argumentsJson = JSON.stringify(params.arguments ?? []); const returnByValue = params.returnByValue === true; const awaitPromise = params.awaitPromise === true; @@ -3529,19 +3693,9 @@ case "Runtime.runIfWaitingForDebugger": return {}; case "Runtime.releaseObject": - await executeUserScript( - tabId, - operationScript(`__state.objects.delete(${JSON.stringify(params.objectId)}); __state.objectGroups.delete(${JSON.stringify(params.objectId)}); return {};`), - { frameId }, - ); - return {}; + return executeCspSafePageOperation(tabId, "releaseObject", { objectId: params.objectId }, { frameId }); case "Runtime.releaseObjectGroup": - await executeUserScript( - tabId, - operationScript(`for (const [id, group] of __state.objectGroups) { if (group === ${JSON.stringify(params.objectGroup)}) { __state.objectGroups.delete(id); __state.objects.delete(id); } } return {};`), - { frameId }, - ); - return {}; + return executeCspSafePageOperation(tabId, "releaseObjectGroup", { objectGroup: params.objectGroup }, { frameId }); case "Runtime.addBinding": { const names = bindingNamesByTab.get(tabId) ?? new Set(); names.add(params.name); diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 0000000..ac75144 --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1,13 @@ +# Computer-use protocol fixtures + +`cua-ax-functions.json` records the node-bound hit-test callback sent by the +installed Browser Use runtime (26.903.71938) during `tab.click(elementIndex)`. +It is passed unchanged to Runtime.callFunctionOn in the real Firefox test. +The static equivalent in firefox-compat.js preserves its hit-test semantics +without compiling JavaScript inside the website's CSP realm. + +The test sends an actual CSP response header, resolves the input through the AX +and DOM protocol, runs this callback, dispatches mouse events, inserts text, and +checks the input value and foreground-tab preservation. To demonstrate the +regression against a prior adapter, set FIREFOX_COMPAT_SOURCE to that adapter's +path when running npm run test:live. diff --git a/tests/fixtures/cua-ax-functions.json b/tests/fixtures/cua-ax-functions.json new file mode 100644 index 0000000..1e97c83 --- /dev/null +++ b/tests/fixtures/cua-ax-functions.json @@ -0,0 +1,3 @@ +{ + "hitTest": "function (argument) { const __name = (target) => target; return (function nX(){let e=this instanceof Element?this:this.parentElement;if(e==null)throw new Error(\"Cannot hit-test a detached accessibility element\");let t=e.getClientRects()[0]??e.getBoundingClientRect();if(this instanceof Text){let A=document.createRange();A.selectNode(this),t=A.getClientRects()[0]??A.getBoundingClientRect()}let r={x:t.left+t.width/2,y:t.top+t.height/2},n=e.ownerDocument.defaultView;if(n==null)throw new Error(\"Cannot hit-test an accessibility element without a viewport\");let o=n.visualViewport,i=o?.offsetLeft??0,s=o?.offsetTop??0,a=Math.max(t.left,i),l=Math.min(t.right,i+(o?.width??n.innerWidth)),c=Math.max(t.top,s),p=Math.min(t.bottom,s+(o?.height??n.innerHeight)),m=l>a&&p>c?{x:(a+l)/2,y:(c+p)/2}:r,h=e instanceof HTMLInputElement||e instanceof HTMLButtonElement||e instanceof HTMLSelectElement||e instanceof HTMLTextAreaElement?e:void 0,y=[e,...Array.from(h?.labels??[])],w=[\"button\",\"a[href]\",\"input\",\"select\",\"textarea\",'[role=\"button\"]','[role=\"checkbox\"]','[role=\"combobox\"]','[role=\"link\"]','[role=\"listbox\"]','[role=\"menuitem\"]','[role=\"option\"]','[role=\"radio\"]','[role=\"slider\"]','[role=\"spinbutton\"]','[role=\"switch\"]','[role=\"tab\"]','[role=\"textbox\"]','[tabindex]:not([tabindex=\"-1\"])'].join(\",\"),x=(A,L=m.x,$=m.y)=>{let U=A.getRootNode(),X=U instanceof ShadowRoot?U.elementFromPoint(L,$):A.ownerDocument.elementFromPoint(L,$);for(;X?.shadowRoot!=null;){let oe=X.shadowRoot.elementFromPoint(L,$);if(oe==null||oe===X)break;X=oe}return X},S=(A,L,$=!1)=>{let U=!1;for(let X=L;X!=null;){if(X===A)return!$||U;let oe=X instanceof Element||X instanceof Text?X.assignedSlot:null;if(oe!=null)U=!0,X=oe;else if(X.parentNode!=null)X=X.parentNode;else{let me=X.getRootNode(),se=me instanceof ShadowRoot?me.host:null;U||=se!=null,X=se}}return!1},D=(A,L=m.x,$=m.y)=>{let U=x(A,L,$);if(U==null)return!1;let X=U.closest(w),oe=(A instanceof HTMLLabelElement?A.control:null)??A.closest(w);if(oe!=null&&X!=null&&X!==oe&&!S(X,oe))return!1;if(S(A,U)||S(U,A,!0))return!0;let me=A.closest(\"label\")?.control;return me!=null&&(S(U,me)||S(me,U)||A===e&&U.parentElement===me.parentElement&&(X==null||S(X,me)))||oe!=null&&oe===X?!0:getComputedStyle(A).pointerEvents===\"none\"&&S(U,A)},T=y.some(A=>D(A));return{fallbackPoint:(T?null:y.slice(1).flatMap(A=>Array.from(A.getClientRects()).map(L=>({candidate:A,point:{x:L.left+L.width/2,y:L.top+L.height/2},size:{height:L.height,width:L.width}}))).find(({candidate:A,point:L,size:$})=>$.width>1&&$.height>1&&D(A,L.x,L.y))?.point)??null,hitTag:x(e)?.tagName??null,hitsTarget:T,point:m,viewportSize:{height:n.innerHeight,width:n.innerWidth}}}).call(this, argument); }" +} diff --git a/tests/test-firefox-lifecycle-live.mjs b/tests/test-firefox-lifecycle-live.mjs index 7996890..2e119cf 100644 --- a/tests/test-firefox-lifecycle-live.mjs +++ b/tests/test-firefox-lifecycle-live.mjs @@ -14,9 +14,10 @@ if (!process.env.npm_execpath) throw new Error("Run this test with npm run test:live."); const dir = fs.mkdtempSync(path.join(os.tmpdir(), "firefox-lifecycle-test-")); fs.copyFileSync( - path.join(root, "extension/firefox-compat.js"), + process.env.FIREFOX_COMPAT_SOURCE || path.join(root, "extension/firefox-compat.js"), path.join(dir, "firefox-compat.js"), ); +const axFunctions = JSON.parse(fs.readFileSync(path.join(root, "tests/fixtures/cua-ax-functions.json"), "utf8")); const original = JSON.parse( fs.readFileSync(path.join(root, "extension/manifest.json")), ); @@ -34,7 +35,9 @@ const manifest = { fs.writeFileSync(path.join(dir, "manifest.json"), JSON.stringify(manifest)); let finish; const result = new Promise((r) => (finish = r)); +const requests = []; const server = http.createServer((req, res) => { + requests.push(req.url); if (req.url === "/result") { let body = ""; req.on("data", (x) => (body += x)); @@ -44,8 +47,9 @@ const server = http.createServer((req, res) => { }); } else { res.setHeader("content-type", "text/html"); + res.setHeader("Content-Security-Policy", "default-src 'self'; script-src 'self'; object-src 'none'"); res.end( - "Lifecycle fixture

isolated lifecycle test

", + "Lifecycle fixture

isolated lifecycle test

", ); } }); @@ -54,10 +58,15 @@ const url = `http://127.0.0.1:${server.address().port}`; fs.writeFileSync( path.join(dir, "test.js"), `(async()=>{try{ -const target=await browser.tabs.create({url:${JSON.stringify(url)},active:false}); +const target=await browser.tabs.create({url:${JSON.stringify(url)},active:true}); +let initialLoaded=false; +for(let i=0;i<300;i++){ + const tab=await browser.tabs.get(target.id); + if(tab.status==='complete'&&tab.url===${JSON.stringify(url + "/")}){initialLoaded=true;break;} + await new Promise(r=>setTimeout(r,100)); +} +if(!initialLoaded)throw Error('Initial fixture did not load: '+JSON.stringify(await browser.tabs.get(target.id))); const foreground=await browser.tabs.create({url:${JSON.stringify(url + "/foreground")},active:true}); -for(let i=0;i<100&&(await browser.tabs.get(target.id)).status!=='complete';i++)await new Promise(r=>setTimeout(r,100)); -if((await browser.tabs.get(target.id)).status!=='complete')throw Error('Initial fixture did not load'); const activations=[];browser.tabs.onActivated.addListener(info=>activations.push(info.tabId)); const debuggee={tabId:target.id};const events=[]; chrome.debugger.onEvent.addListener((source,method,params)=>{if(source.tabId===target.id)events.push({method,params});}); @@ -66,18 +75,46 @@ await chrome.debugger.sendCommand(debuggee,'Page.enable',{}); await chrome.debugger.sendCommand(debuggee,'Network.enable',{}); await chrome.debugger.sendCommand(debuggee,'Page.setLifecycleEventsEnabled',{enabled:true}); await browser.tabs.update(target.id,{url:${JSON.stringify(url + "/next")}}); -for(let i=0;i<100&&!events.some(e=>e.method==='Page.lifecycleEvent'&&e.params.name==='load');i++)await new Promise(r=>setTimeout(r,100)); -const lifecycle=events.filter(e=>e.method==='Page.lifecycleEvent'); +const navigationLoader=()=>events.findLast(e=>e.method==='Page.frameNavigated'&&e.params.frame.url===${JSON.stringify(url + "/next")})?.params.frame.loaderId; +for(let i=0;i<300&&!events.some(e=>e.method==='Page.lifecycleEvent'&&e.params.name==='load'&&e.params.loaderId===navigationLoader());i++)await new Promise(r=>setTimeout(r,100)); +if(!navigationLoader())throw Error('Fixture navigation did not commit'); +const lifecycle=events.filter(e=>e.method==='Page.lifecycleEvent'&&e.params.loaderId===navigationLoader()); const active=(await browser.tabs.query({active:true,currentWindow:true}))[0]; if(active.id!==foreground.id||activations.includes(target.id))throw Error('Foreground tab changed'); for(const name of ['init','DOMContentLoaded','load'])if(!lifecycle.some(e=>e.params.name===name))throw Error('Missing '+name); const lastLoad=lifecycle.findLast(e=>e.params.name==='load'); -if(new Set(lifecycle.map(e=>e.params.loaderId)).size!==1)throw Error('Mixed navigation events'); const tree=await chrome.debugger.sendCommand(debuggee,'Page.getFrameTree',{}); if(tree.frameTree.frame.loaderId!==lastLoad.params.loaderId)throw Error('Loader mismatch'); -const documentEvents=events.filter(e=>['Network.requestWillBeSent','Network.responseReceived'].includes(e.method)&&e.params.type==='Document'&&e.params.requestId.startsWith('firefox-request-')); -if(!documentEvents.length||documentEvents.some(e=>e.params.loaderId!==lastLoad.params.loaderId))throw Error('Network loader mismatch'); -await fetch(${JSON.stringify(url + "/result")},{method:'POST',body:JSON.stringify({ok:true,backgroundTabPreserved:true,lifecycle:lifecycle.map(e=>e.params.name),loaderId:lastLoad.params.loaderId})}); +const documentEvents=events.filter(e=>['Network.requestWillBeSent','Network.responseReceived'].includes(e.method)&&e.params.type==='Document'&&(e.params.request?.url??e.params.response?.url)===${JSON.stringify(url + "/next")}&&e.params.requestId.startsWith('firefox-request-')); +if(!documentEvents.length||documentEvents.some(e=>e.params.loaderId!==lastLoad.params.loaderId))throw Error('Network loader mismatch: '+JSON.stringify({documentEvents, lifecycle})); +const ax=await chrome.debugger.sendCommand(debuggee,'Accessibility.getFullAXTree',{}); +const input=ax.nodes.find(n=>n.name?.value==='Repository search'); +if(!input)throw Error('Search input missing from AX tree'); +const resolved=await chrome.debugger.sendCommand(debuggee,'DOM.resolveNode',{backendNodeId:input.backendDOMNodeId}); +const hit=await chrome.debugger.sendCommand(debuggee,'Runtime.callFunctionOn',{objectId:resolved.object.objectId,functionDeclaration:${JSON.stringify(axFunctions.hitTest)},arguments:[{}],returnByValue:true,userGesture:true}); +if(!hit.result.value.hitsTarget)throw Error('AX hit test did not find input'); +const point=hit.result.value.point; +for(const type of ['mouseMoved','mousePressed','mouseReleased'])await chrome.debugger.sendCommand(debuggee,'Input.dispatchMouseEvent',{type,...point,button:type==='mouseMoved'?'none':'left',clickCount:1}); +await chrome.debugger.sendCommand(debuggee,'Input.insertText',{text:'icarus'}); +const typed=await chrome.debugger.sendCommand(debuggee,'Accessibility.getFullAXTree',{}); +if(typed.nodes.find(n=>n.name?.value==='Repository search')?.value?.value!=='icarus')throw Error('Click and type did not update input'); +for(const modifiers of [4,2]){ + if(modifiers===2)await chrome.debugger.sendCommand(debuggee,'Input.insertText',{text:'icarus'}); + for(const type of ['keyDown','keyUp'])await chrome.debugger.sendCommand(debuggee,'Input.dispatchKeyEvent',{type,key:'a',code:'KeyA',modifiers,windowsVirtualKeyCode:65}); + for(const type of ['keyDown','keyUp'])await chrome.debugger.sendCommand(debuggee,'Input.dispatchKeyEvent',{type,key:'Backspace',code:'Backspace',modifiers:0,windowsVirtualKeyCode:8}); + const cleared=await chrome.debugger.sendCommand(debuggee,'Accessibility.getFullAXTree',{}); + if(cleared.nodes.find(n=>n.name?.value==='Repository search')?.value?.value!=='')throw Error('Select-all and Backspace did not clear input (modifiers='+modifiers+')'); +} +await chrome.debugger.sendCommand(debuggee,'Runtime.releaseObject',{objectId:resolved.object.objectId}); +const released=await chrome.debugger.sendCommand(debuggee,'Runtime.callFunctionOn',{objectId:resolved.object.objectId,functionDeclaration:${JSON.stringify(axFunctions.hitTest)},arguments:[{}],returnByValue:true}); +if(released.result?.type!=='undefined'||!released.exceptionDetails?.text)throw Error('Released handle must return CDP exceptionDetails'); +const grouped=await chrome.debugger.sendCommand(debuggee,'DOM.resolveNode',{backendNodeId:input.backendDOMNodeId,objectGroup:'cleanup-test'}); +await chrome.debugger.sendCommand(debuggee,'Runtime.releaseObjectGroup',{objectGroup:'cleanup-test'}); +const releasedGroup=await chrome.debugger.sendCommand(debuggee,'Runtime.callFunctionOn',{objectId:grouped.object.objectId,functionDeclaration:${JSON.stringify(axFunctions.hitTest)},arguments:[{}],returnByValue:true}); +if(!releasedGroup.exceptionDetails?.text)throw Error('Group cleanup retained the handle'); +const activeAfter=(await browser.tabs.query({active:true,currentWindow:true}))[0]; +if(activeAfter.id!==foreground.id||activations.includes(target.id))throw Error('CSP click activated background tab'); +await fetch(${JSON.stringify(url + "/result")},{method:'POST',body:JSON.stringify({ok:true,backgroundTabPreserved:true,strictCspAxClickTypeAndClear:true,objectCleanupAndExceptionContract:true,lifecycle:lifecycle.map(e=>e.params.name),loaderId:lastLoad.params.loaderId})}); }catch(e){await fetch(${JSON.stringify(url + "/result")},{method:'POST',body:JSON.stringify({ok:false,error:String(e)+' '+e.stack})});}})();`, ); const child = spawn( @@ -115,6 +152,7 @@ const timeout = setTimeout( 90000, ); const outcome = await result; +if (!outcome.ok) Object.assign(outcome, { requests, logs }); clearTimeout(timeout); try { if (process.platform === "win32")