Fix ambiguous file uploads and enforce recovery inspection

src/chrome/src/agent/adapters.js

@@ -15780,6 +15780,16 @@ const ADAPTERS = [
 - Releases live at /<group>/<project>/-/releases/new. Tag must exist or be created via "Create tag" inline.
 - Merge requests have a "Merge" button that may be disabled until pipelines pass; check the pipeline status before clicking.
 - The sidebar collapses on narrow viewports — scroll horizontally or expand it before clicking sidebar items.`,
+  },
+  {
+    name: 'huggingface',
+    category: 'general',
+    matches: (url) => /^https?:\/\/(?:www\.)?huggingface\.co(?:[/?#]|$)/i.test(url),
+    notes: `
+- Repository upload routes expose two file inputs. Use \`input[type="file"]:not([accept])\` for repository files; \`input[type="file"][accept*="image"]\` belongs to the extended-description editor and does not stage a repository file.
+- When the repository input already exists, call \`upload_file\` directly; do not click "Upload file(s)" or the drop zone first.
+- A filename chip, generated commit summary, and enabled "Commit changes" button mean the file is staged only. Click "Commit changes", wait, and verify the file under "Files and versions" before reporting upload success.
+- For model-card media, commit the asset first, then edit README Markdown to reference it, commit README, and verify the rendered model card.`,
   },
   {
     name: 'mozilla-addons-developer',

src/chrome/src/agent/agent.js

@@ -414,6 +414,7 @@ export class Agent extends LoopDetector {
     this._lastClickProgress = new Map(); // tabId -> { ident, snapshot }
     this._clickAxCdpFallbacks = new Map(); // tabId -> Set(documentToken|ref_id), one trusted fallback per document target
     this._lastAxScopes = new Map(); // tabId -> { documentToken, pageUrl }, captured by the latest AX read
+    this._uploadSelectorRecoveryRequired = new Map(); // tabId -> prior ambiguous match count; cleared only by inspection/navigation/cleanup
     // Productive browsing often mixes reads and scrolling, so exact-call loop
     // detection cannot tell when the agent already has enough evidence to
     // answer. Track long observation-only streaks and remind it to deliver a
@@ -1664,6 +1665,7 @@ export class Agent extends LoopDetector {
    */
   _clearPageLoopState(tabId) {
     super._clearPageLoopState(tabId);
+    this._uploadSelectorRecoveryRequired.delete(tabId);
     this.deliveryObservationStreaks.delete(tabId);
     this.bulkApiMutationClicks.delete(tabId);
     this.bulkApiMutationHints.delete(tabId);
@@ -1675,6 +1677,19 @@ export class Agent extends LoopDetector {
     }
   }
 
+  _clearUploadSelectorRecoveryAfterInspection(tabId, name, response) {
+    if (name !== 'get_interactive_elements' || !Array.isArray(response)) return false;
+    const hasVerifiedFileInputSelector = response.some(element => (
+      element?.tag === 'input'
+      && String(element.type || '').toLowerCase() === 'file'
+      && typeof element.selector === 'string'
+      && element.selector.trim().length > 0
+    ));
+    if (!hasVerifiedFileInputSelector) return false;
+    this._uploadSelectorRecoveryRequired.delete(tabId);
+    return true;
+  }
+
   _rememberAxScope(tabId, documentToken, pageUrl = '') {
     const next = {
       documentToken: String(documentToken || ''),
@@ -1715,6 +1730,7 @@ export class Agent extends LoopDetector {
       && this._isSuccessfulExecutionEvidence(result)
       && result?.noProgress !== true
       && result?.verified !== false
+      && result?.remoteStateVerified !== false
       && result?.inconclusive !== true;
   }
 
@@ -3588,7 +3604,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
     ) {
       return 'action_failed';
     }
-    if (toolResult?.inconclusive || toolResult?.verified === false) {
+    if (
+      toolResult?.inconclusive
+      || toolResult?.verified === false
+      || toolResult?.remoteStateVerified === false
+    ) {
       return 'action_unverified';
     }
     if (
@@ -13036,7 +13056,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
       }
       case 'upload_file': {
         if (parsed.success === false) return `upload failed: ${this._truncate(parsed.error || '', 110)}`;
-        if (parsed.attached) return `uploaded ${this._truncate(parsed.attached.name || '', 60)} (${parsed.attached.size} bytes)`;
+        if (parsed.remoteStateVerified === false) {
+          const localState = parsed.attachmentState === 'page_consumed'
+            ? 'page consumed attachment'
+            : 'file attached to input';
+          return `${localState} (remote submission unverified)`;
+        }
+        if (parsed.attached) return `attached ${this._truncate(parsed.attached.name || '', 60)} (${parsed.attached.size} bytes)`;
         return parsed.verified === false ? `upload sent (unverified)` : `uploaded ${this._truncate(parsed.file || '', 70)}`;
       }
       case 'new_tab': {
@@ -16650,6 +16676,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
 
     if (name === 'upload_file') {
       args = args || {};
+      if (this._uploadSelectorRecoveryRequired.has(tabId)) {
+        return {
+          success: false,
+          dispatched: false,
+          noDispatch: true,
+          ambiguous: true,
+          matchCount: Number(this._uploadSelectorRecoveryRequired.get(tabId)) || 0,
+          recoveryRequired: 'get_interactive_elements',
+          error: 'A previous upload selector matched multiple file inputs. Call get_interactive_elements now and use the exact selector returned on the intended file-input record before retrying upload_file; do not guess another selector variant.',
+        };
+      }
       // Accept a downloadId as an alternative to filePath. After context
       // compaction the model often can't recall the exact on-disk path, but the
       // small integer id (returned by download_files/list_downloads and
@@ -16697,9 +16734,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
           return { success: false, error: `File input not found for selector "${args.selector}". Re-inspect the page with get_interactive_elements or get_accessibility_tree to find the real <input type=file> (some upload widgets hide it until you click their "add files" button first).` };
         }
         if (objectIds.length > 1) {
+          this._uploadSelectorRecoveryRequired.set(tabId, objectIds.length);
           return {
             success: false,
-            error: `Selector "${args.selector}" matched ${objectIds.length} elements across the document and open shadow roots. Use an exact, unique selector for the intended <input type=file>; do not use a generic input[type=file] selector when multiple inputs exist.`,
+            dispatched: false,
+            noDispatch: true,
+            ambiguous: true,
+            matchCount: objectIds.length,
+            recoveryRequired: 'get_interactive_elements',
+            error: `Selector "${args.selector}" matched ${objectIds.length} elements across the document and open shadow roots. Call get_interactive_elements and use the exact, unique selector returned on the intended file-input record; do not guess another selector variant.`,
           };
         }
 
@@ -16766,7 +16809,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
             // success for the model to confirm against the page, NOT a hard
             // failure: the old hard failure made the model loop, re-uploading a
             // file that was already attached and clobbering the page.
-            return { success: true, file: args.filePath, verified: false, note: `The file input is empty after upload — this usually means an async uploader (e.g. a GitHub release attachment) already consumed the file. Confirm "${basename}" now appears attached via get_accessibility_tree before re-uploading; only retry if it is genuinely missing (and if so, re-check the path with list_downloads).` };
+            return {
+              success: true,
+              file: args.filePath,
+              verified: false,
+              attachmentState: 'page_consumed',
+              remoteStateVerified: false,
+              note: `The page consumed the file input, but upload_file does not prove a remote upload or form submission. Confirm "${basename}" appears attached via get_accessibility_tree, then submit/commit the page when the task requires it. Only retry if the file is genuinely missing.`,
+            };
           }
           const attached = files.find(f => f.name === basename) || files[files.length - 1] || null;
           // readable === false means the bytes couldn't be read — the path is
@@ -16777,7 +16827,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
           if (attached.readable === false) {
             return { success: false, dispatched: true, error: `"${args.filePath}" could not be read — it almost certainly does not exist at that path. Confirm the absolute path (use list_downloads to see where files were actually saved) and retry.` };
           }
-          return { success: true, file: args.filePath, attached: { name: attached.name, size: attached.size } };
+          return {
+            success: true,
+            file: args.filePath,
+            attached: { name: attached.name, size: attached.size },
+            verified: false,
+            attachmentState: 'input_attached',
+            remoteStateVerified: false,
+          };
         }
 
         // Could not read the FileList back. If the probe confirmed the path is
@@ -16788,7 +16845,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
         if (!pathConfirmed) {
           return { success: false, dispatched: true, error: `Could not confirm "${basename}" uploaded: the input.files list was unreadable and the local path "${args.filePath}" was not validated. Check whether "${basename}" appears attached via get_accessibility_tree — if it does, you're done; if not, re-check the path with list_downloads and the selector, then retry.` };
         }
-        return { success: true, file: args.filePath, verified: false, note: 'Attachment could not be verified (the input.files list was unreadable), but the local path validated as readable. If the file does not appear attached on the page, re-check the selector.' };
+        return {
+          success: true,
+          file: args.filePath,
+          verified: false,
+          attachmentState: 'page_consumed',
+          remoteStateVerified: false,
+          note: 'The local file was readable and the page handled the attachment event, but upload_file could not read the resulting FileList. This does not prove a remote upload or form submission; verify the page state and submit/commit when required.',
+        };
       } catch (e) {
         return { success: false, dispatched: uploadDispatched, error: `Upload failed: ${e.message}` };
       } finally {
@@ -18560,6 +18624,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
       if (name === 'read_page') {
         response = applyReadPageWindow(response, args);
       }
+      this._clearUploadSelectorRecoveryAfterInspection(tabId, name, response);
       return response;
     } finally {
       clickAxSideEffectWatch?.stop();

src/chrome/src/agent/tools.js

@@ -923,7 +923,7 @@ export const AGENT_TOOLS = [
     type: 'function',
     function: {
       name: 'upload_file',
-      description: 'Upload a file directly to an existing file input without opening the page or OS file-picker dialog. Do NOT click "Choose file", "Select a file", an upload drop zone, or the input first when the input already exists. Provide EITHER downloadId (preferred — the id from download_files/list_downloads; you do not need to recall the path) OR filePath (absolute local path). Never guess a downloadId. If both are accidentally provided, a valid downloadId is preferred; if that id cannot resolve, the supplied filePath is used as a fallback. If no file input exists because the widget creates it lazily, one guarded click on its add-files control may initialize the widget; then retry upload_file with the exact selector returned or discovered. The file must exist on the local filesystem.',
+      description: 'Attach a file directly to an existing file input without opening the page or OS file-picker dialog. This only proves that the page input received or consumed the file; it does NOT prove a remote upload, form submission, or repository commit. Do NOT click "Choose file", "Select a file", an upload drop zone, or the input first when the input already exists. Provide EITHER downloadId (preferred — the id from download_files/list_downloads; you do not need to recall the path) OR filePath (absolute local path). Never guess a downloadId. If both are accidentally provided, a valid downloadId is preferred; if that id cannot resolve, the supplied filePath is used as a fallback. If the selector is ambiguous, call get_interactive_elements and use the exact selector on the intended file-input record before retrying. If no file input exists because the widget creates it lazily, one guarded click on its add-files control may initialize the widget; then retry upload_file with the exact selector returned or discovered. The file must exist on the local filesystem.',
       parameters: {
         type: 'object',
         properties: {
@@ -1616,7 +1616,7 @@ CLICKING — read this:
 - For buttons and links you can SEE, click by visible text: \`click({text: "Publish release"})\`. Default matching is EXACT (case-insensitive). If exact fails (no match), the system automatically tries prefix then substring matching — but if multiple elements match at any level, it returns an ambiguity error instead of guessing.
 - If you get an ambiguity error, use a more specific text string, switch to \`click({index: N})\` from \`get_interactive_elements\`, or use a selector.
 - You can explicitly control matching with \`textMatch\`: \`"exact"\` (default), \`"prefix"\`, or \`"contains"\`.
-- FILE UPLOADS: when the page already has an \`<input type="file">\`, do not click "Choose file", "Select a file", "Browse", the upload drop zone, or the input first. Find the exact selector and call \`upload_file({selector, downloadId})\` directly; it attaches the file without opening a native dialog. Exception: if \`upload_file\` reports that no input exists because the widget creates it lazily, make one guarded click on the widget's add-files control to initialize it. A blocked-picker result may return the new exact selector; retry \`upload_file\` with that selector. Never substitute a generic \`input[type="file"]\` selector when multiple file inputs exist.
+- FILE UPLOADS: when the page already has an \`<input type="file">\`, do not click "Choose file", "Select a file", "Browse", the upload drop zone, or the input first. Call \`get_interactive_elements\` when needed and use the exact \`selector\` returned on the intended file-input record, then call \`upload_file({selector, downloadId})\`. \`attachmentState\` proves only local input attachment/page consumption; it does NOT prove a remote upload or submit. Verify the filename/status in the page, then activate and verify the required Submit/Commit control. If \`upload_file\` reports an ambiguous selector, a fresh \`get_interactive_elements\` call is required before retrying. Exception: if no input exists because the widget creates it lazily, make one guarded click on its add-files control to initialize it.
 - Order of preference:
   1. \`click_ax({ref_id: "ref_N"})\` — ref_id from get_accessibility_tree. Most reliable; carries role+name so you always know what you're clicking, and ref_ids are stable across calls.
   2. \`click({text: "..."})\` — visible button/link text. Good fallback if the tree didn't surface the element cleanly.

src/chrome/src/content/content.js

@@ -2842,6 +2842,27 @@
       return a.rect.left - b.rect.left;
     });
 
+    // Upload controls are commonly CSS-hidden behind a styled drop zone. Keep
+    // visible action indices stable by appending any omitted file inputs only
+    // after the visual sort; their records expose selectors for upload_file.
+    const appendOmittedFileInputs = (root) => {
+      try {
+        root.querySelectorAll('input[type="file"]').forEach(el => {
+          if (seen.has(el)) return;
+          seen.add(el);
+          collected.push({
+            el,
+            rect: el.getBoundingClientRect(),
+            inShadow: root !== document,
+          });
+        });
+        root.querySelectorAll('*').forEach(host => {
+          if (host.shadowRoot) appendOmittedFileInputs(host.shadowRoot);
+        });
+      } catch (e) {}
+    };
+    appendOmittedFileInputs(document);
+
     return collected;
   }
 
@@ -2854,10 +2875,73 @@
     return queryInteractiveFull().map(c => c.el);
   }
 
+  function _cssString(value) {
+    return String(value || '')
+      .replace(/\\/g, '\\\\')
+      .replace(/"/g, '\\"')
+      .replace(/\0/g, '\uFFFD')
+      .replace(/[\n\r\f]/g, ch => `\\${ch.codePointAt(0).toString(16)} `);
+  }
+
+  function _deepSelectorMatches(selector) {
+    const matches = [];
+    const visit = (root) => {
+      root.querySelectorAll(selector).forEach(el => matches.push(el));
+      root.querySelectorAll('*').forEach(host => {
+        if (host.shadowRoot) visit(host.shadowRoot);
+      });
+    };
+    try { visit(document); } catch { return []; }
+    return matches;
+  }
+
+  function _fileInputPath(el) {
+    const parts = [];
+    for (let node = el; node && node.nodeType === 1 && parts.length < 10; node = node.parentElement) {
+      let part = String(node.tagName || '').toLowerCase();
+      if (!part) break;
+      if (node.id) {
+        try { part += `#${CSS.escape(node.id)}`; } catch {}
+        parts.unshift(part);
+        break;
+      }
+      const parent = node.parentElement;
+      if (parent) {
+        const sameTag = Array.from(parent.children).filter(child => child.tagName === node.tagName);
+        if (sameTag.length > 1) part += `:nth-of-type(${sameTag.indexOf(node) + 1})`;
+      }
+      parts.unshift(part);
+    }
+    return parts.join(' > ');
+  }
+
+  function _uniqueFileInputSelector(el) {
+    if (!(el instanceof HTMLInputElement) || el.type !== 'file') return '';
+    const candidates = [];
+    if (el.id) {
+      try { candidates.push(`#${CSS.escape(el.id)}`); } catch {}
+    }
+    if (el.name) candidates.push(`input[type="file"][name="${_cssString(el.name)}"]`);
+    const accept = el.getAttribute('accept');
+    const acceptPart = accept == null
+      ? ':not([accept])'
+      : `[accept="${_cssString(accept)}"]`;
+    const multiplePart = el.hasAttribute('multiple') ? '[multiple]' : ':not([multiple])';
+    candidates.push(`input[type="file"]${acceptPart}${multiplePart}`);
+    candidates.push(`input[type="file"]${acceptPart}`);
+    const path = _fileInputPath(el);
+    if (path) candidates.push(path);
+    for (const selector of candidates) {
+      const matches = _deepSelectorMatches(selector);
+      if (matches.length === 1 && matches[0] === el) return selector;
+    }
+    return '';
+  }
+
   function getInteractiveElementsFull() {
     return queryInteractiveFull().map((c, i) => {
       const el = c.el;
-      return {
+      const result = {
         index: i,
         tag: el.tagName.toLowerCase(),
         type: el.type || '',
@@ -2869,6 +2953,13 @@
         rect: { x: Math.round(c.rect.x), y: Math.round(c.rect.y), w: Math.round(c.rect.width), h: Math.round(c.rect.height) },
         inShadowDOM: c.inShadow,
       };
+      if (el instanceof HTMLInputElement && el.type === 'file') {
+        const selector = _uniqueFileInputSelector(el);
+        result.accept = el.getAttribute('accept');
+        result.multiple = el.hasAttribute('multiple');
+        if (selector) result.selector = selector;
+      }
+      return result;
     });
   }