feat: add safe granular tool failure diagnostics

README.md

@@ -248,6 +248,40 @@ Then point your MCP client at `http://localhost:8080/mcp` using the same header/
 | `AMPLITUDE_API_KEY`          | No       | —                                        | Amplitude project API key. Sends MCP usage analytics — SDK lifecycle events plus our own tool/skill events                                                 |
 | `MCP_COMPLIANCE_MODE`        | No       | unset (full surface)                     | Serve the reduced, directory-compliant surface. Fails closed: any set value except `false`/`0`/`no`/`off` enables it                                       |
 
+### Failure diagnostics
+
+`MCP Tool Request` retains `analytics_version=2`, the existing coarse
+`error_category`, `status_code`, timing and tool-specific properties. These
+additive diagnostic fields are failure-only; a successful retry has none of them.
+
+| Property               | Meaning                                                                                                                                                                                                                                                                                                  |
+| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `error_reason`         | `selector_miss`, `invalid_params`, `unknown_method`, `script_error`, `unauthorized`, `forbidden`, `not_found`, `server_error`, `session_lost`, `navigation_failed`, `timeout`, or `unknown`. Agent errors retain their existing detailed classification; local URL/batch validation is `invalid_params`. |
+| `error_source`         | `validation`, `script`, `target_website`, `api`, `transport`, or `unknown`. This identifies the observed failure boundary, not blame. A 403 alone does not identify its source.                                                                                                                          |
+| `failed_command_index` | Zero-based index in the invocation's command batch, not the session-wide command counter. Omitted when setup/validation fails before a command starts.                                                                                                                                                   |
+| `failed_method`        | The failed command's recognized typed method name. Unrecognized free-form method names are omitted to avoid emitting arbitrary input; the index still identifies the command.                                                                                                                            |
+| `error_code`           | Allowlisted structured codes: the uppercase reason names above, `SELECTOR_NOT_FOUND`, `BROWSER_CRASHED`, `ECONNRESET`, `ECONNREFUSED`, `ENOTFOUND`, `EAI_AGAIN`, `ETIMEDOUT`. Opaque/unrecognized codes are omitted, not copied into messages.                                                           |
+| `error_status_code`    | An integer HTTP status (100–599) carried by structured error metadata. Never extracted from error prose.                                                                                                                                                                                                 |
+| `error_status_origin`  | `api` for an observed API response/upgrade, `target_website` for a failed navigation result, otherwise `unknown`. Omitted when no structured status is available.                                                                                                                                        |
+| `error_message`        | A synthesized summary capped at 500 characters. Raw error messages, response bodies, HTML, scripts, selectors, credentials, cookies, authorization headers and URLs are never copied into this field.                                                                                                    |
+
+`status_code` keeps its original tool-specific meaning; the new status fields
+do not replace it or turn successful target-page HTTP responses into failures.
+HTTP failures retain API response status even when thrown. Codes are retained
+when already available in structured errors or the JSON body read by the existing
+4xx error handler; diagnostics do not read additional bodies on 5xx failures.
+
+An unsuccessful search without structured evidence reports `error_reason=unknown`
+and `error_message="Unclassified search failure."`. Its legacy `user_error`
+category remains for chart compatibility, not as evidence of caller fault.
+
+Example breakdowns: filter `success=false` and group by `tool → error_reason`;
+for agent calls, group by `failed_method → error_reason`; for HTTP failures,
+group by `error_status_origin → error_status_code`. Missing fields in older
+events mean unavailable instrumentation, not an `unknown` failure. There is no
+historical backfill. Verify representative received events after deployment
+before treating these properties as available in production.
+
 ## MCP Resources
 
 | Resource URI             | Description                     |

src/lib/api-client.ts

@@ -113,7 +113,16 @@ async function defaultHandleResponse<T>(res: Response): Promise<T> {
   if (!res.ok) {
     const errorBody = await res.text().catch(() => res.statusText);
     const message = errorBody.trim() || res.statusText;
-    throw new Error(`Server error ${res.status}: ${message}`);
+    let code: unknown;
+    try {
+      const body = JSON.parse(errorBody);
+      code = body?.code ?? body?.error?.code;
+    } catch {
+      /* Non-JSON errors retain their existing message. */
+    }
+    throw Object.assign(new Error(`Server error ${res.status}: ${message}`), {
+      apiCode: code,
+    });
   }
   return (await res.json()) as T;
 }
@@ -146,8 +155,15 @@ function apiFetch<T>(
       );
       try {
         const res = await fetch(url, { ...init, signal: controller.signal });
-        await throwIfProfileMissing(res, opts.profile);
-        return await handle(res);
+        try {
+          await throwIfProfileMissing(res, opts.profile);
+          return await handle(res);
+        } catch (error) {
+          if (error instanceof Error && !res.ok) {
+            Object.assign(error, { apiStatus: res.status });
+          }
+          throw error;
+        }
       } finally {
         clearTimeout(timeoutId);
       }

src/lib/define-tool.ts

@@ -11,6 +11,7 @@ import { ResponseCache } from './cache.js';
 import { AnalyticsHelper } from './analytics.js';
 import { setAmplitudeToolContext } from './amplitude-analytics.js';
 import { categorizeThrown, categoryFromStatus } from './error-classifier.js';
+import { failureDetails, failureFields } from './failure-details.js';
 import type {
   ApiClient,
   BrowserlessSession,
@@ -187,11 +188,23 @@ export function defineTool<P, R>(
       let fired = false;
       // Held so a `format` that throws still reports the run's `ok`/`status_code`.
       let resultProps: Record<string, unknown> | undefined;
+      let validating = true;
 
       const enrich = (props: Record<string, unknown>) => {
         const success = normalizeSuccess(props);
+        const cleanProps = { ...props };
+        if (success) {
+          for (const field of failureFields) delete cleanProps[field];
+          delete cleanProps.error_category;
+        } else {
+          for (const [field, value] of Object.entries(
+            failureDetails(undefined),
+          )) {
+            if (cleanProps[field] === undefined) cleanProps[field] = value;
+          }
+        }
         return {
-          ...props,
+          ...cleanProps,
           success,
           duration_ms: Date.now() - startedAt,
           analytics_version: ANALYTICS_VERSION,
@@ -237,6 +250,7 @@ export function defineTool<P, R>(
           apiUrl = s.apiUrl;
         }
         def.validateUrl?.(params);
+        validating = false;
 
         await reportProgress({ progress: 0, total: 100 });
 
@@ -284,6 +298,12 @@ export function defineTool<P, R>(
 
         if (!fired) {
           emit({
+            ...failureDetails(
+              err,
+              validating
+                ? { category: 'INVALID_PARAMS', source: 'validation' }
+                : {},
+            ),
             ...resultProps,
             success: false,
             error_category:

src/lib/failure-details.ts

@@ -0,0 +1,119 @@
+import type { ErrorCategory } from '../@types/types.js';
+
+type Source =
+  'validation' | 'script' | 'target_website' | 'api' | 'transport' | 'unknown';
+
+const reasons = {
+  SELECTOR_MISS: 'selector_miss',
+  INVALID_PARAMS: 'invalid_params',
+  UNKNOWN_METHOD: 'unknown_method',
+  SCRIPT_ERROR: 'script_error',
+  UNAUTHORIZED: 'unauthorized',
+  FORBIDDEN: 'forbidden',
+  NOT_FOUND: 'not_found',
+  SERVER_ERROR: 'server_error',
+  SESSION_LOST: 'session_lost',
+  NAVIGATION_FAILED: 'navigation_failed',
+  TIMEOUT: 'timeout',
+  UNKNOWN: 'unknown',
+} satisfies Record<ErrorCategory, string>;
+
+// Codes are untrusted text too. Only documented categories and transport codes
+// are safe to publish; an opaque provider code could itself contain a secret.
+const safeCodes = new Set([
+  ...Object.keys(reasons),
+  'SELECTOR_NOT_FOUND',
+  'BROWSER_CRASHED',
+  'ECONNRESET',
+  'ECONNREFUSED',
+  'ENOTFOUND',
+  'EAI_AGAIN',
+  'ETIMEDOUT',
+]);
+
+export const failureFields = [
+  'error_reason',
+  'error_source',
+  'error_code',
+  'error_message',
+  'error_status_code',
+  'error_status_origin',
+  'failed_method',
+  'failed_command_index',
+] as const;
+
+/** Build analytics only from structured evidence, never arbitrary error prose. */
+export function failureDetails(
+  error: unknown,
+  options: {
+    category?: ErrorCategory;
+    source?: Source;
+  } = {},
+): Record<string, unknown> {
+  const err =
+    error && typeof error === 'object'
+      ? (error as {
+          code?: unknown;
+          status?: unknown;
+          statusCode?: unknown;
+          apiStatus?: unknown;
+          apiCode?: unknown;
+        })
+      : {};
+  const rawStatus = err.apiStatus ?? err.status ?? err.statusCode;
+  const status =
+    typeof rawStatus === 'number' &&
+    Number.isInteger(rawStatus) &&
+    rawStatus >= 100 &&
+    rawStatus <= 599
+      ? rawStatus
+      : undefined;
+  const rawCode = err.apiCode ?? err.code;
+  const code =
+    typeof rawCode === 'string' && safeCodes.has(rawCode) ? rawCode : undefined;
+  const category =
+    options.category ??
+    (code === 'SELECTOR_NOT_FOUND'
+      ? 'SELECTOR_MISS'
+      : code === 'BROWSER_CRASHED'
+        ? 'SESSION_LOST'
+        : code && Object.hasOwn(reasons, code)
+          ? (code as ErrorCategory)
+          : status === 401
+            ? 'UNAUTHORIZED'
+            : status === 403
+              ? 'FORBIDDEN'
+              : status === 404
+                ? 'NOT_FOUND'
+                : status !== undefined && status >= 500
+                  ? 'SERVER_ERROR'
+                  : 'UNKNOWN');
+  const reason = reasons[category];
+  const source =
+    options.source ??
+    (err.apiStatus !== undefined
+      ? 'api'
+      : code === 'INVALID_PARAMS' || code === 'UNKNOWN_METHOD'
+        ? 'validation'
+        : 'unknown');
+  return {
+    error_reason: reason,
+    error_source: source,
+    // Synthesized summaries deliberately omit raw text rather than attempting
+    // best-effort regex redaction of arbitrary scripts, selectors, or bodies.
+    error_message: `Request failed: ${reason.replaceAll('_', ' ')}.`.slice(
+      0,
+      500,
+    ),
+    ...(code === undefined ? {} : { error_code: code }),
+    ...(status === undefined
+      ? {}
+      : {
+          error_status_code: status,
+          error_status_origin:
+            source === 'api' || source === 'target_website'
+              ? source
+              : 'unknown',
+        }),
+  };
+}

src/tools/agent.ts

@@ -17,6 +17,7 @@ import {
   closeSession,
   destroySession,
   isRetryableUpgradeError,
+  UpgradeError,
 } from '../lib/agent-client.js';
 import type {
   AgentParams,
@@ -32,6 +33,7 @@ import {
   toAnalyticsCategory,
 } from '../lib/error-classifier.js';
 import { AnalyticsHelper } from '../lib/analytics.js';
+import { failureDetails } from '../lib/failure-details.js';
 import { defineTool } from '../lib/define-tool.js';
 import {
   markFired,
@@ -686,13 +688,16 @@ export function registerAgentTools(
           .array(compliant ? CompliantAgentCommandSchema : AgentCommandSchema)
           .safeParse(params.commands);
         if (!commandContract.success) {
-          throw new UserError(
-            commandContract.error.issues
-              .map(
-                (i) =>
-                  (i.path.length ? `${i.path.join('.')}: ` : '') + i.message,
-              )
-              .join('; '),
+          throw Object.assign(
+            new UserError(
+              commandContract.error.issues
+                .map(
+                  (i) =>
+                    (i.path.length ? `${i.path.join('.')}: ` : '') + i.message,
+                )
+                .join('; '),
+            ),
+            { code: 'INVALID_PARAMS' },
           );
         }
         // Forward the parsed batch, not the raw one: the per-command schemas
@@ -748,6 +753,7 @@ export function registerAgentTools(
       }
 
       let lastCategory: ErrorCategory | undefined;
+      let lastFailure: Record<string, unknown> | undefined;
 
       const sendAnalytics = (success: boolean, err?: unknown) => {
         analytics?.fireToolRequest(token, 'browserless_agent', {
@@ -760,6 +766,7 @@ export function registerAgentTools(
           ...(success
             ? {}
             : {
+                ...(lastFailure ?? failureDetails(err)),
                 error_category: lastCategory
                   ? toAnalyticsCategory(lastCategory)
                   : categorizeThrown(err),
@@ -781,6 +788,10 @@ export function registerAgentTools(
       const proxyCmd = commands.find((c) => c.method === 'proxy');
       if (proxyCmd) {
         lastCategory = 'INVALID_PARAMS';
+        lastFailure = failureDetails(undefined, {
+          category: lastCategory,
+          source: 'validation',
+        });
         sendAnalytics(false);
         throw new UserError(
           'Invalid command: "proxy" is not a BQL mutation. Proxy config is a top-level tool argument (proxy, proxyCountry, proxyState, proxyCity, proxySticky, proxyLocaleMatch, proxyPreset, externalProxyServer) and is read once at session creation. ' +
@@ -829,6 +840,9 @@ export function registerAgentTools(
             record,
           );
         } catch (connErr: unknown) {
+          lastFailure = failureDetails(connErr, {
+            source: connErr instanceof UpgradeError ? 'api' : 'unknown',
+          });
           sendAnalytics(false, connErr);
           throw new UserError(formatConnectError(connErr));
         }
@@ -843,6 +857,8 @@ export function registerAgentTools(
       }
 
       const runCommands = async (isRetry: boolean): Promise<Content[]> => {
+        lastFailure = undefined;
+        lastCategory = undefined;
         let agentSession;
         try {
           agentSession = await getOrCreateSession(
@@ -867,6 +883,9 @@ export function registerAgentTools(
           // with the same (bad token / wrong profile / unsupported params)
           // will just produce the same response and waste time.
           if (isRetry || !isRetryableUpgradeError(connErr)) {
+            lastFailure = failureDetails(connErr, {
+              source: connErr instanceof UpgradeError ? 'api' : 'unknown',
+            });
             throw new UserError(formatConnectError(connErr));
           }
           destroySession(
@@ -891,7 +910,26 @@ export function registerAgentTools(
         // still detects the A→snapshot cross-origin transition.
         let crossOriginBaseline: string | undefined = agentSession.lastUrl;
         let promptSent = false;
-        for (const cmd of commands) {
+        for (const [commandIndex, cmd] of commands.entries()) {
+          const commandFailure = (err: unknown, category: ErrorCategory) => ({
+            ...failureDetails(err, {
+              category,
+              source:
+                category === 'SCRIPT_ERROR'
+                  ? 'script'
+                  : category === 'NAVIGATION_FAILED'
+                    ? 'target_website'
+                    : category === 'SESSION_LOST'
+                      ? 'transport'
+                      : 'unknown',
+            }),
+            failed_command_index: commandIndex,
+            ...(AgentCommandSchema.options[0].options.some(
+              (schema) => schema.shape.method.safeParse(cmd.method).success,
+            )
+              ? { failed_method: cmd.method }
+              : {}),
+          });
           if (cmd.method === 'close') {
             closeSession(
               mcpSessionId,
@@ -965,6 +1003,7 @@ export function registerAgentTools(
               cmd,
             });
             lastCategory = classified.category;
+            lastFailure = commandFailure(sendErr, classified.category);
             throw new UserError(
               formatErrorMessage({
                 category: classified.category,
@@ -997,6 +1036,7 @@ export function registerAgentTools(
 
             const classified = classifyAgentError({ err, cmd });
             lastCategory = classified.category;
+            lastFailure = commandFailure(err, classified.category);
 
             const prefix =
               commands.length > 1
@@ -1047,6 +1087,7 @@ export function registerAgentTools(
           const navFailure = classifyNavigationResult(cmd.method, resp.result);
           if (navFailure) {
             lastCategory = navFailure.category;
+            lastFailure = commandFailure(resp.result, navFailure.category);
             throw new UserError(
               [
                 formatErrorMessage({