fix(copilot): disclose how far a withheld run got instead of one opaque sentinel

A tool result the egress projection cannot vouch for is reduced to a bare
success or to `TOOL_RESULT_UNAVAILABLE_ERROR`. Both drop the execution id
with the payload, and the sentinel also overwrites the real error text, so
a call rejected on its own arguments and a run that already executed come
back byte-identical. Those need opposite retry decisions.

Reproduced against real code by latching a registry the way production
latches one — a child run that returned no provenance envelope — and
driving the real handler and the real projection: a pre-dispatch rejection
and a post-dispatch failure were identical, and a completed run arrived as
`{"success":true}` with nothing to look it up by. Two distinct shapes for
three outcomes.

The registry is right to fail closed; the boundary was discarding facts it
never needed to redact. A tool may now declare a `ToolCallEffect` — a phase
and server-minted ids — which the projection preserves when it withholds
content, because neither is derived from that content. The exemption is
enforced rather than asserted: ids must match the identifier shape this
system mints, and one that does not voids the whole disclosure.

The phase is attached in the application layer from dispatch onward and
nowhere earlier, which is what makes the id's absence the positive
statement that nothing was created rather than an admission of not
knowing. Withholding also now reports its cause — a latched registry names
the guard that tripped, an absent one means no catalog was built, and a
content refusal means the registry was fine — so the next occurrence is
diagnosable from the logs it already writes.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: icecrasher321

apps/sim/app/api/copilot/tools/execute/route.ts

@@ -9,6 +9,7 @@ import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
 import { checkInternalApiKey } from '@/lib/copilot/request/http'
 import { withIncomingGoSpan } from '@/lib/copilot/request/otel'
 import {
+  describeWithholdingCause,
   inspectToolResultForCopilot,
   projectToolErrorMessageForCopilot,
 } from '@/lib/copilot/request/tools/resolved-secret-result'
@@ -158,6 +159,7 @@ export const POST = withRouteHandler((request: NextRequest) =>
             error: projected.error,
             runtimeSucceeded: result.success,
             projectionSafe: projection.safe,
+            ...(projection.safe ? {} : describeWithholdingCause(projection.cause)),
           })
         }
         if (result.success && chatId) {

apps/sim/executor/utils/errors.ts

@@ -36,6 +36,33 @@ export function attachExecutionResult(error: Error, executionResult: ExecutionRe
   Object.assign(error, { executionResult })
 }
 
+const ATTEMPTED_EXECUTION_ID = 'attemptedExecutionId'
+
+/**
+ * Names the run a failure belongs to once dispatch has been attempted.
+ *
+ * A caller that only sees the thrown error cannot tell an authorization refusal — which
+ * created nothing — from a crash after the run was already dispatched, and those need
+ * opposite retry decisions. Attaching the id at the point of no return makes its absence
+ * mean "nothing was started" rather than "we do not know", and its presence a key that
+ * resolves to zero or one executions.
+ *
+ * Distinct from {@link attachExecutionResult}: that says the workflow ran and produced a
+ * result, this says only that it was dispatched.
+ */
+export function attachAttemptedExecutionId(error: unknown, executionId: string): void {
+  if (!(error instanceof Error) || !executionId) return
+  if (ATTEMPTED_EXECUTION_ID in error) return
+  Object.assign(error, { [ATTEMPTED_EXECUTION_ID]: executionId })
+}
+
+/** Reads the dispatched-run id an error carries, if dispatch was reached at all. */
+export function readAttemptedExecutionId(error: unknown): string | undefined {
+  if (!(error instanceof Error) || !(ATTEMPTED_EXECUTION_ID in error)) return undefined
+  const value = (error as Error & Record<string, unknown>)[ATTEMPTED_EXECUTION_ID]
+  return typeof value === 'string' && value.length > 0 ? value : undefined
+}
+
 export interface BlockExecutionErrorDetails {
   block: SerializedBlock
   error: Error | string

apps/sim/lib/copilot/request/tools/executor.ts

@@ -61,7 +61,10 @@ import {
   setTerminalToolCallState,
 } from '@/lib/copilot/request/tool-call-state'
 import { maybeWriteOutputToFile } from '@/lib/copilot/request/tools/files'
-import { inspectToolResultForCopilot } from '@/lib/copilot/request/tools/resolved-secret-result'
+import {
+  describeWithholdingCause,
+  inspectToolResultForCopilot,
+} from '@/lib/copilot/request/tools/resolved-secret-result'
 import { handleResourceSideEffects } from '@/lib/copilot/request/tools/resources'
 import {
   maybeWriteOutputToTable,
@@ -737,15 +740,20 @@ async function executeToolAndReportInner(
     toolSpan.attributes = {
       ...toolSpan.attributes,
       ...summarizeToolResultForSpan(copilotResult),
-      ...(projection.safe ? {} : { resultWithheld: true }),
+      ...(projection.safe
+        ? {}
+        : { resultWithheld: true, ...describeWithholdingCause(projection.cause) }),
     }
     if (!projection.safe) {
       // A withheld SUCCESS otherwise leaves no trace anywhere: the span reads
       // ok and the model just sees a bare `{success: true}` with no output.
+      // The cause is what says whether a guard latched, no catalog was built,
+      // or the payload itself was unprojectable — three different fixes.
       logger.warn('Tool result withheld by egress projection', {
         toolCallId: toolCall.id,
         toolName: toolCall.name,
         runtimeSucceeded: result.success,
+        ...describeWithholdingCause(projection.cause),
       })
     }
 

apps/sim/lib/copilot/request/tools/resolved-secret-result.test.ts

@@ -4,6 +4,8 @@
 import { describe, expect, it } from 'vitest'
 import { RunCode, RunFunction } from '@/lib/copilot/generated/tool-catalog-v1'
 import {
+  describeWithholdingCause,
+  inspectToolResultForCopilot,
   projectToolResultForCopilot,
   READ_TOOL_RESULT_UNAVAILABLE_ERROR,
   TOOL_RESULT_UNAVAILABLE_ERROR,
@@ -457,3 +459,88 @@ describe('projectToolResultForCopilot', () => {
     expect(toolResultUnavailableError(undefined)).toBe(TOOL_RESULT_UNAVAILABLE_ERROR)
   })
 })
+
+describe('effect disclosure on a withheld result', () => {
+  const EXECUTION_ID = '0f4d5a4c-6a1e-4c2f-9b7d-2c8f1a3e5d90'
+
+  it('carries nothing extra for a tool that declared no effect', () => {
+    expect(projectToolResultForCopilot({ success: true, output: { a: 1 } }, undefined)).toEqual({
+      success: true,
+    })
+    expect(projectToolResultForCopilot({ success: false, error: 'why' }, undefined)).toEqual({
+      success: false,
+      error: TOOL_RESULT_UNAVAILABLE_ERROR,
+    })
+  })
+
+  /**
+   * The exemption is what makes the disclosure trustworthy, so it has to be all or
+   * nothing: a disclosure that silently dropped the id it could not vouch for would
+   * read exactly like one that never had a run to name.
+   */
+  it('voids the whole disclosure when an id is not a shape this system mints', () => {
+    expect(
+      projectToolResultForCopilot(
+        {
+          success: false,
+          error: 'why',
+          effect: { phase: 'performed', ids: { executionId: 'sk-live-9Qv2XbTn4LmZa8Rd' } },
+        },
+        undefined,
+        'run_workflow'
+      )
+    ).toEqual({ success: false, error: TOOL_RESULT_UNAVAILABLE_ERROR })
+  })
+
+  it('reports the phase and ids when every id is vouchable', () => {
+    expect(
+      projectToolResultForCopilot(
+        {
+          success: false,
+          error: 'why',
+          effect: { phase: 'attempted', ids: { executionId: EXECUTION_ID } },
+        },
+        undefined,
+        'run_workflow'
+      )
+    ).toEqual({
+      success: false,
+      output: { resultWithheld: true, effect: 'attempted', executionId: EXECUTION_ID },
+      error: expect.stringContaining('At most one run exists'),
+    })
+  })
+
+  it('never leaks the disclosure into a result that projected cleanly', () => {
+    const registry = new ResolvedSecretTraceRegistry()
+
+    expect(
+      projectToolResultForCopilot(
+        {
+          success: true,
+          output: { executionId: EXECUTION_ID },
+          effect: { phase: 'performed', ids: { executionId: EXECUTION_ID } },
+        },
+        registry,
+        'run_workflow'
+      )
+    ).toEqual({ success: true, output: { executionId: EXECUTION_ID } })
+  })
+
+  it('names why the content was withheld, for the surface about to log it', () => {
+    const latched = createRegistry()
+    latched.markIncomplete('source-provenance-incomplete', { origin: 'test.origin' })
+
+    const projection = inspectToolResultForCopilot({ success: false }, latched, 'run_workflow')
+    expect(projection.safe).toBe(false)
+    // The per-call fork adds its own propagation reason; the guard that originally
+    // tripped has to survive alongside it, or a refusal names only the messenger.
+    expect(projection.safe === false && describeWithholdingCause(projection.cause)).toEqual({
+      withheldCause: 'registry-incomplete',
+      withheldReasons: expect.arrayContaining(['source-provenance-incomplete']),
+      withheldOrigins: ['test.origin'],
+    })
+
+    const absent = inspectToolResultForCopilot({ success: false }, undefined)
+    expect(absent.safe === false && absent.cause).toEqual({ kind: 'registry-absent' })
+  })
+})

apps/sim/lib/copilot/request/tools/resolved-secret-result.ts

@@ -1,6 +1,10 @@
-import type { ToolExecutionResult } from '@/lib/copilot/tool-executor/types'
+import type { ToolCallEffect, ToolExecutionResult } from '@/lib/copilot/tool-executor/types'
+import { TOOL_EFFECT_PHASE } from '@/lib/copilot/tool-executor/types'
 import { projectResolvedSecretModelJsonContent } from '@/executor/utils/resolved-secret-content-projection'
-import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
+import type {
+  ResolvedSecretIncompletenessReason,
+  ResolvedSecretTraceRegistry,
+} from '@/executor/utils/resolved-secret-trace-registry'
 
 export const TOOL_RESULT_UNAVAILABLE_ERROR =
   'Tool execution settled, but its result could not be returned safely. Do not retry a mutation automatically.'
@@ -13,33 +17,131 @@ export const TOOL_RESULT_UNAVAILABLE_ERROR =
 export const READ_TOOL_RESULT_UNAVAILABLE_ERROR =
   'Tool executed, but its result could not be returned safely. The call was read-only, so you may retry it or continue without the result.'
 
+/**
+ * Withheld-result wording for a call that disclosed how far its side effect got.
+ *
+ * The generic message above has to cover both "nothing happened" and "it happened,
+ * you just cannot see it", which is why a caller could not build a retry policy from
+ * it: a rejected call and a completed mutation read identically. A tool that declares
+ * its {@link ToolCallEffect} gets the phrasing its phase actually warrants.
+ */
+const WITHHELD_ERROR_BY_EFFECT_PHASE: Record<ToolCallEffect['phase'], string> = {
+  [TOOL_EFFECT_PHASE.notAttempted]:
+    'Tool call was rejected before it ran, so nothing was created or changed. The reason could not be returned safely — correct the call and try again.',
+  [TOOL_EFFECT_PHASE.attempted]:
+    'Tool execution was dispatched but its outcome could not be returned safely. At most one run exists for the ids in this result — resolve it before retrying a mutation.',
+  [TOOL_EFFECT_PHASE.performed]:
+    'Tool execution completed but its result could not be returned safely. Do not retry — read the outcome using the ids in this result.',
+}
+
 const READ_ONLY_RESULT_TOOLS = new Set(['read', 'glob', 'grep'])
 
+/**
+ * The shape of an identifier this system mints — `generateId`'s UUID, and the
+ * database ids that share it. Effect ids bypass secret projection, so the set of
+ * values that may occupy one is pinned to a syntax no credential we issue or store
+ * takes. A caller with a differently shaped id has to widen this deliberately,
+ * where the exemption is reviewed, rather than by passing it.
+ */
+const SERVER_MINTED_ID_PATTERN =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
+
 /** Chooses the withheld-result message a tool's caller should surface. */
 export function toolResultUnavailableError(toolId?: string): string {
   return toolId && READ_ONLY_RESULT_TOOLS.has(toolId)
     ? READ_TOOL_RESULT_UNAVAILABLE_ERROR
     : TOOL_RESULT_UNAVAILABLE_ERROR
 }
 
+/**
+ * Why complete content could not cross, for the caller that is about to log a refusal.
+ *
+ * The three causes need different fixes — a latched registry names the guard that tripped,
+ * an absent one means the surface never built a catalog, and a content refusal means the
+ * registry was fine and the payload itself was unprojectable — so they are not collapsed.
+ */
+export type ToolResultWithholdingCause =
+  | {
+      kind: 'registry-incomplete'
+      reasons: readonly ResolvedSecretIncompletenessReason[]
+      origins: readonly string[]
+    }
+  | { kind: 'registry-absent' }
+  | { kind: 'content-refused' }
+
+export type CopilotToolResultProjection =
+  | { safe: true; result: ToolExecutionResult }
+  | { safe: false; result: ToolExecutionResult; cause: ToolResultWithholdingCause }
+
 function structuralResult(result: ToolExecutionResult): ToolExecutionResult {
   return { success: result.success === true }
 }
 
+/**
+ * Reduces a withheld result to the facts the tool asserted about the call itself.
+ *
+ * Content is dropped because nothing here can prove it secret-free. The effect
+ * disclosure survives because it is not derived from content: the phase is a
+ * code-defined literal and every id is checked against {@link SERVER_MINTED_ID_PATTERN}.
+ * An id that fails that check voids the whole disclosure rather than being dropped
+ * on its own — a partially honoured exemption is the one shape a reader would
+ * misread as complete.
+ */
 function omittedResult(result: ToolExecutionResult, toolId?: string): ToolExecutionResult {
-  if (result.success) return { success: true }
-  return { success: false, error: toolResultUnavailableError(toolId) }
+  const effect = vouchableEffect(result.effect)
+  if (!effect) {
+    return result.success
+      ? { success: true }
+      : { success: false, error: toolResultUnavailableError(toolId) }
+  }
+
+  return {
+    success: result.success === true,
+    output: { resultWithheld: true, effect: effect.phase, ...effect.ids },
+    ...(result.success ? {} : { error: WITHHELD_ERROR_BY_EFFECT_PHASE[effect.phase] }),
+  }
 }
 
-export type CopilotToolResultProjection =
-  | { safe: true; result: ToolExecutionResult }
-  | { safe: false; result: ToolExecutionResult }
+/** Returns the disclosure only when every id it carries is a shape this system mints. */
+function vouchableEffect(effect: ToolCallEffect | undefined): ToolCallEffect | undefined {
+  if (!effect) return undefined
+  for (const value of Object.values(effect.ids ?? {})) {
+    if (typeof value !== 'string' || !SERVER_MINTED_ID_PATTERN.test(value)) return undefined
+  }
+  return effect
+}
+
+function withholdingCause(
+  registry: ResolvedSecretTraceRegistry | undefined
+): ToolResultWithholdingCause {
+  if (!registry) return { kind: 'registry-absent' }
+  const diagnostics = registry.getIncompletenessDiagnostics()
+  return diagnostics
+    ? {
+        kind: 'registry-incomplete',
+        reasons: diagnostics.reasons,
+        origins: diagnostics.origins,
+      }
+    : { kind: 'content-refused' }
+}
+
+function withheld(
+  result: ToolExecutionResult,
+  registry: ResolvedSecretTraceRegistry | undefined,
+  toolId: string | undefined
+): CopilotToolResultProjection {
+  return {
+    safe: false,
+    result: omittedResult(result, toolId),
+    cause: withholdingCause(registry),
+  }
+}
 
 /**
  * Projects terminal tool content and reports whether the complete content was safe to cross.
  * Callers that isolate provenance per tool call may merge that child registry only when `safe`
  * is true and the child is complete. The returned result is always safe to expose: an unsafe
- * projection is reduced to a structural success or failure.
+ * projection is reduced to a structural success or failure, plus any effect the tool disclosed.
  */
 export function inspectToolResultForCopilot(
   result: ToolExecutionResult,
@@ -54,15 +156,15 @@ export function inspectToolResultForCopilot(
     if (Object.hasOwn(result, 'error')) content.error = result.error
     const projection = projectResolvedSecretModelJsonContent(content, resultRegistry)
     if (!projection.safe || !projection.value || typeof projection.value !== 'object') {
-      return { safe: false, result: omittedResult(result, toolId) }
+      return withheld(result, resultRegistry, toolId)
     }
 
     const projectedContent = projection.value as Record<string, unknown>
     const projected = structuralResult(result)
     if (Object.hasOwn(projectedContent, 'output')) projected.output = projectedContent.output
     if (Object.hasOwn(projectedContent, 'error')) {
       if (typeof projectedContent.error !== 'string') {
-        return { safe: false, result: omittedResult(result, toolId) }
+        return withheld(result, resultRegistry, toolId)
       }
       projected.error = projectedContent.error
     }
@@ -74,7 +176,7 @@ export function inspectToolResultForCopilot(
     }
     return { safe: true, result: projected }
   } catch {
-    return { safe: false, result: omittedResult(result, toolId) }
+    return withheld(result, registry, toolId)
   }
 }
 
@@ -98,3 +200,16 @@ export function projectToolErrorMessageForCopilot(
 ): string {
   return projectToolResultForCopilot({ success: false, error }, registry, toolId).error ?? ''
 }
+
+/** Flattens a withholding cause into log/span fields, so every surface reports it alike. */
+export function describeWithholdingCause(
+  cause: ToolResultWithholdingCause
+): Record<string, unknown> {
+  return cause.kind === 'registry-incomplete'
+    ? {
+        withheldCause: cause.kind,
+        withheldReasons: [...cause.reasons],
+        ...(cause.origins.length > 0 ? { withheldOrigins: [...cause.origins] } : {}),
+      }
+    : { withheldCause: cause.kind }
+}