improvement(provenance): attribute stored-envelope display reads to their execution

A display materialization of an execution log imports the row's stored
provenance envelopes into throwaway registries, and each import of an
incomplete envelope re-emitted the registry's own summary — per envelope,
per view, carrying counts and a workspace but never the execution id. A
reader repeatedly materializing the same stored rows produced hundreds of
identical lines that could not say which executions to go look at, and
the volume scaled with views of a state that was fully recorded when the
run wrote it.

Verified against production before changing anything: essentially no new
incomplete envelopes are being stored since the writer fix shipped, and
no data drains exist — the stream is bounded re-reads of old rows through
the display paths, not a live producer.

The display registries are now staged — the existing concept for a
registry that filters one value for a caller that reports against the
real boundary — and each display function reports once per
materialization with the execution id, workflow, workspace, and the
parts that could not be vouched for. Severity is preserved: an
incomplete stored envelope stays at warn, a malformed one stays at
error. Projection behavior is unchanged everywhere — incomplete and
malformed envelopes still fail their values closed exactly as before;
only the reporting moves to the boundary that knows the execution.

Author: icecrasher321

apps/sim/lib/logs/execution/trace-store.test.ts

@@ -3,10 +3,16 @@
  */
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { decryptSecretMock, materializeLargeValueRefMock, storeLargeValueMock } = vi.hoisted(() => ({
-  decryptSecretMock: vi.fn(),
-  materializeLargeValueRefMock: vi.fn(),
-  storeLargeValueMock: vi.fn(),
+const { decryptSecretMock, materializeLargeValueRefMock, storeLargeValueMock, mockLogger } =
+  vi.hoisted(() => ({
+    decryptSecretMock: vi.fn(),
+    materializeLargeValueRefMock: vi.fn(),
+    storeLargeValueMock: vi.fn(),
+    mockLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
+  }))
+
+vi.mock('@sim/logger', () => ({
+  createLogger: () => mockLogger,
 }))
 
 vi.mock('@/lib/core/security/encryption', () => ({
@@ -536,3 +542,138 @@ describe('projectExecutionDataForDisplay provenance handling', () => {
     expect(displayData.traceSpans).toEqual([])
   })
 })
+
+describe('stored provenance display reporting', () => {
+  const REGISTRY_SUMMARY_MESSAGES = [
+    'Resolved secret registry marked incomplete',
+    'Resolved secret input path marked incomplete',
+  ]
+
+  function registrySummaryLines(): unknown[] {
+    return [...mockLogger.warn.mock.calls, ...mockLogger.error.mock.calls].filter(([message]) =>
+      REGISTRY_SUMMARY_MESSAGES.includes(message as string)
+    )
+  }
+
+  /**
+   * The stored state was recorded when the run wrote it; a view re-deriving it must say which
+   * execution it served, once — not restate the latch through registry summaries that name none.
+   */
+  it('reports an incomplete stored envelope once, naming the execution and the parts', async () => {
+    const displayData = await projectExecutionDataForDisplay(
+      {
+        finalOutput: { result: 'value' },
+        executionState: {
+          resolvedSecretTraceProvenance: { version: 1, complete: false, entries: [] },
+          finalOutputResolvedSecretTraceProvenance: { version: 1, complete: false, entries: [] },
+        },
+      },
+      CONTEXT
+    )
+
+    expect(displayData).not.toHaveProperty('finalOutput')
+    expect(registrySummaryLines()).toHaveLength(0)
+    expect(mockLogger.warn).toHaveBeenCalledWith(
+      'Stored execution provenance cannot vouch for display content',
+      expect.objectContaining({
+        site: 'traceStore.displayProjection',
+        executionId: 'execution-1',
+        workflowId: 'workflow-1',
+        workspaceId: 'workspace-1',
+        parts: ['traceSpans', 'finalOutput'],
+        partCount: 2,
+      })
+    )
+    expect(mockLogger.error).not.toHaveBeenCalled()
+  })
+
+  it('reports a malformed stored envelope at error, keeping the value withheld', async () => {
+    const displayData = await projectExecutionDataForDisplay(
+      {
+        finalOutput: { result: 'value' },
+        executionState: {
+          resolvedSecretTraceProvenance: {
+            version: 1,
+            complete: true,
+            entries: [],
+            scope: { userId: 'user-1', workspaceId: 'workspace-1' },
+          },
+          finalOutputResolvedSecretTraceProvenance: 'garbage',
+        },
+      },
+      CONTEXT
+    )
+
+    expect(displayData).not.toHaveProperty('finalOutput')
+    expect(registrySummaryLines()).toHaveLength(0)
+    expect(mockLogger.error).toHaveBeenCalledWith(
+      'Stored execution provenance is malformed',
+      expect.objectContaining({
+        site: 'traceStore.displayProjection',
+        executionId: 'execution-1',
+        parts: ['finalOutput'],
+      })
+    )
+  })
+
+  it('stays silent when every stored envelope is complete', async () => {
+    const displayData = await projectExecutionDataForDisplay(
+      {
+        finalOutput: { result: 'direct-literal' },
+        executionState: {
+          resolvedSecretTraceProvenance: {
+            version: 1,
+            complete: true,
+            entries: [],
+            scope: { userId: 'user-1', workspaceId: 'workspace-1' },
+          },
+          finalOutputResolvedSecretTraceProvenance: {
+            version: 1,
+            complete: true,
+            entries: [],
+            scope: { userId: 'user-1', workspaceId: 'workspace-1' },
+          },
+        },
+      },
+      CONTEXT
+    )
+
+    expect(displayData.finalOutput).toEqual({ result: 'direct-literal' })
+    expect(mockLogger.warn).not.toHaveBeenCalled()
+    expect(mockLogger.error).not.toHaveBeenCalled()
+  })
+
+  it('reports incomplete block-output envelopes once for the whole block read', async () => {
+    const result = await materializeExecutionDataForDisplayWithBlockOutputs(
+      {
+        executionState: {
+          resolvedSecretTraceProvenance: {
+            version: 1,
+            complete: true,
+            entries: [],
+            scope: { userId: 'user-1', workspaceId: 'workspace-1' },
+          },
+          blockStates: {
+            'block-1': {
+              output: { value: 1 },
+              resolvedSecretTraceProvenance: { version: 1, complete: false, entries: [] },
+            },
+          },
+        },
+      },
+      CONTEXT,
+      ['block-1']
+    )
+
+    expect(result.blockOutputs.has('block-1')).toBe(false)
+    expect(registrySummaryLines()).toHaveLength(0)
+    expect(mockLogger.warn).toHaveBeenCalledWith(
+      'Stored execution provenance cannot vouch for display content',
+      expect.objectContaining({
+        site: 'traceStore.blockOutputs',
+        executionId: 'execution-1',
+        parts: ['blockOutput:block-1'],
+      })
+    )
+  })
+})

apps/sim/lib/logs/execution/trace-store.ts

@@ -300,11 +300,16 @@ export async function materializeExecutionDataForDisplayWithBlockOutputs(
     return { executionData: displayData, blockOutputs: new Map() }
   }
 
+  const runProvenance =
+    materialized[RESOLVED_SECRET_PROVENANCE_KEY] ?? executionState?.[RESOLVED_SECRET_PROVENANCE_KEY]
   const runRegistry = await importResolvedSecretTraceRegistry(
-    materialized[RESOLVED_SECRET_PROVENANCE_KEY] ??
-      executionState?.[RESOLVED_SECRET_PROVENANCE_KEY],
+    runProvenance,
     'traceStore.blockOutputRunProvenance'
   )
+  const provenanceFaults = new Map<string, StoredDisplayProvenanceFault>()
+  if (isResolvedSecretTraceProvenanceV1(runProvenance) && !runProvenance.complete) {
+    provenanceFaults.set('run', 'incomplete')
+  }
   const blockOutputs = new Map<string, unknown>()
   const projectionStore = createReadOnlyProjectionStore(context)
 
@@ -313,6 +318,12 @@ export async function materializeExecutionDataForDisplayWithBlockOutputs(
     if (!blockState || blockState.output === undefined) continue
 
     const hasExactProvenance = Object.hasOwn(blockState, RESOLVED_SECRET_PROVENANCE_KEY)
+    if (hasExactProvenance) {
+      const blockProvenance = blockState[RESOLVED_SECRET_PROVENANCE_KEY]
+      if (isResolvedSecretTraceProvenanceV1(blockProvenance) && !blockProvenance.complete) {
+        provenanceFaults.set(`blockOutput:${blockId}`, 'incomplete')
+      }
+    }
     const registry = hasExactProvenance
       ? await importResolvedSecretTraceRegistry(
           blockState[RESOLVED_SECRET_PROVENANCE_KEY],
@@ -338,6 +349,7 @@ export async function materializeExecutionDataForDisplayWithBlockOutputs(
       blockOutputs.set(blockId, projected.output.value)
     }
   }
+  reportStoredDisplayProvenanceFaults('traceStore.blockOutputs', context, provenanceFaults)
 
   return { executionData: displayData, blockOutputs }
 }
@@ -346,17 +358,67 @@ function readRecord(value: unknown): Record<string, unknown> | undefined {
   return isRecordLike(value) ? (value as Record<string, unknown>) : undefined
 }
 
+/**
+ * Staged: display registries filter stored values for one materialization and are discarded, and
+ * their own mark-time summaries name no execution — the read boundary reports instead, through
+ * {@link reportStoredDisplayProvenanceFaults}. A stored envelope's incompleteness is not an event
+ * on this path; it was recorded when the run wrote it, and every later view re-derives it.
+ */
 async function importResolvedSecretTraceRegistry(
   provenance: unknown,
   origin: string
 ): Promise<ResolvedSecretTraceRegistry | undefined> {
   if (!isResolvedSecretTraceProvenanceV1(provenance)) return undefined
 
-  const registry = new ResolvedSecretTraceRegistry([], provenance.scope)
+  const registry = new ResolvedSecretTraceRegistry([], provenance.scope, { staged: true })
   await registry.importProvenance(provenance, { trusted: true, origin })
   return registry
 }
 
+type StoredDisplayProvenanceFault = 'incomplete' | 'malformed'
+
+const MAX_REPORTED_PROVENANCE_FAULT_PARTS = 20
+
+/**
+ * One attributed line per display materialization, in place of one registry summary per envelope
+ * per view.
+ *
+ * The registry summaries these replace carried counts and a workspace but no execution id, so a
+ * reader repeatedly materializing the same stored rows produced an unattributable stream — the
+ * lines could not say which executions to go look at. Incomplete stays at warn (a stored state
+ * being re-read); malformed stays at error (a stored envelope that cannot be parsed is a fault
+ * wherever it is met, matching the level its registry reason carries elsewhere).
+ */
+function reportStoredDisplayProvenanceFaults(
+  site: string,
+  context: TraceStoreReadContext,
+  faults: ReadonlyMap<string, StoredDisplayProvenanceFault>
+): void {
+  if (faults.size === 0) return
+  const partsByFault = { incomplete: [] as string[], malformed: [] as string[] }
+  for (const [part, fault] of faults) partsByFault[fault].push(part)
+  const details = {
+    site,
+    executionId: context.executionId,
+    ...(context.workflowId ? { workflowId: context.workflowId } : {}),
+    ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
+  }
+  if (partsByFault.incomplete.length > 0) {
+    logger.warn('Stored execution provenance cannot vouch for display content', {
+      ...details,
+      parts: partsByFault.incomplete.slice(0, MAX_REPORTED_PROVENANCE_FAULT_PARTS),
+      partCount: partsByFault.incomplete.length,
+    })
+  }
+  if (partsByFault.malformed.length > 0) {
+    logger.error('Stored execution provenance is malformed', {
+      ...details,
+      parts: partsByFault.malformed.slice(0, MAX_REPORTED_PROVENANCE_FAULT_PARTS),
+      partCount: partsByFault.malformed.length,
+    })
+  }
+}
+
 function createReadOnlyProjectionStore(context: TraceStoreReadContext) {
   return {
     workspaceId: context.workspaceId ?? undefined,
@@ -426,6 +488,11 @@ export async function projectExecutionDataForDisplay(
 
   const projectionStore = createReadOnlyProjectionStore(context)
 
+  const provenanceFaults = new Map<string, StoredDisplayProvenanceFault>()
+  if (isResolvedSecretTraceProvenanceV1(provenance) && !provenance.complete) {
+    provenanceFaults.set('traceSpans', 'incomplete')
+  }
+
   const exactValueProjections = new Map<string, unknown>()
   for (const [valueKey, provenanceKey] of Object.entries(EXACT_LOG_VALUE_PROVENANCE_KEYS)) {
     if (
@@ -438,14 +505,16 @@ export async function projectExecutionDataForDisplay(
 
     const exactProvenance = executionState[provenanceKey]
     const exactRegistry = isResolvedSecretTraceProvenanceV1(exactProvenance)
-      ? new ResolvedSecretTraceRegistry([], exactProvenance.scope)
-      : new ResolvedSecretTraceRegistry()
+      ? new ResolvedSecretTraceRegistry([], exactProvenance.scope, { staged: true })
+      : new ResolvedSecretTraceRegistry([], undefined, { staged: true })
     if (isResolvedSecretTraceProvenanceV1(exactProvenance)) {
+      if (!exactProvenance.complete) provenanceFaults.set(valueKey, 'incomplete')
       await exactRegistry.importProvenance(exactProvenance, {
         trusted: true,
         origin: 'traceStore.exactProvenance',
       })
     } else {
+      provenanceFaults.set(valueKey, 'malformed')
       exactRegistry.markIncomplete('untrusted-provenance', { origin: 'traceStore.exactProvenance' })
     }
 
@@ -467,6 +536,7 @@ export async function projectExecutionDataForDisplay(
       exactValueProjections.set(valueKey, projected.output.value)
     }
   }
+  reportStoredDisplayProvenanceFaults('traceStore.displayProjection', context, provenanceFaults)
 
   const envelope: Record<string, unknown> = {}
   for (const key of LOG_DISPLAY_CONTENT_KEYS) {