Skip to content

Commit 95a3277

Browse files
icecrasher321icecrasher321
committed
fix(provenance): feature flagged, inexact sidecars
1 parent 90a76dd commit 95a3277

14 files changed

Lines changed: 577 additions & 73 deletions

‎apps/docs/content/docs/en/platform/self-hosting/environment-variables.mdx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ See [Observability](/platform/self-hosting/observability).
181181
| `NEXT_PUBLIC_CHAT_DISABLED` | Set to `true` to hide the Chat module: the workspace lands on your first workflow, with no chats list, scheduled tasks, or editor Chat panel. Chat is shown when unset; `bun run setup` sets it for you if you skip the chat key |
182182
| `PII_REDACTION` | Redact PII from workflow logs via Data Retention rules; requires the PII service and a cluster-reachable `INTERNAL_API_BASE_URL` |
183183
| `PII_GRANULAR_REDACTION` | Additionally expose the execution-altering redaction stages |
184+
| `DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES` | Durable stores where a value whose secret provenance was never recorded fails the run instead of logging a warning. `all`, or a comma-separated subset of `memory`, `table-row`, `knowledge`. Unset (nothing enforced) by default |
184185
| `ADMIN_API_KEY` | Admin API key for GitOps operations and organization provisioning |
185186

186187
## Enterprise Features

‎apps/sim/app/api/knowledge/search/route.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -604,7 +604,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
604604
!(await importDurableSecretProvenance(
605605
resultSecretRegistry,
606606
metadata.provenance,
607-
renderedMetadata
607+
renderedMetadata,
608+
'knowledge'
608609
))
609610
) {
610611
resultSecretRegistry.markIncomplete()

‎apps/sim/app/api/memory/secret-provenance.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ export async function createMemoryResponse(options: {
137137
status: sidecar?.status ?? null,
138138
entries: sidecar?.entries,
139139
})
140-
await importDurableSecretProvenance(registry, provenance, record.data)
140+
await importDurableSecretProvenance(registry, provenance, record.data, 'memory')
141141
}
142142
}
143143
}

‎apps/sim/executor/handlers/agent/memory.ts‎

Lines changed: 38 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ import {
1111
importDurableSecretProvenance,
1212
mergeDurableSecretProvenance,
1313
} from '@/lib/execution/durable-secret-provenance'
14+
import {
15+
isDurableSecretProvenanceEnforced,
16+
reportUnrecordedDurableProvenance,
17+
} from '@/lib/execution/durable-secret-provenance-enforcement'
1418
import { redactObjectStrings } from '@/lib/logs/execution/pii-redaction'
1519
import {
1620
readBoundMemorySecretProvenance,
@@ -75,16 +79,32 @@ export class Memory {
7579
stored.provenance,
7680
messages
7781
)
78-
if (
79-
selectedProvenance.status === 'unknown' ||
80-
(selectedProvenance.entries.length > 0 && !ctx.resolvedSecretTraceRegistry) ||
81-
(ctx.resolvedSecretTraceRegistry &&
82-
!(await importDurableSecretProvenance(
83-
ctx.resolvedSecretTraceRegistry,
84-
selectedProvenance,
85-
messages
86-
)))
87-
) {
82+
/**
83+
* Unrecorded provenance is checked through the same policy the shared import uses, so stored
84+
* memory written by a run that could not vouch does not permanently refuse every later turn.
85+
*/
86+
let refuseStoredProvenance: boolean
87+
if (selectedProvenance.status === 'unknown') {
88+
refuseStoredProvenance = isDurableSecretProvenanceEnforced('memory')
89+
if (!refuseStoredProvenance) {
90+
reportUnrecordedDurableProvenance({
91+
surface: 'memory',
92+
cause: 'stored-memory-provenance-unknown',
93+
...(ctx.workspaceId ? { workspaceId: ctx.workspaceId } : {}),
94+
})
95+
}
96+
} else {
97+
refuseStoredProvenance =
98+
(selectedProvenance.entries.length > 0 && !ctx.resolvedSecretTraceRegistry) ||
99+
(ctx.resolvedSecretTraceRegistry !== undefined &&
100+
!(await importDurableSecretProvenance(
101+
ctx.resolvedSecretTraceRegistry,
102+
selectedProvenance,
103+
messages,
104+
'memory'
105+
)))
106+
}
107+
if (refuseStoredProvenance) {
88108
refuseResolvedSecretProjection({
89109
site: 'memory.storedProvenanceImport',
90110
message: MEMORY_CONTENT_REFUSAL,
@@ -102,7 +122,14 @@ export class Memory {
102122
[],
103123
ctx.resolvedSecretTraceRegistry?.exportProvenance().scope
104124
)
105-
if (!(await importDurableSecretProvenance(modelRegistry, messageProvenance, message))) {
125+
if (
126+
!(await importDurableSecretProvenance(
127+
modelRegistry,
128+
messageProvenance,
129+
message,
130+
'memory'
131+
))
132+
) {
106133
refuseResolvedSecretProjection({
107134
site: 'memory.messageProvenanceImport',
108135
message: MEMORY_CONTENT_REFUSAL,

‎apps/sim/executor/utils/resolved-secret-trace-registry.test.ts‎

Lines changed: 74 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1056,11 +1056,13 @@ describe('ResolvedSecretTraceRegistry', () => {
10561056
})
10571057
})
10581058

1059-
it('marks a bounded cross-boundary scan incomplete when an enumerable accessor is opaque', () => {
1059+
it('keeps every candidate when a bounded cross-boundary scan hits an opaque accessor', () => {
10601060
const registry = new ResolvedSecretTraceRegistry([
10611061
{ name: 'TOKEN', plaintext: 'secret', encryptedValue: 'ciphertext' },
1062+
{ name: 'ABSENT', plaintext: 'never-present', encryptedValue: 'absent-ciphertext' },
10621063
])
10631064
registry.recordResolved('TOKEN', 'secret')
1065+
registry.recordResolved('ABSENT', 'never-present')
10641066
const value = {}
10651067
Object.defineProperty(value, 'opaque', {
10661068
enumerable: true,
@@ -1069,16 +1071,18 @@ describe('ResolvedSecretTraceRegistry', () => {
10691071

10701072
expect(registry.exportProvenanceForValue(value, { anonymous: true })).toEqual({
10711073
version: 1,
1072-
complete: false,
1073-
entries: [],
1074+
complete: true,
1075+
entries: [{ encryptedValue: 'absent-ciphertext' }, { encryptedValue: 'ciphertext' }],
10741076
})
10751077
})
10761078

1077-
it('does not claim a complete cross-boundary scan for opaque large-value refs', () => {
1079+
it('keeps every candidate rather than voiding provenance for an opaque large-value ref', () => {
10781080
const registry = new ResolvedSecretTraceRegistry([
10791081
{ name: 'TOKEN', plaintext: 'secret', encryptedValue: 'ciphertext' },
1082+
{ name: 'ABSENT', plaintext: 'never-present', encryptedValue: 'absent-ciphertext' },
10801083
])
10811084
registry.recordResolved('TOKEN', 'secret')
1085+
registry.recordResolved('ABSENT', 'never-present')
10821086

10831087
expect(
10841088
registry.exportProvenanceForValue(
@@ -1091,6 +1095,70 @@ describe('ResolvedSecretTraceRegistry', () => {
10911095
},
10921096
{ anonymous: true }
10931097
)
1098+
).toEqual({
1099+
version: 1,
1100+
complete: true,
1101+
entries: [{ encryptedValue: 'absent-ciphertext' }, { encryptedValue: 'ciphertext' }],
1102+
})
1103+
})
1104+
1105+
it('lets a model input path survive an upstream output the scan could not read', async () => {
1106+
const scope = { userId: 'user-1', workspaceId: 'workspace-1' }
1107+
const catalog = [
1108+
{ name: 'TOKEN', plaintext: 'decrypted:ciphertext', encryptedValue: 'ciphertext' },
1109+
]
1110+
const producer = new ResolvedSecretTraceRegistry(catalog, scope)
1111+
producer.recordResolved('TOKEN', 'decrypted:ciphertext')
1112+
1113+
/** A block output past the traversal bound, exactly as compaction leaves a large table read. */
1114+
const upstreamOutput = {
1115+
rows: Array.from({ length: 5_000 }, (_, index) => ({
1116+
id: `row_${index}`,
1117+
a: 'a',
1118+
b: 'b',
1119+
c: 'c',
1120+
d: 'd',
1121+
e: 'e',
1122+
f: 'f',
1123+
g: 'g',
1124+
h: 'h',
1125+
i: 'i',
1126+
j: 'j',
1127+
})),
1128+
}
1129+
const upstreamProvenance = producer.exportCommittedProvenanceForValue(upstreamOutput)
1130+
expect(upstreamProvenance.complete).toBe(true)
1131+
1132+
const consumer = new ResolvedSecretTraceRegistry(catalog, scope)
1133+
await consumer.importProvenanceForValueAtInputPath(
1134+
upstreamProvenance,
1135+
upstreamOutput,
1136+
['userPrompt'],
1137+
{ trusted: true }
1138+
)
1139+
1140+
const modelFork = consumer.forkForInputPaths([['userPrompt'], ['systemPrompt']])
1141+
expect(modelFork.projectResolvedInputSelection({ userPrompt: 'classify these rows' })).toEqual({
1142+
complete: true,
1143+
value: { userPrompt: 'classify these rows' },
1144+
})
1145+
})
1146+
1147+
it('still voids provenance for an unscannable value when the registry cannot vouch', () => {
1148+
const registry = new ResolvedSecretTraceRegistry([
1149+
{ name: 'TOKEN', plaintext: 'secret', encryptedValue: 'ciphertext' },
1150+
])
1151+
registry.recordResolved('TOKEN', 'secret')
1152+
registry.markIncomplete('unverified-resolved-entry')
1153+
1154+
expect(
1155+
registry.exportCommittedProvenanceForValue({
1156+
__simLargeValueRef: true,
1157+
version: 1,
1158+
id: 'lv_ABCDEFGHIJKL',
1159+
kind: 'object',
1160+
size: 1024,
1161+
})
10941162
).toEqual({ version: 1, complete: false, entries: [] })
10951163
})
10961164

@@ -1110,8 +1178,8 @@ describe('ResolvedSecretTraceRegistry', () => {
11101178

11111179
expect(provenance).toEqual({
11121180
version: 1,
1113-
complete: false,
1114-
entries: [],
1181+
complete: true,
1182+
entries: [{ encryptedValue: 'ciphertext' }],
11151183
})
11161184
expect(descriptorSnapshotCalls).toBe(0)
11171185
})

0 commit comments

Comments
 (0)