From f9351176bed572fad8606d33bbd8f15c1034b7e6 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Tue, 13 Oct 2026 09:46:30 +0800 Subject: [PATCH] Mask agent session transcripts and supervisor grade evidence An agent's resumable session file is its own: a tool call that printed its environment, or a goal that carried a workflow secret, puts the value there verbatim. The live stdout path was always masked, but the session went around the run's redactor into result_jsonb's sessionTranscript, the mid-run checkpoint artifacts and the resume payload a retry is handed, so an echoed model key, broker or MCP token, or workflow secret was kept at rest and restored into the next attempt. The capture and the checkpointer now mask the session with the run's own redactor (the same needles as its events, F1's workflow secrets included) before it is kept, on the launch path and on the re-attach that finishes a run whose worker went away. The mask is applied inside the JSON values that hold a secret, not over the raw line: a value the CLI escaped is not in the file as its own characters, and a raw match that ends inside an escape leaves a line the CLI cannot resume from. A line with no secret keeps its exact bytes; a lone surrogate escape, which Node writes when it truncates inside an emoji, no longer throws a complete line out of that path; and a line that is not JSON (a torn tail) is masked by plain replacement of each secret and of the forms a JSON writer escapes it to. A checkpoint's byte watermark still measures the file, not the masked copy. Only the kept copies are masked, not the live conversation. A warm revise round continues the session in this process, and its goal only says to continue, so a secret the original goal carried is in that conversation and nowhere else. The round is handed the session exactly as the agent wrote it, held in memory beside the masked copy the result keeps and never written to task_jsonb. Respawns and checkpoint resumes restore the masked copy and re-send the sealed goal. The per-run spool held out.log and err.log, the agent's output before redaction, at the worker's umask: a 0755 directory around 0644 logs, beside the 0700 config home. The spool is now created 0700 at launch, and the copiers write their logs and FIFOs owner-only under a umask of their own, so the agent keeps the worker's umask for its own files. The supervisor's nine grade lanes handed the grader no secrets, so an operator-floor argv or a model-authored acceptance that resolved one kept it in the evidence artifact a Viewer downloads, its tail on the decision tape and the baseline's evidence. Each lane now passes the secrets the unit's sealed task embeds and the node scope secrets its spec carries; the captured and baseline requests carry them too. Every grade lane masks its result on the way out, so clone-failed, grade-error and patch-apply-failed details are masked like the evidence, and the supervisor's own grade-error details are masked with the node scope before the tape keeps them. Rows written before this change keep what they hold; a retry restoring one is masked again when its own session is captured. --- .github/workflows/sandbox-isolation.yml | 18 +- .../Services/Agents/AgentRunExecutor.cs | 61 +++- .../ArtifactSessionTranscriptCheckpointer.cs | 20 +- .../IAgentSessionTranscriptCheckpointer.cs | 6 +- .../Runners/LocalProcessRunner.Durable.cs | 27 +- .../LocalProcessRunner.NativeLaunch.cs | 1 + .../Services/Agents/SecretRedactor.cs | 3 + .../Agents/SessionTranscriptRedaction.cs | 178 ++++++++++++ .../SupervisorAcceptanceGradeRequest.cs | 6 + .../Supervisor/SupervisorAcceptanceGrader.cs | 34 ++- .../SupervisorTurnService.Rehydrate.cs | 79 +++-- .../Supervisor/SupervisorTurnService.cs | 6 +- .../Workflows/Nodes/Builtin/AgentCodeNode.cs | 4 +- .../Agents/MaskedSessionResumeE2ETests.cs | 108 +++++++ .../AgentSessionTranscriptMaskingFlowTests.cs | 194 +++++++++++++ .../Infrastructure/SessionKeepingCli.cs | 270 +++++++++++++++++ .../Workflows/AgentRunReviseLoopFlowTests.cs | 123 +++++++- ...SupervisorGradeEvidenceMaskingFlowTests.cs | 274 ++++++++++++++++++ .../GoalChannelE2ETests.cs | 44 +++ .../NonRootWorkerE2ETests.cs | 10 + .../SpoolPermissionsE2ETests.cs | 78 +++++ ...AgentSessionTranscriptCheckpointerTests.cs | 24 +- .../Agents/SessionTranscriptRedactionTests.cs | 173 +++++++++++ .../SupervisorBranchlessStopGradeTests.cs | 56 +++- .../AcceptanceGradePostureInventoryTests.cs | 20 +- .../AcceptanceEvidenceSecretMaskingTests.cs | 38 +++ .../CapturedDeliverableGradeTests.cs | 46 ++- .../Workflows/AgentRunExecutorReviseTests.cs | 14 + .../AgentRunExecutorSessionTranscriptTests.cs | 43 ++- .../LocalProcessDurableRunnerTests.cs | 49 ++++ 30 files changed, 1918 insertions(+), 89 deletions(-) create mode 100644 backend/src/CodeSpace.Core/Services/Agents/SessionTranscriptRedaction.cs create mode 100644 backend/tests/CodeSpace.E2ETests/Agents/MaskedSessionResumeE2ETests.cs create mode 100644 backend/tests/CodeSpace.IntegrationTests/Agents/AgentSessionTranscriptMaskingFlowTests.cs create mode 100644 backend/tests/CodeSpace.IntegrationTests/Agents/Infrastructure/SessionKeepingCli.cs create mode 100644 backend/tests/CodeSpace.IntegrationTests/Workflows/SupervisorGradeEvidenceMaskingFlowTests.cs create mode 100644 backend/tests/CodeSpace.SandboxTests/SpoolPermissionsE2ETests.cs create mode 100644 backend/tests/CodeSpace.UnitTests/Agents/SessionTranscriptRedactionTests.cs diff --git a/.github/workflows/sandbox-isolation.yml b/.github/workflows/sandbox-isolation.yml index 4a2887d4c..50b70d71d 100644 --- a/.github/workflows/sandbox-isolation.yml +++ b/.github/workflows/sandbox-isolation.yml @@ -234,8 +234,8 @@ jobs: executed=$(grep -oE 'executed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+') passed=$(grep -oE 'passed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+') echo "executed=${executed:-0} passed=${passed:-0}" - if [ "${executed:-0}" -lt 111 ]; then - echo "::error::Expected >=111 sandbox isolation tests to run (bwrap/prlimit confinement + cap-drop + cgroup-namespace re-root + egress-allowlist filter + cgroup resource cap + durable-launch cgroup wiring + argv/envp per-string kernel ceiling + a prompt past it riding stdin + a read-only workspace mount + a network-off run reaching its broker through the relay and nothing else + an allowlist run relayed to its broker + a read-only reviewer reading its diff with the real CLIs + a bwrap probe that runs the launch argv + the MCP helper bound file by file behind a read-only socket dir + a CLI reaching its broker socket through the relay + a severed child reaching its broker over the lease socket across a worker restart + a pre-relay namespaced run re-bound at its gateway and torn down with its seal + an allowlist run's veth guarded both ways, a flow the worker opened before the run included + forwarding a root worker may not write named before an allowlist is planned + an allowlist run's port 53 open only at the resolvers of the resolv.conf its namespace reads, and still to a resolver address the worker's own NAT rewrites before its forward and its input hooks + a target repository's own CLI config kept out of the run with the real CLIs, every repository's memory read in a multi-repo workspace included + a repository whose memory links outside the workspace left out of a real Claude run, against the CLI following the link once that repository is added + nested memory loaded in place by a real Claude run, single- and multi-repo, and none of it past the in-place budget, where a read below a directory attaches only that directory's pointer + a path-scoped rule's pointer reaching a real Claude run only after a read its glob covers, single- and multi-repo, carrying none of the rule's text, against the rule copied into the config home as its control + pointers attaching on exactly the reads the unpinned CLI attached the repository's own rules on, single- and multi-repo + an Explore subagent pointed at nested memory loaded in place, which the CLI keeps out of its context + a multi-repo Codex run starting at a workspace root that is no git repository and loading no config from it + every repository's AGENTS.md reaching a real multi-repo Codex run once, after the operating contract, an override before the doc beside it and none that links outside the workspace + a repo-less Codex run starting in a scratch directory that is no git repository + a goal handed to the real Claude CLI as text it cannot act on, fresh and resumed, against the text channel as its control + a goal naming the real Codex CLI's bundled skills reaching its model with none of them loaded, fresh and resumed, against the CLI injecting every one as its control + a real Codex output reviewer loading none of the skills its reviewed checkout holds, against the CLI listing and injecting each one as its control + a repository clean filter the capture runs with its egress severed + a real Codex agent that tampers its clone's .git while the platform publishes the branch from a clean repo, its Claude counterpart running in the non-root lane), but only ${executed:-0} did — the Category=Sandbox filter matched too few (trait regression?). If a case was deliberately removed, lower this number in the same PR." + if [ "${executed:-0}" -lt 113 ]; then + echo "::error::Expected >=113 sandbox isolation tests to run (bwrap/prlimit confinement + cap-drop + cgroup-namespace re-root + egress-allowlist filter + cgroup resource cap + durable-launch cgroup wiring + argv/envp per-string kernel ceiling + a prompt past it riding stdin + a read-only workspace mount + a network-off run reaching its broker through the relay and nothing else + an allowlist run relayed to its broker + a read-only reviewer reading its diff with the real CLIs + a bwrap probe that runs the launch argv + the MCP helper bound file by file behind a read-only socket dir + a CLI reaching its broker socket through the relay + a severed child reaching its broker over the lease socket across a worker restart + a pre-relay namespaced run re-bound at its gateway and torn down with its seal + an allowlist run's veth guarded both ways, a flow the worker opened before the run included + forwarding a root worker may not write named before an allowlist is planned + an allowlist run's port 53 open only at the resolvers of the resolv.conf its namespace reads, and still to a resolver address the worker's own NAT rewrites before its forward and its input hooks + a target repository's own CLI config kept out of the run with the real CLIs, every repository's memory read in a multi-repo workspace included + a repository whose memory links outside the workspace left out of a real Claude run, against the CLI following the link once that repository is added + nested memory loaded in place by a real Claude run, single- and multi-repo, and none of it past the in-place budget, where a read below a directory attaches only that directory's pointer + a path-scoped rule's pointer reaching a real Claude run only after a read its glob covers, single- and multi-repo, carrying none of the rule's text, against the rule copied into the config home as its control + pointers attaching on exactly the reads the unpinned CLI attached the repository's own rules on, single- and multi-repo + an Explore subagent pointed at nested memory loaded in place, which the CLI keeps out of its context + a multi-repo Codex run starting at a workspace root that is no git repository and loading no config from it + every repository's AGENTS.md reaching a real multi-repo Codex run once, after the operating contract, an override before the doc beside it and none that links outside the workspace + a repo-less Codex run starting in a scratch directory that is no git repository + a goal handed to the real Claude CLI as text it cannot act on, fresh and resumed, against the text channel as its control + a goal naming the real Codex CLI's bundled skills reaching its model with none of them loaded, fresh and resumed, against the CLI injecting every one as its control + a real Codex output reviewer loading none of the skills its reviewed checkout holds, against the CLI listing and injecting each one as its control + a repository clean filter the capture runs with its egress severed + a real Codex agent that tampers its clone's .git while the platform publishes the branch from a clean repo, its Claude counterpart running in the non-root lane + a confined run's owner-only spool + a real Claude session masked as the executor keeps it, still resumed), but only ${executed:-0} did — the Category=Sandbox filter matched too few (trait regression?). If a case was deliberately removed, lower this number in the same PR." exit 1 fi @@ -279,12 +279,16 @@ jobs: # The goal-channel E2E is armed by the same CLI pins and returns early the same way; require each arm's marker, # confined, so the positive control it checks against is this lane's posture and not an unconfined one. - for arm in ('mentions', 'resume', 'slash /fix', 'slash /security-review'): + for arm in ('mentions', 'resume', 'slash /fix', 'slash /security-review', 'masked-resume'): assert f'[goal-channel-e2e] ran {arm} claude-code Confined uid=0 confined=True' in text, f'goal-channel E2E arm "{arm}" did not run confined — check CODESPACE_REQUIRE_REVIEW_CLIS and the CLI install step' - for method, rows in (('A_claude_goal_naming_secrets_reaches_the_model_verbatim_and_reads_none_of_them', 1), ('A_continued_claude_session_takes_its_prompt_the_same_way', 1), ('A_claude_goal_that_opens_with_a_slash_word_reaches_the_model_as_text', 2)): + for method, rows in (('A_claude_goal_naming_secrets_reaches_the_model_verbatim_and_reads_none_of_them', 1), ('A_continued_claude_session_takes_its_prompt_the_same_way', 1), ('A_claude_goal_that_opens_with_a_slash_word_reaches_the_model_as_text', 2), ('A_claude_session_masked_as_the_executor_keeps_it_still_resumes', 1)): cases = [r for r in results if 'GoalChannelE2ETests.' + method in r.get('testName', '')] assert len(cases) == rows and all(r.get('outcome') == 'Passed' for r in cases), f'{method}: all {rows} case(s) must pass' - print('All 4 goal-channel E2E arms ran and passed.') + print('All 5 goal-channel E2E arms ran and passed.') + + # The spool arm asserts owner-only modes under REAL bubblewrap; require its marker so an early return cannot pass. + assert '[spool-perms-e2e] ran uid=0 confined=True' in text, 'spool-permissions E2E did not run confined — check that bubblewrap is required in this lane' + print('The owner-only spool arm ran confined.') # The Codex arms of the same E2E: a goal naming every skill the CLI bundles reaches a run that loads none of them, # against a control that loads and injects every one; and an output reviewer loads none of the skills its reviewed @@ -428,9 +432,9 @@ jobs: root = ET.parse(path).getroot() counters = root.find('.//{*}Counters') executed, passed = int(counters.get('executed')), int(counters.get('passed')) - assert executed >= 19 and passed == executed, f'expected all 19 non-root arms to run and pass, got executed={executed} passed={passed}' + assert executed >= 20 and passed == executed, f'expected all 20 non-root arms to run and pass, got executed={executed} passed={passed}' text = open(path, encoding='utf-8').read() - for marker in ('[non-root-e2e] ran admission uid=1654', '[sealed-egress-e2e] ran non-root durable uid=1654', '[sealed-egress-e2e] ran non-root non-durable uid=1654', '[sealed-egress-e2e] ran non-root restart uid=1654', '[sealed-egress-e2e] ran non-root relay-refused uid=1654', '[sealed-egress-e2e] ran non-root relay-ipv6', '[broker-socket-e2e] ran non-root socket-channel uid=1654', '[review-diff-e2e] ran non-root network-off-relayed claude-code uid=1654', '[review-diff-e2e] ran non-root network-off-relayed codex-cli uid=1654', '[durable-egress-e2e] ran non-root allowlist-severed uid=1654', '[repo-config-e2e] ran non-root claude-code single-repo Standard uid=1654', '[repo-config-e2e] ran non-root claude-code own-stop-hook sealed-egress Standard uid=1654 confined=True', '[repo-config-e2e] ran non-root nested-in-place claude-code single-repo Standard uid=1654', '[repo-config-e2e] ran non-root scoped-rule-pointer claude-code single-repo Standard uid=1654', '[goal-channel-e2e] ran non-root mentions claude-code Standard uid=1654 confined=True', '[goal-channel-e2e] ran non-root resume claude-code Standard uid=1654 confined=True', '[goal-channel-e2e] ran non-root slash /fix claude-code Standard uid=1654 confined=True', '[goal-channel-e2e] ran non-root slash /security-review claude-code Standard uid=1654 confined=True', '[publish-isolation-e2e] ran non-root claude-code uid=1654 confined=True'): + for marker in ('[non-root-e2e] ran admission uid=1654', '[sealed-egress-e2e] ran non-root durable uid=1654', '[sealed-egress-e2e] ran non-root non-durable uid=1654', '[sealed-egress-e2e] ran non-root restart uid=1654', '[sealed-egress-e2e] ran non-root relay-refused uid=1654', '[sealed-egress-e2e] ran non-root relay-ipv6', '[broker-socket-e2e] ran non-root socket-channel uid=1654', '[review-diff-e2e] ran non-root network-off-relayed claude-code uid=1654', '[review-diff-e2e] ran non-root network-off-relayed codex-cli uid=1654', '[durable-egress-e2e] ran non-root allowlist-severed uid=1654', '[repo-config-e2e] ran non-root claude-code single-repo Standard uid=1654', '[repo-config-e2e] ran non-root claude-code own-stop-hook sealed-egress Standard uid=1654 confined=True', '[repo-config-e2e] ran non-root nested-in-place claude-code single-repo Standard uid=1654', '[repo-config-e2e] ran non-root scoped-rule-pointer claude-code single-repo Standard uid=1654', '[goal-channel-e2e] ran non-root mentions claude-code Standard uid=1654 confined=True', '[goal-channel-e2e] ran non-root resume claude-code Standard uid=1654 confined=True', '[goal-channel-e2e] ran non-root slash /fix claude-code Standard uid=1654 confined=True', '[goal-channel-e2e] ran non-root slash /security-review claude-code Standard uid=1654 confined=True', '[publish-isolation-e2e] ran non-root claude-code uid=1654 confined=True', '[spool-perms-e2e] ran non-root uid=1654 confined=True'): assert marker in text, f'non-root arm marker "{marker}" is missing — the arm returned early or did not run as the worker uid' print(f'All {executed} non-root arms ran as uid 1654 and passed.') PY diff --git a/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs b/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs index 9bd59723f..92a7004ef 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs @@ -203,6 +203,13 @@ public sealed class AgentRunExecutor : IAgentRunExecutor, IScopedDependency private Task _sessionCheckpoint = Task.FromResult(null); private DateTimeOffset? _sessionCheckpointAttemptedAt; private readonly SessionCheckpointWatermark _sessionCheckpointWatermark = new(); + // The session the last capture read, EXACTLY as the agent wrote it, for the one reader that may have it: a WARM + // revise round, which continues that conversation in this process and whose goal only says "continue" — so a secret + // the original goal carried is in the conversation and nowhere else. The result carries the masked copy, because + // everything that reads a result keeps it (result_jsonb, its offload, the resume payload); this never leaves the + // process. Paired with the run it was read for and the masked copy it stands behind, so no other run, and no round + // whose own capture kept nothing, is ever handed it. + private CapturedConversation? _capturedConversation; // The launch this pass last got onto the row — cleared the moment another launch begins, set once its handle write // lands. A cancel's kill of the recorded handle proves the clone empty only while the two agree (see // StopCancelledAttemptAsync): mid-launch, the row still names the previous round's finished process while a new @@ -579,7 +586,7 @@ public async Task ExecuteAsync(Guid agentRunId, CancellationToken cancellationTo // leaves this promise Intended; recovery marks it INDETERMINATE — visible, never a silent Succeeded. await _captureIntents.OpenAsync(agentRunId, run.TeamId, run.WorkflowRunId, claimedEpoch, CaptureExpectationsOf(effectiveTask), cancellationToken).ConfigureAwait(false); - result = await VerifyProducedWorkAsync(new(owner, run, harness, contract, workspace) { AcceptanceContext = localAcceptance }, result, cancellationToken).ConfigureAwait(false); + result = await VerifyProducedWorkAsync(new(owner, run, harness, contract, workspace) { AcceptanceContext = localAcceptance, Redactor = redactor }, result, cancellationToken).ConfigureAwait(false); // S6: the bounded REVISE loop — when the objective oracle failed on something the agent can fix, or the // Improve-mode critic flagged the output, feed the failure detail back to the SAME agent (same workspace; @@ -612,7 +619,7 @@ public async Task ExecuteAsync(Guid agentRunId, CancellationToken cancellationTo await AppendReviseEventAsync(owner, reason, round, reviseBudget, cancellationToken).ConfigureAwait(false); - var reviseTask = BuildReviseTask(effectiveTask, result, reason) with { Model = dispatchedModel }; + var reviseTask = BuildReviseTask(effectiveTask, result, reason, conversation: ConversationBehind(agentRunId, result)) with { Model = dispatchedModel }; // D3: the round that just failed IS the evidence. When it says the model was the limit — an // over-claim, or a check that failed on real work, never a grader / environment / gateway fault — @@ -698,7 +705,7 @@ public async Task ExecuteAsync(Guid agentRunId, CancellationToken cancellationTo // Verify under the ORIGINAL goal: the composed REVISE goal is for the harness invocation only — the // output critic must judge goal-alignment against what the task actually asked for, not the feedback // wrapper (which quotes the failure and could bias or blind the reviewer). - result = await VerifyProducedWorkAsync(new(owner, run, harness, reviseTask with { Goal = contract.Goal }, workspace) { AcceptanceContext = localAcceptance }, result, cancellationToken).ConfigureAwait(false); + result = await VerifyProducedWorkAsync(new(owner, run, harness, reviseTask with { Goal = contract.Goal }, workspace) { AcceptanceContext = localAcceptance, Redactor = redactor }, result, cancellationToken).ConfigureAwait(false); priorReason = reason; } @@ -1125,7 +1132,7 @@ async Task PersistFrameAsync(SandboxOutputFrame output) sandbox = await capture.ObserveAsync((capturedHandle, token) => { var replayHandle = capturedHandle with { StdoutOffset = Math.Min(capturedHandle.StdoutOffset, native.ReplayStartOffset) }; - return context.Durable.AttachAsync(replayHandle, (frame, _) => PersistFrameAsync(frame), token, CheckpointHandleOffset(context.Owner, capturedHandle, new HarnessSinks(writer, native, CheckpointTickFor(context.Task, context.TeamId, context.Harness, context.Task.WorkspaceDirectory, facts)))); + return context.Durable.AttachAsync(replayHandle, (frame, _) => PersistFrameAsync(frame), token, CheckpointHandleOffset(context.Owner, capturedHandle, new HarnessSinks(writer, native, CheckpointTickFor(context.Task, new SessionCheckpointTick(context.TeamId, context.Harness, context.Task.WorkspaceDirectory, facts, redactor))))); }, cancellationToken).ConfigureAwait(false); } finally @@ -1152,7 +1159,7 @@ async Task PersistFrameAsync(SandboxOutputFrame output) // Capture the resumable session transcript here too — a run that completes via durable re-attach (worker restart // mid-run) is exactly the durability case continuity serves; the config home still lives under the handle's spool. - return await CaptureSessionTranscriptAsync(new SessionCapture(context.RunId, context.Task, context.Harness, capture.Handle), result, cancellationToken).ConfigureAwait(false); + return await CaptureSessionTranscriptAsync(new SessionCapture(context.RunId, context.Task, context.Harness, capture.Handle, redactor), result, cancellationToken).ConfigureAwait(false); } /// @@ -1697,7 +1704,10 @@ internal async Task CaptureSessionTranscriptAsync(SessionCapture } using var reader = new StreamReader(stream); - var transcript = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false); + var conversation = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false); + var transcript = SessionTranscriptRedaction.Redact(conversation, capture.Redactor); + + _capturedConversation = new CapturedConversation(capture.RunId, transcript, conversation); return BackfillTranscriptModel(result, capture.Harness) with { SessionTranscript = transcript }; @@ -1717,8 +1727,15 @@ AgentRunResult BackfillTranscriptModel(AgentRunResult captured, IAgentHarness h) } } - /// The coordinates one session capture needs, as a record rather than a parameter list: which run owns it, the task whose workspace directory keys the harness's session path, the harness that locates the file, and the durable handle whose spool holds the config home (null on the live path, which re-reads the handle recorded at launch). - internal sealed record SessionCapture(Guid RunId, AgentTask Task, IAgentHarness Harness, SandboxHandle? Handle); + /// A captured session as it is kept (masked) and as the agent wrote it (exact), with the run it was read for — see . + private sealed record CapturedConversation(Guid RunId, string Kept, string Exact); + + /// The session behind 's kept transcript exactly as the agent wrote it, when this executor captured it for this run; null otherwise. + private string? ConversationBehind(Guid runId, AgentRunResult result) => + _capturedConversation is { } captured && captured.RunId == runId && string.Equals(captured.Kept, result.SessionTranscript, StringComparison.Ordinal) ? captured.Exact : null; + + /// The coordinates one session capture needs, as a record rather than a parameter list: which run owns it, the task whose workspace directory keys the harness's session path, the harness that locates the file, the durable handle whose spool holds the config home (null on the live path, which re-reads the handle recorded at launch), and the run's redactor, which masks the file before it is kept. + internal sealed record SessionCapture(Guid RunId, AgentTask Task, IAgentHarness Harness, SandboxHandle? Handle, SecretRedactor Redactor); /// /// P3 (3.2c): resolve a REFERENCED restored transcript (the producer stamped RestoredTranscriptArtifactId to @@ -2348,7 +2365,7 @@ private async Task VerifyProducedWorkAsync(ProducedWorkContext c var runId = owner.RunId; var claimedEpoch = owner.Epoch; await _runs.AssertOwnershipAsync(owner, cancellationToken).ConfigureAwait(false); - result = await CaptureSessionTranscriptAsync(new SessionCapture(runId, task, harness, Handle: null), result, cancellationToken).ConfigureAwait(false); + result = await CaptureSessionTranscriptAsync(new SessionCapture(runId, task, harness, Handle: null, context.Redactor), result, cancellationToken).ConfigureAwait(false); result = await EnrichWithWorkspaceChangesAsync(new(owner, run.TeamId, task, workspace), result, cancellationToken).ConfigureAwait(false); @@ -2794,8 +2811,13 @@ prior is null ? current /// home, so the instruction is just the delta. COLD otherwise: a fresh conversation in the same workspace, so the /// instruction restates the original goal too. Any ancestor continue-resume riding the task is superseded by THIS /// run's own session; a stale offloaded-transcript ref is dropped with it. + /// + /// A warm round restores — the session behind the result's kept transcript as + /// the agent wrote it — when the caller has it: the kept copy is masked for storage, and the warm goal does not + /// restate the original one, so a secret that goal carried is in the conversation and nowhere else. This task is the + /// round's invocation only; it is never the envelope written back to task_jsonb. /// - internal static AgentTask BuildReviseTask(AgentTask task, AgentRunResult result, string reason, bool mayResume = true) + internal static AgentTask BuildReviseTask(AgentTask task, AgentRunResult result, string reason, bool mayResume = true, string? conversation = null) { // mayResume is false when the warm round would not fit the launch pipe: the same repair goes on cold, and the // cold goal restates the contract no conversation now holds. @@ -2807,7 +2829,7 @@ internal static AgentTask BuildReviseTask(AgentTask task, AgentRunResult result, { Goal = ComposeReviseGoal(task.Goal, diagnosis, warm), ResumeFromSessionId = warm ? result.SessionId : null, - RestoredTranscript = warm ? result.SessionTranscript : null, + RestoredTranscript = warm ? conversation ?? result.SessionTranscript : null, RestoredTranscriptArtifactId = null, }; } @@ -4540,7 +4562,7 @@ async Task PersistAsync(string line, SandboxOutputFrame? output) SandboxResult sandbox; try { - sandbox = await RunSandboxAsync(context, PersistLineAsync, PersistFrameAsync, new HarnessSinks(writer, native, CheckpointTickFor(context.Task, context.TeamId, context.Harness, context.Spec.WorkingDirectory, facts)), cancellationToken).ConfigureAwait(false); + sandbox = await RunSandboxAsync(context, PersistLineAsync, PersistFrameAsync, new HarnessSinks(writer, native, CheckpointTickFor(context.Task, new SessionCheckpointTick(context.TeamId, context.Harness, context.Spec.WorkingDirectory, facts, context.Redactor))), cancellationToken).ConfigureAwait(false); } finally { @@ -5882,7 +5904,7 @@ private void StartSessionTranscriptCheckpoint(AgentRunOwnerToken owner, SandboxH return; } - var request = new Recovery.SessionTranscriptCheckpointRequest(tick.TeamId, owner, path, sessionId, _sessionCheckpointWatermark.Begin(path)); + var request = new Recovery.SessionTranscriptCheckpointRequest(tick.TeamId, owner, path, sessionId, _sessionCheckpointWatermark.Begin(path)) { Redactor = tick.Redactor }; _sessionCheckpoint = Task.Run(() => CheckpointInOwnScopeAsync(request), CancellationToken.None); } @@ -6121,6 +6143,9 @@ private async Task RejectUnclaimedAndNotifyAsync(AgentRun run, AgentRunResult re private sealed record ProducedWorkContext(AgentRunOwnerToken Owner, AgentRun Run, IAgentHarness Harness, AgentTask Task, IWorkspaceHandle? Workspace) { public LocalAcceptanceContext? AcceptanceContext { get; init; } + + /// The run's redactor — every secret the launch handed the agent — which masks what the chain keeps of the agent's own files (its session transcript) before it is stored. + public required SecretRedactor Redactor { get; init; } } internal sealed record AcceptanceInvocation(AgentRun Run, AgentTask Task, IWorkspaceHandle? Workspace, AgentRunOwnerToken? Owner = null, LocalAcceptanceContext? LocalContext = null); private sealed record WorkspaceCaptureContext(AgentRunOwnerToken Owner, Guid TeamId, AgentTask Task, IWorkspaceHandle? Workspace); @@ -6152,8 +6177,12 @@ private sealed record HarnessSinks(BufferedEventWriter Events, AgentNativeRecord /// checkpoint, so the lifecycle line carrying the id is usually behind the replay head and its fresh facts never /// see one. The launch's own last checkpoint then stands, which is the right answer — it is the newest /// conversation anyone can prove. + /// + /// is the run's own — the same secrets that mask its events — because a checkpoint + /// is the agent's own file, and an agent that echoed its key or a workflow secret into a tool result has it there + /// too. The checkpoint is a team artifact a retry restores; it is masked before it is stored. /// - private sealed record SessionCheckpointTick(Guid TeamId, IAgentHarness Harness, string? WorkingDirectory, AgentRunFacts Facts); + private sealed record SessionCheckpointTick(Guid TeamId, IAgentHarness Harness, string? WorkingDirectory, AgentRunFacts Facts, SecretRedactor Redactor); /// /// The checkpoint coordinates for a run whose envelope OPTED IN, else null — the one place the opt-in is read, so @@ -6161,8 +6190,8 @@ private sealed record SessionCheckpointTick(Guid TeamId, IAgentHarness Harness, /// attempt nobody can retry writes nothing, which is what keeps the artifact store free of a per-minute /// transcript copy for every benchmark cell, review child and supervisor unit that no retry could resume. /// - private static SessionCheckpointTick? CheckpointTickFor(AgentTask task, Guid teamId, IAgentHarness harness, string? workingDirectory, AgentRunFacts facts) => - task.CheckpointSessionTranscript ? new SessionCheckpointTick(teamId, harness, workingDirectory, facts) : null; + private static SessionCheckpointTick? CheckpointTickFor(AgentTask task, SessionCheckpointTick tick) => + task.CheckpointSessionTranscript ? tick : null; private sealed record HarnessRunContext { diff --git a/backend/src/CodeSpace.Core/Services/Agents/Recovery/Checkpoints/ArtifactSessionTranscriptCheckpointer.cs b/backend/src/CodeSpace.Core/Services/Agents/Recovery/Checkpoints/ArtifactSessionTranscriptCheckpointer.cs index a54ce4826..0a57a3fdf 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Recovery/Checkpoints/ArtifactSessionTranscriptCheckpointer.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Recovery/Checkpoints/ArtifactSessionTranscriptCheckpointer.cs @@ -21,6 +21,13 @@ namespace CodeSpace.Core.Services.Agents.Recovery.Checkpoints; /// Bounded by — the SAME cap the end-of-run capture /// reads, deliberately not a second knob. Over it the checkpoint skips exactly as that capture does. /// +/// Masked with the run's redactor () before it is stored, +/// as the end-of-run capture is: the agent can write its key or a workflow secret into its own session, and a checkpoint +/// is a team artifact a retry restores. The mask keeps every line valid JSON (), +/// so the restored conversation still resumes. The checkpoint's stays +/// the length of the FILE's complete prefix, not of the masked copy: it is the growth watermark the next tick measures +/// the file against. +/// /// STATELESS, and resolved in a DI scope of the caller's own. The executor dispatches this OFF its drain tick, /// and the tick is using the executor's scoped DbContext every 250 ms for its event flush and spool-offset /// write — so a checkpointer sharing that context would put two operations on one EF context concurrently. EF refuses @@ -117,7 +124,7 @@ private static bool IsDue(long? previousBytes, long length) if (request.PreviousBytes is { } previous && complete <= previous) return null; // the growth was entirely inside the torn tail - var write = await _artifacts.PutDeclaredAsync(new ArtifactRetentionWriteRequest(request.TeamId, buffer.AsMemory(0, complete), TranscriptContentType, ArtifactRetentionClass.SessionTranscriptCheckpoint, CheckpointHolderKind, runId), cancellationToken).ConfigureAwait(false); + var write = await _artifacts.PutDeclaredAsync(new ArtifactRetentionWriteRequest(request.TeamId, Masked(buffer.AsMemory(0, complete), request.Redactor), TranscriptContentType, ArtifactRetentionClass.SessionTranscriptCheckpoint, CheckpointHolderKind, runId), cancellationToken).ConfigureAwait(false); var checkpoint = new SessionTranscriptCheckpoint(write.ArtifactId, _clock.GetUtcNow(), complete); if (!await _runs.StampSessionTranscriptCheckpointAsync(request.Owner, checkpoint, request.SessionId, cancellationToken).ConfigureAwait(false)) @@ -132,6 +139,17 @@ private static bool IsDue(long? previousBytes, long length) return checkpoint; } + /// The complete prefix as it may be stored: masked with the run's secrets, and the read bytes themselves when there was nothing to mask. + private static ReadOnlyMemory Masked(ReadOnlyMemory prefix, SecretRedactor redactor) + { + if (redactor.IsEmpty) return prefix; + + var text = System.Text.Encoding.UTF8.GetString(prefix.Span); + var masked = SessionTranscriptRedaction.Redact(text, redactor); + + return ReferenceEquals(masked, text) ? prefix : System.Text.Encoding.UTF8.GetBytes(masked); + } + /// /// The length of the COMPLETE prefix: everything up to and including the last newline. Zero when the file holds /// no newline at all. The CLIs append whole JSON lines, so a tail after the last newline is a line still being diff --git a/backend/src/CodeSpace.Core/Services/Agents/Recovery/IAgentSessionTranscriptCheckpointer.cs b/backend/src/CodeSpace.Core/Services/Agents/Recovery/IAgentSessionTranscriptCheckpointer.cs index ad765d9ad..1d07a6239 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Recovery/IAgentSessionTranscriptCheckpointer.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Recovery/IAgentSessionTranscriptCheckpointer.cs @@ -56,4 +56,8 @@ public interface IAgentSessionTranscriptCheckpointer : IScopedDependency /// The absolute path of the live session transcript, already clamped within the run's config home by the caller (AgentRunExecutor.ResolveSessionTranscriptPath — the agent can write there, so the path is untrusted until that walk has run). /// The harness-native session id naming that transcript. Stamped with the checkpoint because a checkpoint nothing can ADDRESS is not resumable — the retry hands this to the CLI as its --resume id. /// The complete-prefix length of the last checkpoint taken of THIS path, or null when there is none. The growth watermark: bytes that have not changed carry no new conversation, so re-storing them buys nothing. The caller keys it on the path (a revise round opens a new config home whose transcript legitimately starts smaller) and passes null when the path moved. -public sealed record SessionTranscriptCheckpointRequest(Guid TeamId, AgentRunOwnerToken Owner, string TranscriptPath, string? SessionId, long? PreviousBytes); +public sealed record SessionTranscriptCheckpointRequest(Guid TeamId, AgentRunOwnerToken Owner, string TranscriptPath, string? SessionId, long? PreviousBytes) +{ + /// The run's redactor. The transcript is the agent's own file, so whatever the agent echoed into it — its model key, a broker or MCP token, a workflow secret its task embeds — is there too; the checkpoint is masked with this before it is stored. Required, so no caller can store one unmasked by leaving it out. + public required SecretRedactor Redactor { get; init; } +} diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Durable.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Durable.cs index 6cbf28d46..78572f7f5 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Durable.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.Durable.cs @@ -67,19 +67,21 @@ public sealed partial class LocalProcessRunner public static string ConfigHomePath(string spoolDirectory) => Path.Combine(spoolDirectory, AgentConfigHomeDir); /// - /// Create the per-run config-home restricted to the owner (0700) — the same treatment the non-durable command path - /// already gives its own config home. It holds the MCP declaration, which carries the run's live capability token, - /// so leaving it at whatever the umask happened to be made that token readable to every other local user on the - /// host. A no-op without a config home, and on Windows where unix modes don't apply. + /// Create a per-run directory restricted to the owner (0700), or narrow it to that when it already exists. Two + /// callers, both holding a secret at rest. The SPOOL holds out.log/err.log, the agent's output BEFORE + /// redaction — an echoed key, a broker token, a workflow secret in a tool result. The CONFIG HOME (the same treatment + /// the non-durable command path already gives its own) holds the MCP declaration, which carries the run's live + /// capability token. Left at whatever the umask happened to be, either was readable to every other local user on the + /// host. A no-op without a directory, and on Windows where unix modes don't apply. /// - private static void CreateOwnerOnlyConfigHome(string? configHome) + private static void CreateOwnerOnlyDirectory(string? directory) { - if (configHome is null) return; + if (directory is null) return; - Directory.CreateDirectory(configHome); + Directory.CreateDirectory(directory); if (!OperatingSystem.IsWindows()) - File.SetUnixFileMode(configHome, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + File.SetUnixFileMode(directory, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); } /// @@ -171,8 +173,13 @@ internal static bool PidAnswerableHere(SandboxHandle handle) => /// /bin/sh on macOS and RHEL), so the block size is chosen from $BASH_VERSION and the enforced /// ceiling is the SAME byte count on every host. prlimit cannot do this job at all: it wraps only the agent chain /// inside "$@", never this supervisor, and RLIMIT_FSIZE does not apply to pipe writes. + /// + /// The spool files a copier writes are the agent's output BEFORE redaction, so each copier sets + /// umask 077 and creates them owner-only (0600), and the FIFOs feeding them are made 0600 too. The umask is + /// the copier's own: each runs as a background subshell, so the agent in "$@" keeps the worker's umask for + /// every file it writes. /// - internal const string SupervisorScript = "pid_path=\"$CSP_PID\"; out_path=\"$CSP_OUT\"; err_path=\"$CSP_ERR\"; exit_path=\"$CSP_EXIT\"; copy_status_path=\"$CSP_LOG_COPY_STATUS\"; max_bytes=\"${CSP_MAX_BYTES:-0}\"; in_path=\"${CSP_IN:-/dev/null}\"; unset CSP_PID CSP_OUT CSP_ERR CSP_EXIT CSP_LOG_COPY_STATUS CSP_MAX_BYTES CSP_IN; spool_block=512; [ -n \"$BASH_VERSION\" ] && spool_block=1024; copy_spool() { if [ \"$max_bytes\" -le 0 ]; then cat >\"$1\"; return 0; fi; ulimit -c 0; ulimit -f $((max_bytes/spool_block)) 2>/dev/null; { cat >\"$1\"; } 2>/dev/null; copy_status=$?; if [ \"$copy_status\" -eq 153 ]; then : >\"$1.truncated\"; copy_status=0; fi; cat >/dev/null; return \"$copy_status\"; }; printf '%s' \"$$\" >\"$pid_path\"; out_pipe=\"${out_path}.pipe\"; err_pipe=\"${err_path}.pipe\"; rm -f \"$out_pipe\" \"$err_pipe\"; mkfifo \"$out_pipe\" \"$err_pipe\" || exit 125; copy_spool \"$out_path\" <\"$out_pipe\" & out_cat=$!; copy_spool \"$err_path\" <\"$err_pipe\" & err_cat=$!; \"$@\" >\"$out_pipe\" 2>\"$err_pipe\" <\"$in_path\"; code=$?; wait \"$out_cat\"; out_status=$?; wait \"$err_cat\"; err_status=$?; rm -f \"$out_pipe\" \"$err_pipe\"; printf '%s' \"$code\" >\"$exit_path\"; printf '%s:%s' \"$out_status\" \"$err_status\" >\"$copy_status_path\""; + internal const string SupervisorScript = "pid_path=\"$CSP_PID\"; out_path=\"$CSP_OUT\"; err_path=\"$CSP_ERR\"; exit_path=\"$CSP_EXIT\"; copy_status_path=\"$CSP_LOG_COPY_STATUS\"; max_bytes=\"${CSP_MAX_BYTES:-0}\"; in_path=\"${CSP_IN:-/dev/null}\"; unset CSP_PID CSP_OUT CSP_ERR CSP_EXIT CSP_LOG_COPY_STATUS CSP_MAX_BYTES CSP_IN; spool_block=512; [ -n \"$BASH_VERSION\" ] && spool_block=1024; copy_spool() { umask 077; if [ \"$max_bytes\" -le 0 ]; then cat >\"$1\"; return 0; fi; ulimit -c 0; ulimit -f $((max_bytes/spool_block)) 2>/dev/null; { cat >\"$1\"; } 2>/dev/null; copy_status=$?; if [ \"$copy_status\" -eq 153 ]; then : >\"$1.truncated\"; copy_status=0; fi; cat >/dev/null; return \"$copy_status\"; }; printf '%s' \"$$\" >\"$pid_path\"; out_pipe=\"${out_path}.pipe\"; err_pipe=\"${err_path}.pipe\"; rm -f \"$out_pipe\" \"$err_pipe\"; mkfifo -m 600 \"$out_pipe\" \"$err_pipe\" || exit 125; copy_spool \"$out_path\" <\"$out_pipe\" & out_cat=$!; copy_spool \"$err_path\" <\"$err_pipe\" & err_cat=$!; \"$@\" >\"$out_pipe\" 2>\"$err_pipe\" <\"$in_path\"; code=$?; wait \"$out_cat\"; out_status=$?; wait \"$err_cat\"; err_status=$?; rm -f \"$out_pipe\" \"$err_pipe\"; printf '%s' \"$code\" >\"$exit_path\"; printf '%s:%s' \"$out_status\" \"$err_status\" >\"$copy_status_path\""; /// /// The per-spool-file byte budget for this run's FIFO copiers: the documented @@ -989,7 +996,7 @@ internal static ProcessStartInfo BuildDurableStartInfo(SandboxSpec spec, string // sandbox's writable HOME + a writable bind, so compute + create it BEFORE building the command. Created // here (before launch) so bwrap's --bind source exists when the process starts. var configHome = spec.ConfigHomeEnvVars.Count > 0 ? Path.Combine(spoolDir, AgentConfigHomeDir) : null; - CreateOwnerOnlyConfigHome(configHome); + CreateOwnerOnlyDirectory(configHome); // Write the run-scoped MCP server declaration (0600 — it carries the run token) into the config-home BEFORE // launch so the harness reads it on start. No-op when the run has no tool fabric (spec.Mcp null) or no diff --git a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.NativeLaunch.cs b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.NativeLaunch.cs index 1fbd1d88f..2d6031239 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.NativeLaunch.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Sandbox/Runners/LocalProcessRunner.NativeLaunch.cs @@ -70,6 +70,7 @@ public async Task LaunchOrDiscoverAsync(SandboxLaunchRequest requ ValidateLaunchKey(request); var hash = NativeLaunchProtocol.SpecHash(spec); var spool = SpoolDirectoryFor(request.SpoolKey); + CreateOwnerOnlyDirectory(spool); // its out.log/err.log are the agent's output before redaction var directory = NativeLaunchFiles.DirectoryFor(spool); Directory.CreateDirectory(directory); if (!OperatingSystem.IsWindows()) File.SetUnixFileMode(directory, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); diff --git a/backend/src/CodeSpace.Core/Services/Agents/SecretRedactor.cs b/backend/src/CodeSpace.Core/Services/Agents/SecretRedactor.cs index 6a06c425a..c7118b69b 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/SecretRedactor.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/SecretRedactor.cs @@ -65,6 +65,9 @@ public string? Fingerprint /// public SecretRedactor With(IEnumerable additional) => new(_secrets.Concat(additional)); + /// This redactor plus every form gives each of its secrets — a new instance, for text that holds a secret written another way (escaped inside JSON); the receiver is unchanged. + internal SecretRedactor WithVariants(Func> variants) => With(_secrets.SelectMany(variants)); + public string Redact(string text) { if (string.IsNullOrEmpty(text) || IsEmpty) return text; diff --git a/backend/src/CodeSpace.Core/Services/Agents/SessionTranscriptRedaction.cs b/backend/src/CodeSpace.Core/Services/Agents/SessionTranscriptRedaction.cs new file mode 100644 index 000000000..f33fcaba5 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Agents/SessionTranscriptRedaction.cs @@ -0,0 +1,178 @@ +using System.Buffers; +using System.Globalization; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; + +namespace CodeSpace.Core.Services.Agents; + +/// +/// Masks a run's secrets in a CLI session transcript — the newline-delimited JSON a harness restores verbatim for +/// --resume — without breaking it. A secret is replaced INSIDE the JSON value that holds it (a string, a property +/// name, a number), so every line that parsed before still parses, the line count is unchanged, and a line that holds no +/// secret keeps its exact bytes. +/// +/// Plain text replacement over the line is not enough on either count. A value the CLI escaped — a secret with a +/// quote, a backslash or a control character in it — is not in the file as its own characters, so exact matching never +/// sees it; and a match that ends inside an escape sequence leaves a lone backslash that escapes whatever follows, which +/// the CLI then refuses to resume. A line that is not one JSON value (a torn tail, an interleaved log line) has no value +/// to mask inside and nothing to keep valid, so it gets plain replacement — of each secret and of the forms a JSON writer +/// escapes it to, since the tail of a value the CLI was writing is still escaped text. +/// +public static class SessionTranscriptRedaction +{ + /// Deep enough for any tool payload a CLI records. A deeper line falls back to the plain mask rather than failing. + private const int MaxDepth = 1000; + + private static readonly JsonReaderOptions ReaderOptions = new() { MaxDepth = MaxDepth }; + + /// The CLI's own escaping is not reproduced; any valid escaping is. Relaxed, so a rewritten line keeps its non-ASCII text readable rather than \uXXXX-encoded. + private static readonly JsonWriterOptions WriterOptions = new() { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, MaxDepth = MaxDepth }; + + /// with every secret of masked as ; the same instance when nothing was masked. + public static string Redact(string transcript, SecretRedactor redactor) + { + if (redactor.IsEmpty || string.IsNullOrEmpty(transcript)) return transcript; + + var unparsed = redactor.WithVariants(JsonEscapedForms); + var lines = transcript.Split('\n'); + var changed = false; + + for (var i = 0; i < lines.Length; i++) + { + var masked = RedactLine(lines[i], redactor, unparsed); + + changed |= !ReferenceEquals(masked, lines[i]); + lines[i] = masked; + } + + return changed ? string.Join('\n', lines) : transcript; + } + + /// + /// One line, masked. A line with no secret in its text and no escape in it cannot hold a secret in any value, so it + /// is returned as is without being parsed — the common case, and the reason a large transcript costs one scan. A + /// line that is not one JSON value is masked by : the secrets and their escaped forms. + /// + private static string RedactLine(string line, SecretRedactor redactor, SecretRedactor unparsed) + { + if (ReferenceEquals(redactor.Redact(line), line) && !line.Contains('\\')) return line; + + return RedactJsonValues(line, redactor) ?? unparsed.Redact(line); + } + + /// + /// The forms a JSON writer puts a secret in inside a string: with the short escapes (\n, \", \\) the + /// CLIs' writers use, and with an ASCII-only writer's \uXXXX ones. Every form that differs from the secret holds + /// a backslash, which is what lets a line without one skip the parse. + /// + private static IEnumerable JsonEscapedForms(string secret) => + [JsonEncodedText.Encode(secret, JavaScriptEncoder.UnsafeRelaxedJsonEscaping).ToString(), JsonEncodedText.Encode(secret, JavaScriptEncoder.Default).ToString()]; + + /// The line re-written token by token with its values masked; the line itself when no value held a secret; null when it is not one JSON value. + private static string? RedactJsonValues(string line, SecretRedactor redactor) + { + var reader = new Utf8JsonReader(Encoding.UTF8.GetBytes(line), ReaderOptions); + var output = new ArrayBufferWriter(); + var changed = false; + + try + { + using var writer = new Utf8JsonWriter(output, WriterOptions); + + while (reader.Read()) changed |= CopyToken(ref reader, writer, redactor); + } + catch (Exception notOneValue) when (notOneValue is JsonException or InvalidOperationException) + { + return null; + } + + return changed ? Encoding.UTF8.GetString(output.WrittenSpan) : line; + } + + /// Copy the reader's current token to , masked; true when masking changed it. + private static bool CopyToken(ref Utf8JsonReader reader, Utf8JsonWriter writer, SecretRedactor redactor) + { + switch (reader.TokenType) + { + case JsonTokenType.StartObject: writer.WriteStartObject(); return false; + case JsonTokenType.EndObject: writer.WriteEndObject(); return false; + case JsonTokenType.StartArray: writer.WriteStartArray(); return false; + case JsonTokenType.EndArray: writer.WriteEndArray(); return false; + case JsonTokenType.PropertyName: return WriteMasked(Text(ref reader), redactor, writer.WritePropertyName); + case JsonTokenType.String: return WriteMasked(Text(ref reader), redactor, writer.WriteStringValue); + case JsonTokenType.Number: return WriteNumber(reader.ValueSpan, redactor, writer); + case JsonTokenType.True: writer.WriteBooleanValue(true); return false; + case JsonTokenType.False: writer.WriteBooleanValue(false); return false; + case JsonTokenType.Null: writer.WriteNullValue(); return false; + default: return false; + } + } + + /// + /// The token's text, unescaped here rather than by , which refuses a lone + /// surrogate escape (\ud83d) — what Node's JSON.stringify writes when a truncated tool output split an + /// emoji. The line is valid JSON, and refusing it would leave the whole line to the plain mask. Written back, such a + /// half character becomes U+FFFD; only a line that held a secret is written back at all. + /// + private static string Text(ref Utf8JsonReader reader) + { + var raw = Encoding.UTF8.GetString(reader.ValueSpan); + + return reader.ValueIsEscaped ? Unescape(raw) : raw; + } + + /// A JSON string's raw text, unescaped. The reader has already refused any malformed escape, so every backslash here starts a valid one. + private static string Unescape(string raw) + { + var text = new StringBuilder(raw.Length); + + for (var i = 0; i < raw.Length; i++) + { + if (raw[i] != '\\') + { + text.Append(raw[i]); + continue; + } + + var escape = raw[++i]; + + if (escape == 'u') + { + text.Append((char)ushort.Parse(raw.AsSpan(i + 1, 4), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture)); + i += 4; + continue; + } + + text.Append(escape switch { 'b' => '\b', 'f' => '\f', 'n' => '\n', 'r' => '\r', 't' => '\t', _ => escape }); + } + + return text.ToString(); + } + + private static bool WriteMasked(string value, SecretRedactor redactor, Action write) + { + var masked = redactor.Redact(value); + + write(masked); + + return !ReferenceEquals(masked, value); + } + + /// A number is kept exactly as written unless a secret is in it; then it becomes the masked string, since *** is no number. + private static bool WriteNumber(ReadOnlySpan raw, SecretRedactor redactor, Utf8JsonWriter writer) + { + var text = Encoding.UTF8.GetString(raw); + var masked = redactor.Redact(text); + + if (ReferenceEquals(masked, text)) + { + writer.WriteRawValue(raw, skipInputValidation: true); + return false; + } + + writer.WriteStringValue(masked); + + return true; + } +} diff --git a/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorAcceptanceGradeRequest.cs b/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorAcceptanceGradeRequest.cs index 7074f53a8..1a200629c 100644 --- a/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorAcceptanceGradeRequest.cs +++ b/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorAcceptanceGradeRequest.cs @@ -32,6 +32,9 @@ public sealed record CapturedAcceptanceGradeRequest /// The producing run's sandbox posture the grade runs under. Required so no lane can forget it; null grades with network off under the Confined tier's ceilings. public required AcceptanceGradingPosture? Posture { get; init; } + + /// The exact workflow-secret values the producing run's task embeds (AgentTask.WorkflowSecrets). An authored acceptance argv can carry one, and the grade's evidence repeats the argv and the check's output, so everything the grade persists masks them. The check itself runs with the exact argv. Null masks nothing. + public IReadOnlyList? WorkflowSecrets { get; init; } } /// A delayed patch acceptance grade with the producer identity that authored the patch. @@ -65,6 +68,9 @@ public sealed record BaseAcceptanceGradeRequest /// The producing run's sandbox posture the grade runs under. Required so no lane can forget it; null grades with network off under the Confined tier's ceilings. public required AcceptanceGradingPosture? Posture { get; init; } + + /// The exact workflow-secret values the producing run's task embeds (AgentTask.WorkflowSecrets). An authored acceptance argv can carry one, and the grade's evidence repeats the argv and the check's output, so everything the grade persists masks them. The check itself runs with the exact argv. Null masks nothing. + public IReadOnlyList? WorkflowSecrets { get; init; } } /// A grade of a repo-less run's live workspace directory, under the posture of the run that wrote it. diff --git a/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorAcceptanceGrader.cs b/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorAcceptanceGrader.cs index 01310a88c..7761a9ac6 100644 --- a/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorAcceptanceGrader.cs +++ b/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorAcceptanceGrader.cs @@ -65,7 +65,10 @@ public Task GradeAsync(Guid repositoryId, Guid teamId, string br public Task GradeAsync(Guid repositoryId, Guid teamId, string branch, SupervisorAcceptanceSpec spec, int timeoutSeconds, OracleAnchor anchor, CancellationToken cancellationToken) => GradeAsync(new RepositoryAcceptanceGradeRequest { RepositoryId = repositoryId, TeamId = teamId, Branch = branch, Spec = spec, TimeoutSeconds = timeoutSeconds, Anchor = anchor, Posture = null }, cancellationToken); - public async Task GradeAsync(RepositoryAcceptanceGradeRequest request, CancellationToken cancellationToken) + public async Task GradeAsync(RepositoryAcceptanceGradeRequest request, CancellationToken cancellationToken) => + MaskedGrade(await GradeBranchAsync(request, cancellationToken).ConfigureAwait(false), request.WorkflowSecrets); + + private async Task GradeBranchAsync(RepositoryAcceptanceGradeRequest request, CancellationToken cancellationToken) { var (repositoryId, teamId, branch, spec, timeoutSeconds, anchor) = (request.RepositoryId, request.TeamId, request.Branch, request.Spec, request.TimeoutSeconds, request.Anchor); @@ -110,7 +113,10 @@ public async Task GradeAsync(RepositoryAcceptanceGradeRequest re public Task GradeDirectoryAsync(string directory, SupervisorAcceptanceSpec spec, Guid teamId, int timeoutSeconds, CancellationToken cancellationToken) => GradeDirectoryAsync(new DirectoryAcceptanceGradeRequest { Directory = directory, Spec = spec, TeamId = teamId, TimeoutSeconds = timeoutSeconds, Posture = null }, cancellationToken); - public async Task GradeDirectoryAsync(DirectoryAcceptanceGradeRequest request, CancellationToken cancellationToken) + public async Task GradeDirectoryAsync(DirectoryAcceptanceGradeRequest request, CancellationToken cancellationToken) => + MaskedGrade(await GradeLiveDirectoryAsync(request, cancellationToken).ConfigureAwait(false), request.WorkflowSecrets); + + private async Task GradeLiveDirectoryAsync(DirectoryAcceptanceGradeRequest request, CancellationToken cancellationToken) { if (!Directory.Exists(request.Directory)) return new BenchmarkGrade { Passed = false, Detail = "grade-error: the workspace directory no longer exists", Class = Messages.Agents.Benchmark.GradeFailureClass.Environment }; @@ -121,7 +127,10 @@ public async Task GradeDirectoryAsync(DirectoryAcceptanceGradeRe public Task GradeCapturedAsync(Guid agentRunId, Guid teamId, SupervisorAcceptanceSpec spec, int timeoutSeconds, CancellationToken cancellationToken) => GradeCapturedAsync(new CapturedAcceptanceGradeRequest { AgentRunId = agentRunId, TeamId = teamId, Spec = spec, TimeoutSeconds = timeoutSeconds, Posture = null }, cancellationToken); - public async Task GradeCapturedAsync(CapturedAcceptanceGradeRequest request, CancellationToken cancellationToken) + public async Task GradeCapturedAsync(CapturedAcceptanceGradeRequest request, CancellationToken cancellationToken) => + MaskedGrade(await GradeRebuiltWorldAsync(request, cancellationToken).ConfigureAwait(false), request.WorkflowSecrets); + + private async Task GradeRebuiltWorldAsync(CapturedAcceptanceGradeRequest request, CancellationToken cancellationToken) { var (agentRunId, teamId, spec, timeoutSeconds) = (request.AgentRunId, request.TeamId, request.Spec, request.TimeoutSeconds); var directory = Path.Combine(LocalGitWorkspaceProvider.WorkspacesRoot, "captured-" + Guid.NewGuid().ToString("N")); @@ -135,7 +144,7 @@ public async Task GradeCapturedAsync(CapturedAcceptanceGradeRequ return Failed(ISupervisorAcceptanceGrader.NoDeliverablesCaptured, GradeFailureClass.Genuine); } - return await GradeWorkspaceAsync(new WorkspaceGradeRequest(directory, spec, teamId, timeoutSeconds, request.Posture, request.ProducerModel), cancellationToken).ConfigureAwait(false); + return await GradeWorkspaceAsync(new WorkspaceGradeRequest(directory, spec, teamId, timeoutSeconds, request.Posture, request.ProducerModel, WorkflowSecrets: request.WorkflowSecrets), cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { @@ -239,7 +248,10 @@ public Task GradePatchAsync(Guid repositoryId, Guid teamId, stri public Task GradePatchAsync(Guid repositoryId, Guid teamId, string baseSha, string inlinePatch, Guid? patchArtifactId, SupervisorAcceptanceSpec spec, int timeoutSeconds, IReadOnlyList? oracleFloorPrograms, CancellationToken cancellationToken) => GradePatchAsync(new PatchAcceptanceGradeRequest { RepositoryId = repositoryId, TeamId = teamId, BaseSha = baseSha, InlinePatch = inlinePatch, PatchArtifactId = patchArtifactId, Spec = spec, TimeoutSeconds = timeoutSeconds, OracleFloorPrograms = oracleFloorPrograms, Posture = null }, cancellationToken); - public async Task GradePatchAsync(PatchAcceptanceGradeRequest request, CancellationToken cancellationToken) + public async Task GradePatchAsync(PatchAcceptanceGradeRequest request, CancellationToken cancellationToken) => + MaskedGrade(await GradeAppliedPatchAsync(request, cancellationToken).ConfigureAwait(false), request.WorkflowSecrets); + + private async Task GradeAppliedPatchAsync(PatchAcceptanceGradeRequest request, CancellationToken cancellationToken) { var (repositoryId, teamId, baseSha, inlinePatch, patchArtifactId, spec, timeoutSeconds, oracleFloorPrograms) = (request.RepositoryId, request.TeamId, request.BaseSha, request.InlinePatch, request.PatchArtifactId, request.Spec, request.TimeoutSeconds, request.OracleFloorPrograms); var directory = Path.Combine(LocalGitWorkspaceProvider.WorkspacesRoot, "grade-" + Guid.NewGuid().ToString("N")); @@ -302,7 +314,10 @@ public async Task GradePatchAsync(PatchAcceptanceGradeRequest re public Task GradeBaseAsync(Guid repositoryId, Guid teamId, string baseSha, SupervisorAcceptanceSpec spec, int timeoutSeconds, CancellationToken cancellationToken) => GradeBaseAsync(new BaseAcceptanceGradeRequest { RepositoryId = repositoryId, TeamId = teamId, BaseSha = baseSha, Spec = spec, TimeoutSeconds = timeoutSeconds, Posture = null }, cancellationToken); - public async Task GradeBaseAsync(BaseAcceptanceGradeRequest request, CancellationToken cancellationToken) + public async Task GradeBaseAsync(BaseAcceptanceGradeRequest request, CancellationToken cancellationToken) => + MaskedGrade(await GradeBaseTreeAsync(request, cancellationToken).ConfigureAwait(false), request.WorkflowSecrets); + + private async Task GradeBaseTreeAsync(BaseAcceptanceGradeRequest request, CancellationToken cancellationToken) { var (repositoryId, teamId, baseSha, spec, timeoutSeconds) = (request.RepositoryId, request.TeamId, request.BaseSha, request.Spec, request.TimeoutSeconds); var directory = Path.Combine(LocalGitWorkspaceProvider.WorkspacesRoot, "grade-base-" + Guid.NewGuid().ToString("N")); @@ -314,7 +329,7 @@ public async Task GradeBaseAsync(BaseAcceptanceGradeRequest requ await CloneAtBaseAsync(clone, baseSha, directory, cancellationToken).ConfigureAwait(false); - return await GradeWorkspaceAsync(new WorkspaceGradeRequest(directory, spec, teamId, timeoutSeconds, request.Posture), cancellationToken).ConfigureAwait(false); + return await GradeWorkspaceAsync(new WorkspaceGradeRequest(directory, spec, teamId, timeoutSeconds, request.Posture, WorkflowSecrets: request.WorkflowSecrets), cancellationToken).ConfigureAwait(false); } catch (WorkspaceException ex) { @@ -826,6 +841,11 @@ private async Task CaptureEvidenceAsync(BenchmarkGrade grade, Wo /// before it is clipped onto the tail and stored as a team artifact a Viewer can download, the oracle note and the /// detail (a failed setup's detail carries its stderr). The check ran with the exact argv; only what is kept is /// masked. No secrets: the grade unchanged. Internal for direct unit pinning. + /// + /// Applied twice, and idempotent: at the evidence capture, so the stored artifact is masked, and on every + /// lane's way out, so a grade that never reached a check — a failed clone, a patch that would not apply, an oracle + /// restore or a fault the lane caught, each quoting what failed (clone-failed:, grade-error:) — is + /// masked too. /// internal static BenchmarkGrade MaskedGrade(BenchmarkGrade grade, IReadOnlyList? secrets) { diff --git a/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorTurnService.Rehydrate.cs b/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorTurnService.Rehydrate.cs index 1feb8242c..655ef1c36 100644 --- a/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorTurnService.Rehydrate.cs +++ b/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorTurnService.Rehydrate.cs @@ -694,7 +694,7 @@ private async Task GradeResolveAcceptanceAsync(SupervisorPriorDe // gate IS the floor — so its own program file(s) are the run's oracle inventory (C3 narrowing). var spec = new SupervisorAcceptanceSpec { Command = command }; var oracleFloorPrograms = AcceptanceOracleProtection.ProgramCandidates(command); - var posture = resolver is null ? null : await ProducerPostureAsync(resolver.AgentRunId, teamId, cancellationToken).ConfigureAwait(false); + var producer = await ProducerGradingAsync(resolver?.AgentRunId, teamId, spec, cancellationToken).ConfigureAwait(false); // The branch arm carries the SAME inventory the patch arm below does. The floor-less overload compiles // and greens, but it silently reduces the grade to authored-only protection — which on this lane means @@ -702,12 +702,12 @@ private async Task GradeResolveAcceptanceAsync(SupervisorPriorDe // saying it went unanchored. There is no base sha to pair it with here (this lane resolves none), and // the anchor is what keeps that from reading as an oversight. if (!string.IsNullOrEmpty(resolver?.ProducedBranch)) - return await _acceptanceGrader.GradeAsync(new RepositoryAcceptanceGradeRequest { RepositoryId = repositoryId.Value, TeamId = teamId, Branch = resolver.ProducedBranch, Spec = spec, TimeoutSeconds = SupervisorLane.AcceptanceGradeTimeoutSeconds, Anchor = new OracleAnchor(null, oracleFloorPrograms), ProducerModel = ProducerModelOf(resolver!), Posture = posture }, cancellationToken).ConfigureAwait(false); + return await _acceptanceGrader.GradeAsync(new RepositoryAcceptanceGradeRequest { RepositoryId = repositoryId.Value, TeamId = teamId, Branch = resolver.ProducedBranch, Spec = spec, TimeoutSeconds = SupervisorLane.AcceptanceGradeTimeoutSeconds, Anchor = new OracleAnchor(null, oracleFloorPrograms), ProducerModel = ProducerModelOf(resolver!), Posture = producer.Posture, WorkflowSecrets = producer.WorkflowSecrets }, cancellationToken).ConfigureAwait(false); var manifest = resolver is not null ? await ResolveUnitManifestAsync(resolver.AgentRunId, repositoryId.Value, teamId, cancellationToken).ConfigureAwait(false) : null; if (manifest is { PatchArtifactId: not null, BaseSha: not null }) - return await _acceptanceGrader.GradePatchAsync(new PatchAcceptanceGradeRequest { RepositoryId = repositoryId.Value, TeamId = teamId, BaseSha = manifest.BaseSha!, PatchArtifactId = manifest.PatchArtifactId, Spec = spec, TimeoutSeconds = SupervisorLane.AcceptanceGradeTimeoutSeconds, OracleFloorPrograms = oracleFloorPrograms, ProducerModel = ProducerModelOf(resolver!), Posture = posture }, cancellationToken).ConfigureAwait(false); + return await _acceptanceGrader.GradePatchAsync(new PatchAcceptanceGradeRequest { RepositoryId = repositoryId.Value, TeamId = teamId, BaseSha = manifest.BaseSha!, PatchArtifactId = manifest.PatchArtifactId, Spec = spec, TimeoutSeconds = SupervisorLane.AcceptanceGradeTimeoutSeconds, OracleFloorPrograms = oracleFloorPrograms, ProducerModel = ProducerModelOf(resolver!), Posture = producer.Posture, WorkflowSecrets = producer.WorkflowSecrets }, cancellationToken).ConfigureAwait(false); return new BenchmarkGrade { Passed = false, Detail = "no-branch-or-repo" }; } @@ -722,7 +722,7 @@ private async Task GradeResolveAcceptanceAsync(SupervisorPriorDe catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogWarning(ex, "Acceptance grade for resolve decision {DecisionId} failed unexpectedly; recording not-accepted", decision.Id); - return new BenchmarkGrade { Passed = false, Detail = $"grade-error: {ex.Message}", Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault }; + return new BenchmarkGrade { Passed = false, Detail = MaskedDetail($"grade-error: {ex.Message}"), Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault }; } } @@ -966,19 +966,19 @@ private async Task GradeUnitAcceptanceAsync(SupervisorAgentResul if (repositoryId is null) return await GradeCapturedUnitAsync(result, spec, timeoutSeconds, teamId, cancellationToken).ConfigureAwait(false); - var posture = await ProducerPostureAsync(result.AgentRunId, teamId, cancellationToken).ConfigureAwait(false); + var producer = await ProducerGradingAsync(result.AgentRunId, teamId, spec, cancellationToken).ConfigureAwait(false); if (!string.IsNullOrEmpty(result.ProducedBranch)) { var anchor = await OracleAnchorAsync(result.AgentRunId, repositoryId.Value, spec, oracleFloorPrograms, teamId, cancellationToken).ConfigureAwait(false); - return await _acceptanceGrader.GradeAsync(new RepositoryAcceptanceGradeRequest { RepositoryId = repositoryId.Value, TeamId = teamId, Branch = result.ProducedBranch, Spec = spec, TimeoutSeconds = timeoutSeconds, Anchor = anchor, ProducerModel = ProducerModelOf(result), Posture = posture }, cancellationToken).ConfigureAwait(false); + return await _acceptanceGrader.GradeAsync(new RepositoryAcceptanceGradeRequest { RepositoryId = repositoryId.Value, TeamId = teamId, Branch = result.ProducedBranch, Spec = spec, TimeoutSeconds = timeoutSeconds, Anchor = anchor, ProducerModel = ProducerModelOf(result), Posture = producer.Posture, WorkflowSecrets = producer.WorkflowSecrets }, cancellationToken).ConfigureAwait(false); } var manifest = await ResolveUnitManifestAsync(result.AgentRunId, repositoryId.Value, teamId, cancellationToken).ConfigureAwait(false); if (manifest is { PatchArtifactId: not null, BaseSha: not null }) - return await _acceptanceGrader.GradePatchAsync(new PatchAcceptanceGradeRequest { RepositoryId = repositoryId.Value, TeamId = teamId, BaseSha = manifest.BaseSha!, PatchArtifactId = manifest.PatchArtifactId, Spec = spec, TimeoutSeconds = timeoutSeconds, OracleFloorPrograms = oracleFloorPrograms, ProducerModel = ProducerModelOf(result), Posture = posture }, cancellationToken).ConfigureAwait(false); + return await _acceptanceGrader.GradePatchAsync(new PatchAcceptanceGradeRequest { RepositoryId = repositoryId.Value, TeamId = teamId, BaseSha = manifest.BaseSha!, PatchArtifactId = manifest.PatchArtifactId, Spec = spec, TimeoutSeconds = timeoutSeconds, OracleFloorPrograms = oracleFloorPrograms, ProducerModel = ProducerModelOf(result), Posture = producer.Posture, WorkflowSecrets = producer.WorkflowSecrets }, cancellationToken).ConfigureAwait(false); return NotApplicableOrFailed(expectsChanges); } @@ -993,7 +993,7 @@ private async Task GradeUnitAcceptanceAsync(SupervisorAgentResul catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogWarning(ex, "Per-unit acceptance grade for agent {AgentRunId} in decision {DecisionId} failed unexpectedly; recording not-accepted", result.AgentRunId, decisionId); - return new BenchmarkGrade { Passed = false, Detail = $"grade-error: {ex.Message}", Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault }; + return new BenchmarkGrade { Passed = false, Detail = MaskedDetail($"grade-error: {ex.Message}"), Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault }; } } @@ -1016,9 +1016,9 @@ private async Task GradeCapturedUnitAsync(SupervisorAgentResult if (!Agents.AgentAcceptanceContract.GradesFromDeliverables(spec)) return new BenchmarkGrade { Passed = false, Detail = "no-branch-or-repo" }; - var posture = await ProducerPostureAsync(result.AgentRunId, teamId, cancellationToken).ConfigureAwait(false); + var producer = await ProducerGradingAsync(result.AgentRunId, teamId, spec, cancellationToken).ConfigureAwait(false); - var grade = await _acceptanceGrader.GradeCapturedAsync(new CapturedAcceptanceGradeRequest { AgentRunId = result.AgentRunId, TeamId = teamId, Spec = spec, TimeoutSeconds = timeoutSeconds, ProducerModel = ProducerModelOf(result), Posture = posture }, cancellationToken).ConfigureAwait(false); + var grade = await _acceptanceGrader.GradeCapturedAsync(new CapturedAcceptanceGradeRequest { AgentRunId = result.AgentRunId, TeamId = teamId, Spec = spec, TimeoutSeconds = timeoutSeconds, ProducerModel = ProducerModelOf(result), Posture = producer.Posture, WorkflowSecrets = producer.WorkflowSecrets }, cancellationToken).ConfigureAwait(false); return grade.Detail == ISupervisorAcceptanceGrader.NoDeliverablesCaptured ? DisambiguateEmptyWorld(result, grade) : grade; } @@ -1080,16 +1080,16 @@ private static BenchmarkGrade DisambiguateEmptyWorld(SupervisorAgentResult resul // The baseline runs under the CANDIDATE's posture: a differential between a base graded with network and // a candidate graded without it would measure the two sandboxes, not the work. - var posture = await ProducerPostureAsync(result.AgentRunId, teamId, cancellationToken).ConfigureAwait(false); + var producer = await ProducerGradingAsync(result.AgentRunId, teamId, spec, cancellationToken).ConfigureAwait(false); // The memo key is the FULL spec identity, not just the argv: Kind routes a different oracle, and // SetupCommand/Rubric/Schema change what the same argv means — two subtasks may share a Command yet // measure different contracts (adversarial-scan M1). The posture is part of it for the reason above. - var key = $"{repositoryId}@{manifest.BaseSha}#{System.Text.Json.JsonSerializer.Serialize(spec, Agents.AgentJson.Options)}#{System.Text.Json.JsonSerializer.Serialize(posture, Agents.AgentJson.Options)}"; + var key = $"{repositoryId}@{manifest.BaseSha}#{System.Text.Json.JsonSerializer.Serialize(spec, Agents.AgentJson.Options)}#{System.Text.Json.JsonSerializer.Serialize(producer.Posture, Agents.AgentJson.Options)}"; if (baselines.TryGetValue(key, out var memoized)) return memoized; - var grade = await _acceptanceGrader.GradeBaseAsync(new BaseAcceptanceGradeRequest { RepositoryId = repositoryId.Value, TeamId = teamId, BaseSha = manifest.BaseSha!, Spec = spec, TimeoutSeconds = spec.TimeoutSeconds ?? SupervisorLane.AcceptanceGradeTimeoutSeconds, Posture = posture }, cancellationToken).ConfigureAwait(false); + var grade = await _acceptanceGrader.GradeBaseAsync(new BaseAcceptanceGradeRequest { RepositoryId = repositoryId.Value, TeamId = teamId, BaseSha = manifest.BaseSha!, Spec = spec, TimeoutSeconds = spec.TimeoutSeconds ?? SupervisorLane.AcceptanceGradeTimeoutSeconds, Posture = producer.Posture, WorkflowSecrets = producer.WorkflowSecrets }, cancellationToken).ConfigureAwait(false); // A MEASURED baseline is shareable; an infra-classed one (a transient clone fault) is not — stamping it // onto every sibling off the same base would spread one blip across the whole fan-out. @@ -1121,7 +1121,7 @@ private async Task GradeUnitAcceptanceMultiRepoAsync(SupervisorA if (targets.Count == 0) return NotApplicableOrFailed(expectsChanges); - var posture = await ProducerPostureAsync(result.AgentRunId, teamId, cancellationToken).ConfigureAwait(false); + var producer = await ProducerGradingAsync(result.AgentRunId, teamId, spec, cancellationToken).ConfigureAwait(false); foreach (var target in targets) { @@ -1130,7 +1130,7 @@ private async Task GradeUnitAcceptanceMultiRepoAsync(SupervisorA { var anchor = await OracleAnchorAsync(result.AgentRunId, target.RepositoryId!.Value, spec, oracleFloorPrograms, teamId, cancellationToken).ConfigureAwait(false); - grade = await _acceptanceGrader.GradeAsync(new RepositoryAcceptanceGradeRequest { RepositoryId = target.RepositoryId!.Value, TeamId = teamId, Branch = target.ProducedBranch!, Spec = spec, TimeoutSeconds = spec.TimeoutSeconds ?? SupervisorLane.AcceptanceGradeTimeoutSeconds, Anchor = anchor, ProducerModel = ProducerModelOf(result), Posture = posture }, cancellationToken).ConfigureAwait(false); + grade = await _acceptanceGrader.GradeAsync(new RepositoryAcceptanceGradeRequest { RepositoryId = target.RepositoryId!.Value, TeamId = teamId, Branch = target.ProducedBranch!, Spec = spec, TimeoutSeconds = spec.TimeoutSeconds ?? SupervisorLane.AcceptanceGradeTimeoutSeconds, Anchor = anchor, ProducerModel = ProducerModelOf(result), Posture = producer.Posture, WorkflowSecrets = producer.WorkflowSecrets }, cancellationToken).ConfigureAwait(false); } catch (Workflows.Llm.LlmBudgetExceededException refused) { @@ -1143,7 +1143,7 @@ private async Task GradeUnitAcceptanceMultiRepoAsync(SupervisorA catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogWarning(ex, "Per-unit multi-repo acceptance grade for agent {AgentRunId} repo '{Alias}' in decision {DecisionId} failed unexpectedly; recording not-accepted", result.AgentRunId, target.Alias, decisionId); - return new BenchmarkGrade { Passed = false, Detail = $"repo '{target.Alias}': grade-error: {ex.Message}", Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault }; + return new BenchmarkGrade { Passed = false, Detail = MaskedDetail($"repo '{target.Alias}': grade-error: {ex.Message}"), Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault }; } // P5-2: re-emit the failing repo's grade via `with` (only the detail gains the repo tag) so its Class, @@ -1390,7 +1390,7 @@ internal async Task ApplyStopAcceptanceGradeAsync(Superviso // must too. _logger.LogWarning(ex, "Stop acceptance target resolution for run {RunId} failed unexpectedly; recording not-accepted", context.SupervisorRunId); - return execution with { OutcomeJson = SupervisorOutcome.AppendAcceptanceGrade(execution.OutcomeJson, false, $"grade-error: {ex.Message}") }; + return execution with { OutcomeJson = SupervisorOutcome.AppendAcceptanceGrade(execution.OutcomeJson, false, MaskedDetail($"grade-error: {ex.Message}")) }; } // C1 — a branchless stop is no longer an unconditional skip. A DELIVERABLE-kind gate does not need a head to @@ -1486,9 +1486,9 @@ internal async Task ApplyStopAcceptanceGradeAsync(Superviso { foreach (var unit in units) { - var posture = await ProducerPostureAsync(unit.AgentRunId, teamId, cancellationToken).ConfigureAwait(false); + var producer = await ProducerGradingAsync(unit.AgentRunId, teamId, spec, cancellationToken).ConfigureAwait(false); - var grade = await _acceptanceGrader.GradeCapturedAsync(new CapturedAcceptanceGradeRequest { AgentRunId = unit.AgentRunId, TeamId = teamId, Spec = spec, TimeoutSeconds = timeoutSeconds, ProducerModel = ProducerModelOf(unit), Posture = posture }, cancellationToken).ConfigureAwait(false); + var grade = await _acceptanceGrader.GradeCapturedAsync(new CapturedAcceptanceGradeRequest { AgentRunId = unit.AgentRunId, TeamId = teamId, Spec = spec, TimeoutSeconds = timeoutSeconds, ProducerModel = ProducerModelOf(unit), Posture = producer.Posture, WorkflowSecrets = producer.WorkflowSecrets }, cancellationToken).ConfigureAwait(false); if (grade.Passed) return (grade, false); @@ -1504,7 +1504,7 @@ internal async Task ApplyStopAcceptanceGradeAsync(Superviso catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogWarning(ex, "Branchless stop acceptance grade for run {RunId} failed unexpectedly; recording not-accepted", context.SupervisorRunId); - return (new BenchmarkGrade { Passed = false, Detail = $"grade-error: {ex.Message}", Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault }, false); + return (new BenchmarkGrade { Passed = false, Detail = MaskedDetail($"grade-error: {ex.Message}"), Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault }, false); } if (everCaptured) return (firstFailure!, false); @@ -1545,7 +1545,7 @@ internal async Task ApplyStopAcceptanceGradeAsync(Superviso ProducerModel = new ReviewModelIdentity { ModelCredentialModelId = context.SupervisorModelId, ConfiguredModel = decision.Usage?.RequestedModel, ObservedModel = decision.Usage?.ObservedModel }, }, cancellationToken).ConfigureAwait(false); - if (verdict.Failed) return (new BenchmarkGrade { Passed = false, Detail = $"grade-error: {verdict.FailureDetail}", Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault }, false); + if (verdict.Failed) return (new BenchmarkGrade { Passed = false, Detail = MaskedDetail($"grade-error: {verdict.FailureDetail}"), Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault }, false); var grade = Agents.Eval.Benchmark.Graders.LlmJudgeGrader.Aggregate(rubric, verdict); @@ -1679,7 +1679,7 @@ private async Task GradeStopTargetsAsync(Guid teamId, IReadOnlyL // by rewriting the check script the operator's floor runs. The floor's inventory gates BOTH // gates: the model's own tightening command can name a file the goal required editing, and // restoring that would void the very work the stop is shipping. - grade = await _acceptanceGrader.GradeAsync(new RepositoryAcceptanceGradeRequest { RepositoryId = target.RepositoryId, TeamId = teamId, Branch = target.Branch, Spec = spec, TimeoutSeconds = spec.TimeoutSeconds ?? SupervisorLane.AcceptanceGradeTimeoutSeconds, Anchor = new OracleAnchor(oracleBaseShas.GetValueOrDefault(target.RepositoryId), oracleFloorPrograms), Posture = posture }, cancellationToken).ConfigureAwait(false); + grade = await _acceptanceGrader.GradeAsync(new RepositoryAcceptanceGradeRequest { RepositoryId = target.RepositoryId, TeamId = teamId, Branch = target.Branch, Spec = spec, TimeoutSeconds = spec.TimeoutSeconds ?? SupervisorLane.AcceptanceGradeTimeoutSeconds, Anchor = new OracleAnchor(oracleBaseShas.GetValueOrDefault(target.RepositoryId), oracleFloorPrograms), Posture = posture, WorkflowSecrets = SecretsIn(spec) }, cancellationToken).ConfigureAwait(false); } catch (Workflows.Llm.LlmBudgetExceededException refused) { @@ -1692,7 +1692,7 @@ private async Task GradeStopTargetsAsync(Guid teamId, IReadOnlyL catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogWarning(ex, "Stop acceptance {Gate} grade for repo {Repo} failed unexpectedly; recording not-accepted", label, target.Alias); - return new BenchmarkGrade { Passed = false, Detail = $"{repoTag}{label}: grade-error: {ex.Message}", Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault }; + return new BenchmarkGrade { Passed = false, Detail = MaskedDetail($"{repoTag}{label}: grade-error: {ex.Message}"), Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault }; } // C3: re-emit via `with` so the graded object's Class, EvidenceArtifactId, EvidenceTail and @@ -1775,17 +1775,38 @@ private async Task> ReadResolve } /// - /// The posture a grade of 's work runs under, read off the unit's own stored task: the - /// tier and permissions it ran with, as admitted. Null when the row is not this team's or its task cannot be read, - /// and the grader then grades fail-closed. Every per-unit lane reads it for the unit whose bytes it runs. + /// What a grade of 's work runs under, read off the unit's own stored task. The posture: + /// the tier and permissions it ran with, as admitted — null when the row is not this team's or its task cannot be + /// read, and the grader then grades fail-closed. The workflow secrets the grade must keep out of everything it + /// stores: the ones the unit's sealed task embeds, and the ones this node's scope resolved into + /// — an operator-floor argv or a model-authored acceptance that carries one runs exactly, + /// and its evidence repeats the argv and the check's output. Every per-unit lane reads it for the unit whose bytes + /// it runs; a null (no unit) reads the spec's secrets alone. /// - private async Task ProducerPostureAsync(Guid agentRunId, Guid teamId, CancellationToken cancellationToken) + private async Task ProducerGradingAsync(Guid? agentRunId, Guid teamId, SupervisorAcceptanceSpec spec, CancellationToken cancellationToken) { - var taskJson = await _db.AgentRun.AsNoTracking().Where(r => r.Id == agentRunId && r.TeamId == teamId).Select(r => r.TaskJson).SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false); + var stored = agentRunId is not { } id ? null : await _db.AgentRun.AsNoTracking().Where(r => r.Id == id && r.TeamId == teamId).Select(r => new StoredTask(r.TaskJson, r.SealedTaskCiphertext)).SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false); - return AcceptanceGradingPosturePolicy.ForStoredTask(taskJson); + return new ProducerGrading(AcceptanceGradingPosturePolicy.ForStoredTask(stored?.TaskJson), [.. OpenedTaskSecrets(stored).Concat(SecretsIn(spec)).Distinct(StringComparer.Ordinal)]); } + /// The posture a unit's grade runs under, and the workflow secrets it masks in what it keeps. + private sealed record ProducerGrading(AcceptanceGradingPosture? Posture, IReadOnlyList WorkflowSecrets); + + private sealed record StoredTask(string TaskJson, string? SealedTaskCiphertext); + + /// The workflow secrets a unit's sealed task embeds (AgentTask.WorkflowSecrets, the values its executor masked); none for an unsealed task, or without a sealer to open it. + private IReadOnlyList OpenedTaskSecrets(StoredTask? stored) => + stored?.SealedTaskCiphertext is not { } ciphertext || _sealer is null ? [] : _sealer.Open(stored.TaskJson, ciphertext).Secrets; + + /// The secrets of this node's scope (the ambient capture redactor the engine pushes for the node) that an acceptance spec carries — the floor's argv the operator configured, or the one the model authored from a goal that resolved a secret. + private static IReadOnlyList SecretsIn(SupervisorAcceptanceSpec spec) => + Workflows.Llm.LlmCallContext.Current?.CaptureRedactor?.FoundIn(JsonSerializer.SerializeToElement(spec, AgentJson.Options)) ?? []; + + /// A fail-closed grade detail quoting what failed, masked with this node's scope secrets before the tape keeps it. + private static string MaskedDetail(string detail) => + Workflows.Llm.LlmCallContext.Current?.CaptureRedactor?.Redact(detail).Value ?? detail; + /// /// The posture of the run-level STOP grade, whose head mixes every unit's work: the run's own autonomy grant, which /// is the tier each unit it spawns is clamped to (RealSupervisorActionExecutor.AutonomyOf), with the diff --git a/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorTurnService.cs b/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorTurnService.cs index 261e885cf..a2cb839e3 100644 --- a/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorTurnService.cs +++ b/backend/src/CodeSpace.Core/Services/Supervisor/SupervisorTurnService.cs @@ -51,9 +51,12 @@ public sealed partial class SupervisorTurnService : ISupervisorTurnService, ISco /// The mode profiles the terminal authority gates a Success claim on — read at rehydrate so the stopped-now recital can name the stage(s) a stop would be refused over. OPTIONAL for the same reason as the judge above; a null one renders the recital exactly as it rendered before the stage line existed (the gate itself is the authority's, and is unaffected). private readonly Completion.IModeProfileRegistry? _modes; + /// Opens a unit's sealed task for the workflow secrets it embeds, so a grade of that unit's work masks them in what it keeps. OPTIONAL for the same reason as the judge above; DI always supplies it, and without it a grade still masks every secret its acceptance argv carries. + private readonly Workflows.Runtime.WorkflowSensitivePayloadSealer? _sealer; + private readonly ILogger _logger; - public SupervisorTurnService(ISupervisorDecisionLog ledger, ISupervisorDecider decider, ISupervisorActionExecutor executor, CodeSpaceDbContext db, ISupervisorAcceptanceGrader acceptanceGrader, IDecisionQueueService decisionQueue, IDecisionArbiter arbiter, IDecisionAnswerService decisionAnswer, Plans.IWorkPlanService workPlans, Workflows.Lifecycle.IRunRecordLogger recordLogger, Workflows.Artifacts.IArtifactOffloader offloader, IPublishManifestStore manifests, ISupervisorPublishedBranchResolver publishedBranches, Completion.ICompletionAssessmentComposer completion, Workflows.Budget.IBudgetLedger budget, Learning.ILessonReader lessons, IServiceScopeFactory scopeFactory, ILogger logger, Review.IRubricJudge? rubricJudge = null, Completion.IModeProfileRegistry? modes = null) + public SupervisorTurnService(ISupervisorDecisionLog ledger, ISupervisorDecider decider, ISupervisorActionExecutor executor, CodeSpaceDbContext db, ISupervisorAcceptanceGrader acceptanceGrader, IDecisionQueueService decisionQueue, IDecisionArbiter arbiter, IDecisionAnswerService decisionAnswer, Plans.IWorkPlanService workPlans, Workflows.Lifecycle.IRunRecordLogger recordLogger, Workflows.Artifacts.IArtifactOffloader offloader, IPublishManifestStore manifests, ISupervisorPublishedBranchResolver publishedBranches, Completion.ICompletionAssessmentComposer completion, Workflows.Budget.IBudgetLedger budget, Learning.ILessonReader lessons, IServiceScopeFactory scopeFactory, ILogger logger, Review.IRubricJudge? rubricJudge = null, Completion.IModeProfileRegistry? modes = null, Workflows.Runtime.WorkflowSensitivePayloadSealer? sealer = null) { _ledger = ledger; _decider = decider; @@ -74,6 +77,7 @@ public SupervisorTurnService(ISupervisorDecisionLog ledger, ISupervisorDecider d _scopeFactory = scopeFactory; _rubricJudge = rubricJudge; _modes = modes; + _sealer = sealer; _logger = logger; } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs index d5d8bbb18..d4c4143cf 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/AgentCodeNode.cs @@ -470,8 +470,10 @@ private static string UnpricedRetrySuffix(bool decides, decimal? maxCostUsd) => /// ONE cause inverts the resume decision: a gateway FORMAT fault /// () is the failure whose repair is to DROP the /// conversation rather than continue it, so the respawn takes the shared mitigation instead — see below. + /// + /// Internal so the resume E2E builds a continuation exactly as a respawn does, from a real resume payload. /// - private static AgentTask ApplyRespawnResumeHint(AgentTask task, JsonElement? priorAttemptPayload) + internal static AgentTask ApplyRespawnResumeHint(AgentTask task, JsonElement? priorAttemptPayload) { if (priorAttemptPayload is not { } payload) return task; diff --git a/backend/tests/CodeSpace.E2ETests/Agents/MaskedSessionResumeE2ETests.cs b/backend/tests/CodeSpace.E2ETests/Agents/MaskedSessionResumeE2ETests.cs new file mode 100644 index 000000000..1fa170395 --- /dev/null +++ b/backend/tests/CodeSpace.E2ETests/Agents/MaskedSessionResumeE2ETests.cs @@ -0,0 +1,108 @@ +using System.Text.Json; +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Services.Agents; +using CodeSpace.Core.Services.Workflows.Engine; +using CodeSpace.Core.Services.Workflows.Nodes.Builtin; +using CodeSpace.Core.Services.Workflows.Runtime; +using CodeSpace.IntegrationTests.Agents.Infrastructure; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Enums; +using Microsoft.EntityFrameworkCore; +using Shouldly; + +namespace CodeSpace.E2ETests.Agents; + +/// +/// The continuation a masked session must still buy. SX-13 masks every secret an agent wrote into its session before +/// the session is kept — and the kept session is exactly what the next attempt restores for --resume. So the +/// whole chain runs here, attempt to attempt: a first attempt writes its model credential and its goal's workflow +/// secret into its session; the executor captures it masked; the resume payload a retry is handed is built from that +/// row; the respawn turns the payload into the continuation exactly as agent.run does; the executor restores the +/// masked session into the second attempt's config home; and the second attempt's CLI reads it back line by line and +/// continues. +/// +/// Each attempt is brokered, as a deployment that requires confinement (this lane's) launches it and as any +/// deployment does whose worker can front the credential: the agent holds the lease's run token where the key would be, +/// and that token is what it echoes. Unbrokered, such a deployment refuses the launch before the agent starts. +/// +/// Fidelity: 🟢 high for everything but the model. The executor, model-credential broker, durable runner, +/// config-home materialization, capture, resume payload () and +/// respawn hint () are production; the CLI is . The pinned +/// real CLI resumes a masked session in the sandbox lane (GoalChannelE2ETests, the masked-resume arm); the +/// direct-key path is masked in AgentSessionTranscriptMaskingFlowTests. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "E2E")] +[Trait("Surface", "Engine")] +public sealed class MaskedSessionResumeE2ETests(PostgresFixture fixture) +{ + [Fact] + public async Task A_second_attempt_resumes_from_the_masked_session_the_first_one_left() + { + if (OperatingSystem.IsWindows()) return; // the scripted CLI is /bin/sh (Rule 12.1) + + // MUTATION this pins: capture the session unmasked → the restored session carries the credential the first + // attempt echoed and its workflow secret → red. A mask that broke a line fails ParseLines here and the CLI's + // read-back below; the escaped-value cases of the mask itself are pinned in SessionTranscriptRedactionTests. + var key = $"CANARY-G2-e2e-sk-ant-{Guid.NewGuid():N}"; + var workflowSecret = $"CANARY-G2-e2e-wf-{Guid.NewGuid():N}"; + var sessionId = $"s-{Guid.NewGuid():N}"; + var firstCli = new SessionKeepingCli(sessionId); + + var (_, first) = await SessionKeepingCli.SeedQueuedRunAsync(fixture, key, workflowSecret); + await SessionKeepingCli.ExecuteAsync(fixture, first, firstCli, brokered: true); + + var payload = await ResumePayloadAsync(first); + payload.GetProperty("sessionId").GetString().ShouldBe(sessionId, "fixture check: the first attempt captured its session"); + var echoed = EchoedCredential(firstCli, key); + + var cli = new SessionKeepingCli($"s-{Guid.NewGuid():N}"); // a fresh id, so only the restored session can name the right file + var (_, second) = await SessionKeepingCli.SeedQueuedRunAsync(fixture, key, workflowSecret, task => AgentCodeNode.ApplyRespawnResumeHint(task, payload)); + await SessionKeepingCli.ExecuteAsync(fixture, second, cli, brokered: true); + + var continued = cli.Invocations.ShouldHaveSingleItem(); + continued.ResumeFromSessionId.ShouldBe(sessionId, "the respawn continues the first attempt's session"); + continued.Goal.ShouldContain(workflowSecret, Case.Sensitive, "the respawn re-sends its sealed goal, so the agent still has the value its restored turn holds masked"); + + var restored = continued.RestoredTranscript.ShouldNotBeNull("the continuation was handed the first attempt's session"); + restored.ShouldNotContain(echoed, Case.Sensitive, "the restored session never carries the credential the first attempt echoed"); + restored.ShouldNotContain(key, Case.Sensitive, "nor the key that credential fronts"); + restored.ShouldNotContain(workflowSecret, Case.Sensitive, "nor the workflow secret its goal embedded"); + var restoredLines = SessionKeepingCli.ParseLines(restored); + restoredLines.Count.ShouldBe(3, "fixture check: init, the echoed credential, the restated goal"); + restoredLines.Single(line => line.GetProperty("type").GetString() == "user").GetProperty("tool_result").GetString() + .ShouldBe(SecretRedactor.Placeholder, "the echoed credential is in the restored session, masked inside its JSON value"); + + using var verify = fixture.BeginScope(); + var row = await verify.Resolve().AgentRun.AsNoTracking().SingleAsync(r => r.Id == second); + row.Status.ShouldBe(AgentRunStatus.Succeeded, $"the continuation must run to the end over its masked session: {row.Error}"); + + var read = SessionKeepingCli.ParseLines(JsonSerializer.Deserialize(row.ResultJson!, AgentJson.Options)!.Transcript); + read.Select(line => line.GetRawText()).Take(restoredLines.Count).ShouldBe(restoredLines.Select(line => line.GetRawText()), "the CLI read back exactly the masked session it was restored, every line parsing"); + read[^1].GetProperty("text").GetString().ShouldBe(SessionKeepingCli.ResumedText, "and continued past it"); + } + + /// The resume payload a retry of the first attempt is handed, built from its stored row by the production notifier. + private async Task ResumePayloadAsync(Guid agentRunId) + { + using var scope = fixture.BeginScope(); + var row = await scope.Resolve().AgentRun.AsNoTracking().SingleAsync(r => r.Id == agentRunId); + + row.Status.ShouldBe(AgentRunStatus.Succeeded, $"fixture check: the first attempt landed — {row.Error}"); + + return JsonDocument.Parse(WorkflowResumeAgentRunCompletionNotifier.BuildResumePayload(row)).RootElement.Clone(); + } + + /// The credential an attempt's agent was handed, and echoed into its session: the lease's run token, never . + private static string EchoedCredential(SessionKeepingCli cli, string key) + { + var environment = cli.Invocations.ShouldHaveSingleItem().Environment; + + environment.Values.ShouldNotContain(key, "fixture check: the attempt was brokered, so the key never reached its agent"); + environment[SessionKeepingCli.KeyEnvVar].ShouldNotBeNullOrWhiteSpace("fixture check: the agent was handed a run token to echo"); + + return environment[SessionKeepingCli.KeyEnvVar]; + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Agents/AgentSessionTranscriptMaskingFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Agents/AgentSessionTranscriptMaskingFlowTests.cs new file mode 100644 index 000000000..0df07033f --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Agents/AgentSessionTranscriptMaskingFlowTests.cs @@ -0,0 +1,194 @@ +using System.Text.Json; +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Services.Agents; +using CodeSpace.Core.Services.Workflows.Engine; +using CodeSpace.IntegrationTests.Agents.Infrastructure; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Enums; +using Microsoft.EntityFrameworkCore; +using Shouldly; + +namespace CodeSpace.IntegrationTests.Agents; + +/// +/// SX-13 against real Postgres, the real executor, the real durable runner, the real checkpointer and the real artifact +/// store: an agent that echoes its model key and its goal's workflow secret into its OWN session file. The live stdout +/// path was always masked; the session transcript went around it, into result_jsonb.sessionTranscript, the +/// mid-run checkpoint artifacts and the resume payload a retry is handed. Each of the three is read back here from +/// where production stores it, and each must hold the mask — inside JSON that still parses, because the next attempt +/// hands exactly these lines to --resume. Both paths that capture a session are driven: the launch, and the +/// re-attach that finishes a run whose launching worker went away. +/// +/// Fidelity: 🟢 high — only the model is scripted (). The key reaches the agent +/// through the production credential resolve and projection, and the workflow secret through the production task +/// seal and open. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "Integration")] +public sealed class AgentSessionTranscriptMaskingFlowTests +{ + private readonly PostgresFixture _fixture; + + public AgentSessionTranscriptMaskingFlowTests(PostgresFixture fixture) => _fixture = fixture; + + [Fact] + public async Task A_key_and_a_workflow_secret_the_agent_wrote_into_its_session_are_masked_wherever_the_session_is_kept() + { + if (OperatingSystem.IsWindows()) return; // the scripted CLI is /bin/sh (Rule 12.1) + + // MUTATIONS this pins: drop the mask from CaptureSessionTranscriptAsync → result_jsonb and the resume payload carry + // both canaries; drop it from the checkpointer → the checkpoint artifact does. + var key = $"CANARY-G2-it-sk-ant-{Guid.NewGuid():N}"; + var workflowSecret = $"CANARY-G2-it-wf-{Guid.NewGuid():N}"; + var (teamId, runId) = await SessionKeepingCli.SeedQueuedRunAsync(_fixture, key, workflowSecret); + var cli = new SessionKeepingCli($"s-{Guid.NewGuid():N}"); + + await SessionKeepingCli.ExecuteAsync(_fixture, runId, cli); + + using var verify = _fixture.BeginScope(); + var db = verify.Resolve(); + var row = await db.AgentRun.AsNoTracking().SingleAsync(r => r.Id == runId); + + row.Status.ShouldBe(AgentRunStatus.Succeeded, $"the run must land for its stored session to mean anything: {row.Error}"); + cli.Invocations.ShouldHaveSingleItem().Environment[SessionKeepingCli.KeyEnvVar].ShouldBe(key, "fixture check: the agent was really handed the key it echoes"); + cli.Invocations[0].Goal.ShouldContain(workflowSecret, Case.Sensitive, "fixture check: the agent was really handed the exact goal, opened from its sealed task"); + + var result = JsonSerializer.Deserialize(row.ResultJson!, AgentJson.Options)!; + AssertMaskedSession("result_jsonb.sessionTranscript", result.SessionTranscript, key, workflowSecret); + + var payload = WorkflowResumeAgentRunCompletionNotifier.BuildResumePayload(row); + payload.ShouldNotContain(key, Case.Sensitive, "the resume payload a retry is handed must not carry the key"); + payload.ShouldNotContain(workflowSecret, Case.Sensitive, "nor the workflow secret"); + AssertMaskedSession("the resume payload's sessionTranscript", JsonDocument.Parse(payload).RootElement.GetProperty("sessionTranscript").GetString(), key, workflowSecret); + + var checkpoints = await SessionKeepingCli.KeptCheckpointsAsync(_fixture, teamId, runId, landed: _ => true); + + checkpoints.ShouldNotBeEmpty("fixture check: the live tick must have checkpointed the session, or this arm proves nothing about checkpoints"); + + foreach (var checkpoint in checkpoints) AssertMaskedSession("a checkpoint artifact", checkpoint, key, workflowSecret); + } + + [Fact] + public async Task A_session_a_re_attach_captures_after_the_launching_worker_went_away_is_masked_wherever_it_is_kept() + { + if (OperatingSystem.IsWindows()) return; // the scripted CLI is /bin/sh (Rule 12.1) + + // The durability path session continuity exists for: the worker that launched the run goes away while the agent + // works, and the next one re-attaches to the live process, tails it to its end, checkpoints and captures its + // session — under the redactor it rebuilds from the sealed task and the credential. The agent names its session + // only once the first worker is gone, so every checkpoint and the capture are the re-attach's own. The task names + // its own workspace, which is what lets a re-attach address a session the Claude layout keys on it. + // MUTATIONS this pins: drop the redactor from the re-attach's capture → result_jsonb and the resume payload carry + // both canaries; drop it from the re-attach's checkpoint tick → a checkpoint artifact does. + using var context = new ReattachedRunContext(); + var key = $"CANARY-G2-reattach-sk-ant-{Guid.NewGuid():N}"; + var workflowSecret = $"CANARY-G2-reattach-wf-{Guid.NewGuid():N}"; + var (teamId, runId) = await SessionKeepingCli.SeedQueuedRunAsync(_fixture, key, workflowSecret, task => task with { WorkspaceDirectory = context.Workspace }); + var cli = new SessionKeepingCli($"s-{Guid.NewGuid():N}", context.ReleaseFile); + + await LaunchAndLoseTheWorkerAsync(runId, cli); + + var reattach = SessionKeepingCli.ReattachAsync(_fixture, await ReserveReattachAsync(runId), cli); + context.Release(); + (await Task.WhenAny(reattach, Task.Delay(TimeSpan.FromSeconds(90)))).ShouldBe(reattach, $"the re-attach of run {runId} did not land within 90s — check agent_run.status and whether its spool holds an exit marker"); + await reattach; + + using var verify = _fixture.BeginScope(); + var row = await verify.Resolve().AgentRun.AsNoTracking().SingleAsync(r => r.Id == runId); + + row.Status.ShouldBe(AgentRunStatus.Succeeded, $"the re-attach must tail the agent to its end for its session to mean anything: {row.Error}"); + cli.Invocations.ShouldHaveSingleItem().Goal.ShouldContain(workflowSecret, Case.Sensitive, "fixture check: the agent was handed the exact goal, opened from its sealed task"); + + var result = JsonSerializer.Deserialize(row.ResultJson!, AgentJson.Options)!; + AssertMaskedSession("result_jsonb.sessionTranscript", result.SessionTranscript, key, workflowSecret); + + var payload = WorkflowResumeAgentRunCompletionNotifier.BuildResumePayload(row); + payload.ShouldNotContain(key, Case.Sensitive, "the resume payload a retry is handed must not carry the key"); + payload.ShouldNotContain(workflowSecret, Case.Sensitive, "nor the workflow secret"); + + var checkpoints = await SessionKeepingCli.KeptCheckpointsAsync(_fixture, teamId, runId, landed: _ => true); + + checkpoints.ShouldNotBeEmpty("fixture check: the re-attach's tick must have checkpointed the session, or this arm proves nothing about its checkpoints"); + + foreach (var checkpoint in checkpoints) AssertMaskedSession("a checkpoint artifact", checkpoint, key, workflowSecret); + } + + /// Launch the run on a worker that then goes away: wait until the launch is on the row, then cancel the job — which leaves the run Running and its agent alive for a re-attach. + private async Task LaunchAndLoseTheWorkerAsync(Guid runId, SessionKeepingCli cli) + { + using var worker = new CancellationTokenSource(); + var execution = SessionKeepingCli.ExecuteAsync(_fixture, runId, cli, worker.Token); + var deadline = DateTimeOffset.UtcNow.AddSeconds(60); + + while (await HandleOfAsync(runId) is null) + { + if (execution.IsCompleted) await execution; // surfaces the launch's own failure instead of waiting out the deadline + + DateTimeOffset.UtcNow.ShouldBeLessThan(deadline, $"run {runId} recorded no runner handle within 60s — the launch never reached the durable runner; check agent_run.error for that run"); + await Task.Delay(100); + } + + await worker.CancelAsync(); + + try { await execution; } + catch (OperationCanceledException) { /* the worker went away — the live agent it left is what the re-attach takes over */ } + + (await HandleOfAsync(runId)).ShouldNotBeNull("fixture check: the run is left for a re-attach"); + } + + private async Task HandleOfAsync(Guid runId) + { + using var scope = _fixture.BeginScope(); + var run = await scope.Resolve().AgentRun.AsNoTracking().SingleAsync(r => r.Id == runId); + + return run.Status == AgentRunStatus.Running ? run.RunnerHandleJson : null; + } + + /// Lapse the run's observation lease and reserve the re-attach — the reconciler's atomic step, taken here so the test owns when the next worker takes over. + private async Task ReserveReattachAsync(Guid runId) + { + using var scope = _fixture.BeginScope(); + + await scope.Resolve().Database.ExecuteSqlInterpolatedAsync($"UPDATE agent_run SET lease_expires_at = clock_timestamp() - interval '1 hour' WHERE id = {runId}"); + + return (await scope.Resolve().ReserveReattachAsync(runId, CancellationToken.None)).ShouldNotBeNull("fixture check: the lapsed run is reservable for a re-attach"); + } + + /// The run's own workspace and the file that releases its agent. Disposing releases the agent too, so a failing test never leaves it waiting. + private sealed class ReattachedRunContext : IDisposable + { + private readonly string _root = Path.Combine(Path.GetTempPath(), $"cs-g2-reattach-{Guid.NewGuid():N}"); + + public ReattachedRunContext() => Directory.CreateDirectory(Workspace); + + public string Workspace => Path.Combine(_root, "workspace"); + + public string ReleaseFile => Path.Combine(_root, "release"); + + public void Release() => File.WriteAllText(ReleaseFile, "go"); + + public void Dispose() + { + try { Release(); } catch (IOException) { /* best effort */ } + + try { Directory.Delete(_root, recursive: true); } catch (IOException) { /* best effort: the agent may still be exiting */ } + } + } + + /// The session as kept: neither canary, every line parses, and the two values the agent wrote are there masked — which also proves the fixture really wrote them. + private static void AssertMaskedSession(string where, string? session, string key, string workflowSecret) + { + session.ShouldNotBeNullOrEmpty($"{where} must hold the captured session — an empty one would pass every check below"); + session!.ShouldNotContain(key, Case.Sensitive, $"{where} carries the model key the agent echoed"); + session.ShouldNotContain(workflowSecret, Case.Sensitive, $"{where} carries the workflow secret the goal embedded"); + + var lines = SessionKeepingCli.ParseLines(session); + + lines.Single(l => l.GetProperty("type").GetString() == "user").GetProperty("tool_result").GetString() + .ShouldBe(SecretRedactor.Placeholder, $"{where}: the echoed key is masked inside its JSON value"); + lines.Single(l => l.TryGetProperty("text", out var text) && text.GetString()!.StartsWith("Deploy", StringComparison.Ordinal)).GetProperty("text").GetString() + .ShouldBe($"Deploy the billing service with {SecretRedactor.Placeholder}", $"{where}: the workflow secret is masked inside its JSON value"); + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Agents/Infrastructure/SessionKeepingCli.cs b/backend/tests/CodeSpace.IntegrationTests/Agents/Infrastructure/SessionKeepingCli.cs new file mode 100644 index 000000000..4c7f20cae --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Agents/Infrastructure/SessionKeepingCli.cs @@ -0,0 +1,270 @@ +using System.Text.Json; +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Agents; +using CodeSpace.Core.Services.Agents.Authority; +using CodeSpace.Core.Services.Agents.Credentials; +using CodeSpace.Core.Services.Agents.Harnesses.Claude; +using CodeSpace.Core.Services.Agents.Recovery; +using CodeSpace.Core.Services.Agents.Sandbox; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Core.Services.Workflows.Artifacts; +using CodeSpace.Core.Services.Workflows.Runtime; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Artifacts; +using CodeSpace.Messages.Authorization; +using CodeSpace.Messages.Constants; +using CodeSpace.Messages.Enums; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; + +namespace CodeSpace.IntegrationTests.Agents.Infrastructure; + +/// +/// A scripted agent CLI that keeps a resumable session the way the real ones do: a JSONL file in its config home, at +/// the path the PRODUCTION locates (never restated here), restored by the runner from +/// and read back on a continuation. Only the model is scripted; the executor, +/// the durable runner, the config-home materialization, the capture, the checkpointer and the artifact store are the +/// production ones. +/// +/// A fresh run writes into its session what a real agent writes there when it runs printenv as a tool +/// call and restates its goal: the model credential the harness injected (), and the goal itself — +/// which carries any workflow secret the task embeds. Its stdout names the session and nothing secret. +/// +/// The credential is whatever the launch handed it: the provider key on the direct path, or — when the executor +/// was given a broker () — the lease's run token in the key's place, as a real harness's +/// brokered projection carries it. A deployment that requires confinement refuses the direct path outright. +/// +/// Given a release file, a fresh run writes its session and then holds — naming nothing on stdout — until that +/// file appears: the shape of an agent whose launching worker goes away before the agent says anything, so everything +/// about its session is the re-attaching worker's to see. +/// +/// A continuation ( set) prints every line of the session file it was +/// restored and then one line of its own. What it printed is what it READ: the restored file as the runner wrote it, +/// so a test parsing those lines is parsing exactly what a resuming CLI parses. +/// +public sealed class SessionKeepingCli : IAgentHarness, IAgentSessionTranscript, IModelCredentialProjector, IBrokeredModelCredentialProjector +{ + public const string HarnessKind = "scripted-session"; + + /// The env var this harness projects the model credential onto — the key, or a brokered run's token — and the one the agent echoes into its session. + public const string KeyEnvVar = "SCRIPTED_API_KEY"; + + /// The env var a brokered launch's broker address rides, beside the run token on . + public const string BaseUrlEnvVar = "SCRIPTED_BASE_URL"; + + /// The line a continuation prints after the restored session — the proof it ran to the end. + public const string ResumedText = "resumed from the restored session"; + + private const string ConfigHomeEnvVar = "CS_TEST_CONFIG_DIR"; + private const string GoalEnvVar = "SCRIPTED_GOAL"; + private static readonly ClaudeCodeHarness Layout = new(); + private readonly string _sessionId; + private readonly string? _releaseFile; + + public SessionKeepingCli(string sessionId, string? releaseFile = null) + { + _sessionId = sessionId; + _releaseFile = releaseFile; + } + + /// The task each launch was built from: what the agent was actually handed. + public List Invocations { get; } = []; + + public string Kind => HarnessKind; + public string Version => "test"; + public IReadOnlyList Models { get; } = ["test-model"]; + public IReadOnlyList SupportedProviders { get; } = ["Anthropic"]; + + public IReadOnlyDictionary ProjectToEnv(ResolvedModelCredential credential) => new Dictionary { [KeyEnvVar] = credential.ApiKey ?? "" }; + + public IReadOnlyDictionary ProjectBrokered(BrokeredModelCredential brokered) => new Dictionary { [BaseUrlEnvVar] = brokered.BaseUrl, [KeyEnvVar] = brokered.RunToken }; + + public string? SessionTranscriptRelativePath(string configHome, string? workspaceDirectory, string? sessionId) => + ((IAgentSessionTranscript)Layout).SessionTranscriptRelativePath(configHome, workspaceDirectory, sessionId); + + public SandboxSpec BuildInvocation(AgentTask task) + { + Invocations.Add(task); + + // A continuation restores the session it was told to resume, where the production layout puts it — as the real + // harness does — so a resume payload naming the wrong session would leave this CLI nothing to read. + var relative = SessionTranscriptRelativePath("", task.WorkspaceDirectory, task.ResumeFromSessionId ?? _sessionId) ?? throw new InvalidOperationException("fixture check: the production layout could not address this run's session"); + var session = $"\"${ConfigHomeEnvVar}/{relative}\""; + var restored = task.RestoredTranscript is { Length: > 0 } transcript ? transcript : null; + + return new SandboxSpec + { + Command = "/bin/sh", + Args = ["-c", restored is null ? FreshScript(session, relative) : ContinueScript(session)], + WorkingDirectory = task.WorkspaceDirectory, + TimeoutSeconds = 120, + ConfigHomeEnvVars = [ConfigHomeEnvVar], + Environment = new Dictionary(task.Environment) { [GoalEnvVar] = task.Goal }, + ConfigHomeFiles = restored is null ? [] : [new ConfigHomeFile { RelativePath = relative, Content = restored }], + }; + } + + /// Write the session first, then — once released, when a release file was given — name it on stdout, then stay alive long enough for the drain tick to checkpoint it. + private string FreshScript(string session, string relative) => + $"mkdir -p \"${ConfigHomeEnvVar}/{Path.GetDirectoryName(relative)!.Replace('\\', '/')}\" && " + + $"printf '{{\"type\":\"system\",\"subtype\":\"init\",\"session_id\":\"{_sessionId}\"}}\\n' > {session} && " + + $"printf '{{\"type\":\"user\",\"session_id\":\"{_sessionId}\",\"tool_result\":\"%s\"}}\\n' \"${KeyEnvVar}\" >> {session} && " + + $"printf '{{\"type\":\"assistant\",\"session_id\":\"{_sessionId}\",\"text\":\"%s\"}}\\n' \"${GoalEnvVar}\" >> {session} && " + + (_releaseFile is null ? "" : $"while [ ! -f '{_releaseFile}' ]; do sleep 0.1; done && ") + + $"printf '{{\"type\":\"assistant\",\"session_id\":\"{_sessionId}\",\"text\":\"working\"}}\\n' && sleep 2"; + + /// Read the restored session back, line for line, the way a resuming CLI loads it — then continue. + private string ContinueScript(string session) => + $"test -f {session} || {{ echo 'restored session missing' >&2; exit 3; }}; " + + $"while IFS= read -r line; do printf '%s\\n' \"$line\"; done < {session}; " + + $"printf '{{\"type\":\"assistant\",\"session_id\":\"{_sessionId}\",\"text\":\"{ResumedText}\"}}\\n'"; + + public IReadOnlyList ParseEvents(string rawLine) => + string.IsNullOrWhiteSpace(rawLine) ? [] : [new AgentEvent { Kind = AgentEventKind.AssistantMessage, Text = rawLine.Trim(), Data = TryParse(rawLine) }]; + + public IAgentEventFolder CreateFolder() => ScriptedFolders.Result(); + + private static JsonElement? TryParse(string rawLine) + { + try { return JsonSerializer.Deserialize(rawLine); } + catch (JsonException) { return null; } + } + + /// + /// Seed what a launch needs to run a agent for real: a team whose owner's authority + /// receipt the executor accepts, a model credential holding , and a QUEUED run whose task + /// is sealed exactly as the agent-run store seals one that embeds . + /// + public static async Task<(Guid TeamId, Guid RunId)> SeedQueuedRunAsync(PostgresFixture fixture, string modelKey, string workflowSecret, Func? shape = null) + { + using var scope = fixture.BeginScope(); + var db = scope.Resolve(); + var (teamId, owner) = await SeedTeamAsync(db); + var credentialId = Guid.NewGuid(); + var runId = Guid.NewGuid(); + + db.ModelCredential.Add(new ModelCredential { Id = credentialId, TeamId = teamId, Provider = "Anthropic", DisplayName = "scripted", EncryptedApiKey = scope.Resolve().Encrypt(modelKey), Status = CredentialStatus.Active }); + + var task = new AgentTask + { + Goal = $"Deploy the billing service with {workflowSecret}", Harness = HarnessKind, TimeoutSeconds = 120, ModelCredentialId = credentialId, + CheckpointSessionTranscript = true, ExecutionAuthority = AuthorityOf(teamId, owner, runId), WorkflowSecrets = [workflowSecret], + }; + var stored = AgentTaskSecrets.Seal(shape is null ? task : shape(task), scope.Resolve()); + + db.AgentRun.Add(new AgentRun { Id = runId, TeamId = teamId, Harness = HarnessKind, Status = AgentRunStatus.Queued, TaskJson = stored.PublicJson, SealedTaskCiphertext = stored.Ciphertext }); + await db.SaveChangesAsync(); + + return (teamId, runId); + } + + /// + /// Drive the REAL executor on with this harness and the container's own collaborators — its + /// checkpointer and payload sealer included. Cancelling is a worker going away: + /// the executor re-raises it and leaves the run Running for a re-attach. hands it the + /// container's model-credential broker too, so the credential is fronted as a confining deployment requires rather + /// than injected — the only way such a deployment launches the run at all. + /// + public static async Task ExecuteAsync(PostgresFixture fixture, Guid runId, IAgentHarness harness, CancellationToken cancellationToken = default, bool brokered = false) + { + using var scope = fixture.BeginScope(); + + await BuildExecutor(scope, harness, brokered ? scope.Resolve() : null).ExecuteAsync(runId, cancellationToken); + } + + /// Drive the REAL executor's re-attach on a reservation the test took, with the same collaborators a launch gets. + public static async Task ReattachAsync(PostgresFixture fixture, AgentRunReattachReservation reservation, IAgentHarness harness) + { + using var scope = fixture.BeginScope(); + + await BuildExecutor(scope, harness, broker: null).ReattachAsync(reservation, CancellationToken.None); + } + + private static AgentRunExecutor BuildExecutor(ILifetimeScope scope, IAgentHarness harness, IModelCredentialBroker? broker) => + new( + scope.Resolve(), new AgentHarnessRegistry([harness]), + new HarnessModelReconciler(new AgentHarnessRegistry([harness]), scope.Resolve(), scope.Resolve()), + scope.Resolve(), scope.Resolve(), scope.Resolve(), + scope.Resolve(), scope.Resolve(), scope.Resolve(), + scope.Resolve(), scope.Resolve(), scope.Resolve(), scope.Resolve(), + scope.Resolve(), scope.Resolve(), + scope.Resolve(), scope.Resolve>(), + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, + credentialBroker: broker, sessionCheckpointer: scope.Resolve(), sealer: scope.Resolve()); + + /// Every complete line of a JSONL text, each parsed — throws on any line a resuming CLI could not read. + public static IReadOnlyList ParseLines(string jsonl) => + jsonl.Split('\n', StringSplitOptions.RemoveEmptyEntries).Select(line => JsonDocument.Parse(line).RootElement.Clone()).ToList(); + + /// + /// Every session-transcript checkpoint the run kept, once one of them satisfies — or what is + /// kept when runs out, for the caller's own fixture check to name. A checkpoint + /// uploads off the run's path and a landing waits only seconds for it, so on a cold store the run can be terminal + /// while its last checkpoint is still being written. + /// + public static async Task> KeptCheckpointsAsync(PostgresFixture fixture, Guid teamId, Guid runId, Func landed) + { + var deadline = DateTimeOffset.UtcNow + CheckpointLandingWait; + + while (true) + { + var kept = await ReadKeptCheckpointsAsync(fixture, teamId, runId); + + if (kept.Any(landed) || DateTimeOffset.UtcNow >= deadline) return kept; + + await Task.Delay(200); + } + } + + /// The checkpointer's own upload budget, and some slack: past it no checkpoint is still coming. + private static readonly TimeSpan CheckpointLandingWait = AgentRunExecutor.SessionCheckpointUploadBudget + TimeSpan.FromSeconds(15); + + /// The run's kept checkpoints, read back from where production stores them: the retention rows that hold them, and the artifact store. + private static async Task> ReadKeptCheckpointsAsync(PostgresFixture fixture, Guid teamId, Guid runId) + { + using var scope = fixture.BeginScope(); + + var artifactIds = await scope.Resolve().WorkflowArtifactRetention.AsNoTracking() + .Where(d => d.TeamId == teamId && d.HolderId == runId && d.RetentionClass == nameof(ArtifactRetentionClass.SessionTranscriptCheckpoint)) + .Select(d => d.ArtifactId).ToListAsync(); + + var checkpoints = new List(); + + foreach (var artifactId in artifactIds) + checkpoints.Add(System.Text.Encoding.UTF8.GetString((await scope.Resolve().GetBytesAsync(teamId, artifactId, CancellationToken.None))!.Bytes)); + + return checkpoints; + } + + private static async Task<(Guid TeamId, (Guid UserId, Guid MembershipId, Guid SecurityStamp) Owner)> SeedTeamAsync(CodeSpaceDbContext db) + { + var userId = Guid.NewGuid(); + var teamId = Guid.NewGuid(); + var membershipId = Guid.NewGuid(); + + db.User.Add(new User { Id = userId, Email = $"session-{userId:N}@test.local", Name = $"session-{userId:N}" }); + db.Team.Add(new Team { Id = teamId, Slug = $"session-{teamId:N}", Name = "Session Team", Kind = TeamKind.Workspace }); + db.TeamMembership.Add(new TeamMembership { Id = membershipId, TeamId = teamId, UserId = userId, Role = TeamRole.Owner }); + await db.SaveChangesAsync(); + + var stamp = (await db.User.AsNoTracking().SingleAsync(u => u.Id == userId)).SecurityStamp; + + return (teamId, (userId, membershipId, stamp)); + } + + private static AgentExecutionAuthority AuthorityOf(Guid teamId, (Guid UserId, Guid MembershipId, Guid SecurityStamp) owner, Guid logicalRunId) => new() + { + Version = ExecutionAuthorityService.ReceiptVersion, + PolicyVersion = ExecutionAuthorityService.PolicyVersion, + TeamId = teamId, + LogicalRunId = logicalRunId, + SourceKind = "standalone", + DefinitionHash = "", + GrantedCeiling = AgentAutonomyPolicy.DeploymentCeiling, + IssuedAt = DateTimeOffset.UtcNow.AddMinutes(-30), + Subjects = [new AgentAuthoritySubject { Kind = "launcher", UserId = owner.UserId, SecurityStamp = owner.SecurityStamp, MembershipId = owner.MembershipId, IssuedRole = TeamRole.Owner, Permission = TeamPermissions.RunsLaunch, GlobalAdmin = false }], + }; +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunReviseLoopFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunReviseLoopFlowTests.cs index 04f3691da..f1c53e922 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunReviseLoopFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunReviseLoopFlowTests.cs @@ -11,7 +11,9 @@ using CodeSpace.Core.Services.Agents.Workspace; using CodeSpace.Core.Services.Credentials; using CodeSpace.Core.Services.Supervisor; +using CodeSpace.Core.Services.Workflows.Engine; using CodeSpace.Core.Services.Workflows.Planning.Planners; +using CodeSpace.IntegrationTests.Agents.Infrastructure; using CodeSpace.IntegrationTests.Infrastructure; using CodeSpace.IntegrationTests.Workflows.Infrastructure; using CodeSpace.Messages.Agents; @@ -96,6 +98,48 @@ public async Task A_revision_whose_session_overflows_the_launch_frame_goes_on_co (await scope.Resolve().GetEventsAsync(runId, run.TeamId, afterSequence: 0, CancellationToken.None)).ShouldContain(e => e.Text == AgentRunExecutor.ReviseRanColdNote); } + [Fact] + public async Task A_warm_revision_continues_the_exact_conversation_while_every_kept_copy_of_it_is_masked() + { + // Tier: high — the real executor, runner, grader, capture, checkpointer and artifact store, and the real Claude + // adapter's spec (restore file, --resume, stdin goal); only the executable is a shell. The goal carries a workflow + // secret, sealed out of task_jsonb, which round 0 writes into its session as its user turn, as the CLI does. A WARM + // revision continues that conversation in this process, and its goal says only "continue" — so the conversation + // is the one place the agent still sees the secret, and it must be handed it exactly. Every copy that is KEPT — + // result_jsonb, the resume payload, each round's checkpoints — holds it masked. + // MUTATIONS this pins: restore the masked copy into the warm round → the revision cannot read its token, exits 4, + // and the run fails; drop the redactor from the revise round's capture → result_jsonb carries the secret. + if (OperatingSystem.IsWindows()) return; + + var secret = $"CANARY-G2-revise-{Guid.NewGuid():N}"; + var (teamId, userId) = await SeedTeamAsync(); + using var remote = new BareRemote(); + await remote.SeedBaseAsync(CheckScript); + var repoId = await SeedBoundRepositoryAsync(teamId, remote.Url); + var task = TaskWith(repoId) with { Goal = $"Configure the deploy token {secret} and make feature.txt say the right thing", MaxReviseRounds = 1, WorkflowSecrets = [secret], CheckpointSessionTranscript = true }; + var runId = await CreateRunAsync(teamId, userId, task); + var harness = new SecretSessionHarness(); + + await ExecuteAsync(runId, harness, clock: new CheckpointEveryTickClock()); + + var (run, result) = await LoadAsync(runId); + run.Status.ShouldBe(AgentRunStatus.Succeeded, $"the warm revision must read its token back from the conversation it continues: {run.Error}"); + result.ReviseRounds.ShouldBe(1); + harness.Built.Count.ShouldBe(2, "fixture check: round 0 and one revision"); + harness.Built[1].Spec.Args.ShouldContain("--resume", customMessage: "fixture check: the revision continued the conversation"); + harness.Built[1].Spec.ConfigHomeFiles.ShouldContain(file => file.Content.Contains(secret, StringComparison.Ordinal), "the warm round restores the conversation exactly as the agent wrote it"); + run.TaskJson!.ShouldNotContain(secret, Case.Sensitive, "the exact conversation stays in the process: the persisted envelope never carries it"); + + AssertMaskedSession("result_jsonb.sessionTranscript", result.SessionTranscript, secret); + result.SessionTranscript.ShouldContain($"{SecretSessionHarness.RevisedLine} {SecretRedactor.Placeholder}", Case.Sensitive, "the kept session is the revision's, with the token it wrote again masked"); + WorkflowResumeAgentRunCompletionNotifier.BuildResumePayload(run).ShouldNotContain(secret, Case.Sensitive, "nor does the resume payload a retry is handed carry it"); + + var checkpoints = await SessionKeepingCli.KeptCheckpointsAsync(_fixture, teamId, runId, landed: text => text.Contains(SecretSessionHarness.RevisedLine, StringComparison.Ordinal)); + checkpoints.ShouldContain(text => text.Contains(SecretSessionHarness.RevisedLine, StringComparison.Ordinal), "fixture check: the revision's own session was checkpointed, or this proves nothing about its checkpoints"); + + foreach (var checkpoint in checkpoints) AssertMaskedSession("a checkpoint artifact", checkpoint, secret); + } + [Fact] public async Task A_cold_revision_refused_for_spend_leaves_no_note_that_it_continued() { @@ -709,7 +753,7 @@ private async Task SeedBoundRepositoryAsync(Guid teamId, string cloneUrlHt // ─── Execution ─────────────────────────────────────────────────────────── - private async Task ExecuteAsync(Guid runId, IAgentHarness harness, ISandboxRunner? runner = null) + private async Task ExecuteAsync(Guid runId, IAgentHarness harness, ISandboxRunner? runner = null, TimeProvider? clock = null) { using var scope = _fixture.BeginScope(); var executor = new AgentRunExecutor( @@ -728,7 +772,10 @@ private async Task ExecuteAsync(Guid runId, IAgentHarness harness, ISandboxRunne scope.Resolve(), scope.Resolve(), scope.Resolve(), scope.Resolve(), scope.Resolve>(), - NullLogger.Instance); + NullLogger.Instance, + sessionCheckpointer: scope.Resolve(), + clock: clock, + sealer: scope.Resolve()); await executor.ExecuteAsync(runId, CancellationToken.None); } @@ -748,6 +795,15 @@ private async Task> LoadEventsAsync(Guid runId) return events.Select(e => e.Text).ToList(); } + /// A kept copy of the session: the user turn the agent wrote is there with its secret masked, the secret is nowhere, and every line still parses for --resume. + private static void AssertMaskedSession(string where, string? session, string secret) + { + session.ShouldNotBeNullOrEmpty($"{where} must hold the captured session — an empty one would pass every check below"); + session!.ShouldNotContain(secret, Case.Sensitive, $"{where} carries the workflow secret"); + session.ShouldContain($"deploy token {SecretRedactor.Placeholder} and make", Case.Sensitive, $"{where}: the user turn is kept, its secret masked"); + SessionKeepingCli.ParseLines(session); + } + // ─── Git helpers ───────────────────────────────────────────────────────── private static async Task GitAvailableAsync() @@ -856,6 +912,69 @@ public SandboxSpec BuildInvocation(AgentTask task) public string? SessionTranscriptRelativePath(string configHome, string? workspaceDirectory, string? sessionId) => _real.SessionTranscriptRelativePath(configHome, workspaceDirectory, sessionId); } + /// + /// A "scripted" harness whose spec, stream parse, fold, run-fact keys and transcript location are the REAL Claude + /// adapter's; only the executable is a shell. Round 0 writes its stdin — the user turn, which carries the goal — into + /// its session as the CLI does, and drafts work the check fails. A revision reads the deploy token back out of the + /// conversation it was restored, refuses to go on when that conversation holds the mask instead, writes the token + /// into its session again and writes the fixed work. Each round names its session early and stays up long enough + /// for the drain tick to checkpoint it. + /// + private sealed class SecretSessionHarness : IAgentHarness, IAgentHarnessRunFactKeys, IAgentSessionTranscript + { + /// What a revision writes into its session ahead of the token it read back. + public const string RevisedLine = "deploying with"; + + private const string SessionId = "sess-secret-revise"; + + /// Names the session early, so the drain tick can address it while the round is still running. + private const string NamesTheSession = $$"""printf '%s\n' '{"type":"system","subtype":"init","session_id":"{{SessionId}}"}'"""; + + private const string Finishes = $$"""printf '%s\n' '{"type":"result","subtype":"success","is_error":false,"result":"done","session_id":"{{SessionId}}"}'"""; + private readonly ClaudeCodeHarness _real = new(); + + public string Kind => "scripted"; + public string Version => "test"; + public IReadOnlyList Models { get; } = new[] { "test-model" }; + public AgentRunFactKeys RunFactKeys => _real.RunFactKeys; + + public List<(AgentTask Task, SandboxSpec Spec)> Built { get; } = new(); + + public SandboxSpec BuildInvocation(AgentTask task) + { + var spec = _real.BuildInvocation(task); + Built.Add((task, spec)); + + var session = $"\"$CLAUDE_CONFIG_DIR/{ClaudeTranscriptPath.For(task.WorkspaceDirectory!, SessionId)}\""; + var work = task.Goal.StartsWith(AgentRunExecutor.ReviseInstructionPrefix, StringComparison.Ordinal) ? Revise(session) : Draft(session); + + return spec with { Command = "/bin/sh", Args = new[] { "-c", $"{work} && {NamesTheSession} && sleep 2 && {Finishes}" } }; + } + + /// Keep the user turn — the goal, with the token in it — as the session's first line, and draft failing work. + private static string Draft(string session) => + $$"""mkdir -p "$(dirname {{session}})" && cat >> {{session}} && printf 'draft\n' > feature.txt"""; + + /// Read the token back out of the restored conversation, refuse the mask, write it into the session again, and fix the work. + private static string Revise(string session) => + $$"""cat >/dev/null && token=$(sed -n 's/.*deploy token \([^ ]*\) and make.*/\1/p' {{session}} | head -n 1) && case "$token" in ''|'***') echo "the restored conversation does not hold the deploy token: $token" >&2; exit 4;; esac && printf '{"type":"assistant","text":"{{RevisedLine}} %s"}\n' "$token" >> {{session}} && printf 'revised clean\n' > feature.txt"""; + + + public IReadOnlyList ParseEvents(string rawLine) => _real.ParseEvents(rawLine); + + public IAgentEventFolder CreateFolder() => _real.CreateFolder(); + + public string? SessionTranscriptRelativePath(string configHome, string? workspaceDirectory, string? sessionId) => _real.SessionTranscriptRelativePath(configHome, workspaceDirectory, sessionId); + } + + /// A clock on which the session-checkpoint cadence has always elapsed — each read is one interval past the last — so a revision's session is checkpointed within its seconds rather than a minute after round 0's. Timers and timestamps stay the system's. + private sealed class CheckpointEveryTickClock : TimeProvider + { + private long _reads; + + public override DateTimeOffset GetUtcNow() => base.GetUtcNow() + Interlocked.Increment(ref _reads) * AgentRunExecutor.SessionCheckpointInterval; + } + /// A "scripted" harness whose spec is the REAL Codex adapter's — its own input cap included — with only the executable swapped; round 0 writes draft work, a revision the fixed work. private sealed class CodexSpecHarness : IAgentHarness { diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/SupervisorGradeEvidenceMaskingFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/SupervisorGradeEvidenceMaskingFlowTests.cs new file mode 100644 index 000000000..319610021 --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/SupervisorGradeEvidenceMaskingFlowTests.cs @@ -0,0 +1,274 @@ +using System.Text; +using System.Text.Json; +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Agents; +using CodeSpace.Core.Services.Agents.Sandbox; +using CodeSpace.Core.Services.Agents.Sandbox.Runners; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Core.Services.Supervisor; +using CodeSpace.Core.Services.Workflows.Artifacts; +using CodeSpace.Core.Services.Workflows.Lifecycle; +using CodeSpace.Core.Services.Workflows.Llm; +using CodeSpace.Core.Services.Workflows.Runtime; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.IntegrationTests.Workflows.Infrastructure; +using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Dtos.Agents; +using CodeSpace.Messages.Enums; +using Microsoft.EntityFrameworkCore; +using Shouldly; + +namespace CodeSpace.IntegrationTests.Workflows; + +/// +/// The supervisor lane's half of "a grade keeps no workflow secret", end to end: the REAL +/// folding a unit's acceptance with the REAL , which clones a real git remote, +/// runs the check for real, and stores its evidence in the real Postgres artifact store. +/// +/// The unit's sealed task handed it a deploy key, which it committed into its branch. The model authored an +/// acceptance check that both prints that file and carries a second value the supervisor node's scope resolved. The +/// check runs with the exact values; the evidence artifact (argv line and output, which a Viewer downloads), the tail +/// on the decision tape and the baseline's evidence must hold only the mask. The supervisor lane used to hand the grader +/// no secrets at all, so both values were stored verbatim. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "Integration")] +public sealed class SupervisorGradeEvidenceMaskingFlowTests(PostgresFixture fixture) +{ + private const string NodeId = "sup"; + private const string Goal = "ship the feature"; + + [Fact] + public async Task A_units_grade_keeps_neither_its_sealed_secret_nor_the_scope_secret_its_check_carries() + { + if (OperatingSystem.IsWindows() || !await GitAvailableAsync()) return; + + // MUTATION this pins: drop WorkflowSecrets from the unit lane's request (or stop opening the unit's sealed task, + // or stop reading the spec's scope secrets) → the evidence artifact stores a canary → red. + var unitSecret = $"CANARY-G2-unit-{Guid.NewGuid():N}"; + var specSecret = $"CANARY-G2-scope-{Guid.NewGuid():N}"; + var (teamId, userId) = await WorkflowsTestSeed.SeedTeamAsync(fixture); + var runId = await SeedSupervisorRunAsync(teamId, userId); + + using var remote = new BareRemote(); + await remote.SeedBaseAsync(new() { ["README.md"] = "base\n" }); + var baseSha = await remote.HeadShaAsync(); + await remote.CommitOnBranchAsync("candidate", new() { ["deploy.env"] = $"DEPLOY_KEY={unitSecret}\n" }); + + var repoId = await SeedBoundRepositoryAsync(teamId, remote.Url); + var agentId = Guid.NewGuid(); + await SeedSealedUnitAsync(teamId, agentId, unitSecret); + + var check = new[] { "sh", "-c", $"cat deploy.env; echo token={specSecret}; exit 1" }; + await SeedDecisionAsync(runId, teamId, 1, SupervisorDecisionKinds.Plan, JsonSerializer.Serialize(new { goal = Goal, subtasks = new[] { new { id = "s1", title = "s1", instruction = "do s1", acceptance = new { command = check } } } }, AgentJson.Options), "{}"); + await SeedDecisionAsync(runId, teamId, 2, SupervisorDecisionKinds.Spawn, """{"subtaskIds":["s1"]}""", JsonSerializer.Serialize(new { agentRunIds = new[] { agentId }, agentCount = 1, agentResults = new[] { new SupervisorAgentResult { AgentRunId = agentId, Status = "Succeeded", Summary = "did it", ProducedBranch = "candidate" } } }, AgentJson.Options)); + await SeedManifestAsync(teamId, agentId, repoId, "candidate", baseSha); + + SupervisorTurnContext context; + using (var scope = fixture.BeginScope()) + using (LlmCallContext.Push(NodeScope(scope, runId, teamId, specSecret))) + context = await scope.Resolve().RehydrateFromDecisionLogAsync(runId, teamId, NodeId, Goal, new SupervisorGoalConfig { Goal = Goal, AgentProfile = new SupervisorAgentProfile { RepositoryId = repoId } }, CancellationToken.None); + + var unit = SupervisorOutcome.ReadAgentResults(context.PriorDecisions.Single(d => d.DecisionKind == SupervisorDecisionKinds.Spawn).OutcomeJson).Single(); + unit.AcceptancePassed.ShouldBe(false, $"fixture check: the real check ran against the real clone and failed as written — detail '{unit.AcceptanceDetail}'"); + + var tape = await LedgerSpawnOutcomeAsync(runId, teamId); + tape.ShouldNotContain(unitSecret, Case.Sensitive, "the decision tape keeps the evidence tail; it must not keep the unit's secret"); + tape.ShouldNotContain(specSecret, Case.Sensitive, "nor the scope secret the check carried"); + + var evidence = await TeamArtifactTextsAsync(teamId); + evidence.Count(text => text.Contains("$ sh -c", StringComparison.Ordinal)).ShouldBeGreaterThanOrEqualTo(2, "fixture check: the candidate grade and its baseline both stored evidence"); + evidence.ShouldAllBe(text => !text.Contains(unitSecret) && !text.Contains(specSecret), "no stored evidence may carry either secret"); + evidence.ShouldContain(text => text.Contains($"DEPLOY_KEY={PersistenceSecretRedactor.Marker}") && text.Contains($"token={PersistenceSecretRedactor.Marker}"), + "the candidate's evidence printed both values — masked, which also proves they were really there"); + } + + // ── chassis ────────────────────────────────────────────────────────────────── + + /// The ambient call scope the engine pushes for the supervisor node; its capture redactor holds the node scope's secrets — here one the check carries and one nothing carries. + private static LlmCallScope NodeScope(ILifetimeScope scope, Guid runId, Guid teamId, string specSecret) => + new(runId, teamId, NodeId, "", "agent.supervisor", scope.Resolve(), scope.Resolve(), CaptureRedactor: new PersistenceSecretRedactor([specSecret, $"CANARY-G2-unused-{Guid.NewGuid():N}"])); + + /// The unit as the agent-run store persists a task that embeds a workflow secret: masked task_jsonb, the exact copy sealed beside it. + private async Task SeedSealedUnitAsync(Guid teamId, Guid agentRunId, string unitSecret) + { + using var scope = fixture.BeginScope(); + var db = scope.Resolve(); + var task = new AgentTask { Goal = $"do s1; deploy key {unitSecret}", Harness = "codex-cli", WorkflowSecrets = [unitSecret] }; + var stored = AgentTaskSecrets.Seal(task, scope.Resolve()); + + stored.Ciphertext.ShouldNotBeNull("fixture check: the unit's task is sealed, so its secret is reachable only by opening it"); + db.AgentRun.Add(new AgentRun { Id = agentRunId, TeamId = teamId, Harness = task.Harness, Status = AgentRunStatus.Succeeded, TaskJson = stored.PublicJson, SealedTaskCiphertext = stored.Ciphertext }); + await db.SaveChangesAsync(); + } + + private async Task> TeamArtifactTextsAsync(Guid teamId) + { + using var scope = fixture.BeginScope(); + var ids = await scope.Resolve().WorkflowArtifact.AsNoTracking().Where(a => a.TeamId == teamId).Select(a => a.Id).ToListAsync(); + var store = scope.Resolve(); + var texts = new List(); + + foreach (var id in ids) + if (await store.GetBytesAsync(teamId, id, CancellationToken.None) is { } bytes) texts.Add(Encoding.UTF8.GetString(bytes.Bytes)); + + return texts; + } + + private async Task SeedDecisionAsync(Guid runId, Guid teamId, int sequence, string kind, string payloadJson, string outcomeJson) + { + using var scope = fixture.BeginScope(); + var db = scope.Resolve(); + var now = DateTimeOffset.UtcNow; + + db.SupervisorDecisionRecord.Add(new SupervisorDecisionRecord + { + Id = Guid.NewGuid(), TeamId = teamId, SupervisorRunId = runId, Sequence = sequence, + DecisionKind = kind, IdempotencyKey = $"{kind}-{Guid.NewGuid():N}", InputHash = "test", + Status = SupervisorDecisionStatus.Succeeded, PayloadJson = payloadJson, OutcomeJson = outcomeJson, + FenceEpoch = 1, CreatedDate = now, CreatedBy = Guid.Empty, LastModifiedDate = now, LastModifiedBy = Guid.Empty, + }); + await db.SaveChangesAsync(); + } + + private async Task LedgerSpawnOutcomeAsync(Guid runId, Guid teamId) + { + using var scope = fixture.BeginScope(); + + return (await scope.Resolve().SupervisorDecisionRecord.AsNoTracking() + .Where(d => d.SupervisorRunId == runId && d.TeamId == teamId && d.DecisionKind == SupervisorDecisionKinds.Spawn) + .Select(d => d.OutcomeJson).SingleAsync()) ?? ""; + } + + private async Task SeedManifestAsync(Guid teamId, Guid agentRunId, Guid repositoryId, string branch, string baseSha) + { + using var scope = fixture.BeginScope(); + var db = scope.Resolve(); + + db.PublishManifest.Add(new PublishManifest + { + Id = Guid.NewGuid(), TeamId = teamId, Kind = PublishManifestKind.Agent, AgentRunId = agentRunId, RepositoryId = repositoryId, + RepositoryAlias = "primary", Branch = branch, BaseSha = baseSha, PublishStateValue = PublishState.Pushed, + }); + await db.SaveChangesAsync(); + } + + private async Task SeedBoundRepositoryAsync(Guid teamId, string cloneUrl) + { + using var scope = fixture.BeginScope(); + var db = scope.Resolve(); + var instanceId = Guid.NewGuid(); + var credentialId = Guid.NewGuid(); + var repoId = Guid.NewGuid(); + + db.ProviderInstance.Add(new ProviderInstance { Id = instanceId, TeamId = teamId, Provider = ProviderKind.GitHub, DisplayName = "local", BaseUrl = "https://local" }); + db.Credential.Add(new Credential + { + Id = credentialId, TeamId = teamId, ProviderInstanceId = instanceId, AuthType = AuthType.Pat, DisplayName = "clone cred", + EncryptedPayload = scope.Resolve().Encrypt(scope.Resolve().Serialize(new PatPayload { Token = "grade-mask-token" })), + Status = CredentialStatus.Active, + }); + db.Repository.Add(new Repository + { + Id = repoId, TeamId = teamId, ProviderInstanceId = instanceId, CredentialId = credentialId, ExternalId = repoId.ToString(), NamespacePath = "org", Name = "repo", + FullPath = "org/repo", DefaultBranch = "main", CloneUrlHttps = cloneUrl, WebUrl = "https://local/org/repo", + }); + await db.SaveChangesAsync(); + + return repoId; + } + + private async Task SeedSupervisorRunAsync(Guid teamId, Guid userId) + { + using var scope = fixture.BeginScopeAs(userId, teamId, Messages.Constants.Roles.Admin); + var workflowId = await scope.Resolve().Send(new Messages.Commands.Workflows.CreateWorkflowCommand + { + Name = "sup-grade-mask-" + Guid.NewGuid().ToString("N")[..6], + Definition = new Messages.Dtos.Workflows.WorkflowDefinition + { + SchemaVersion = 1, + Nodes = + [ + new() { Id = "start", TypeKey = "trigger.manual", Config = WorkflowsTestSeed.EmptyJson(), Inputs = WorkflowsTestSeed.EmptyJson() }, + new() { Id = NodeId, TypeKey = "agent.supervisor", Config = WorkflowsTestSeed.Json("""{"goal":"ship the feature"}"""), Inputs = WorkflowsTestSeed.EmptyJson() }, + new() { Id = "end", TypeKey = "builtin.terminal", Config = WorkflowsTestSeed.EmptyJson(), Inputs = WorkflowsTestSeed.EmptyJson() }, + ], + Edges = [new() { From = "start", To = NodeId }, new() { From = NodeId, To = "end" }], + }, + Activations = [], + Enabled = true, + }); + + return await WorkflowsTestSeed.SeedManualRunAsync(fixture, workflowId, teamId); + } + + private static async Task GitAvailableAsync() + { + try { return (await new LocalProcessRunner().RunAsync(new SandboxSpec { Command = "git", Args = ["--version"], TimeoutSeconds = 10 }, CancellationToken.None)).Status == SandboxStatus.Success; } + catch { return false; } + } + + /// A bare git remote on local disk with a base commit and a candidate branch — what the grader clones. + private sealed class BareRemote : IDisposable + { + private readonly string _root = Path.Combine(Path.GetTempPath(), "cs-grade-mask-" + Guid.NewGuid().ToString("N")); + private readonly string _bare; + private readonly string _seed; + + public BareRemote() + { + Directory.CreateDirectory(_root); + _bare = Path.Combine(_root, "remote.git"); + _seed = Path.Combine(_root, "seed"); + } + + public string Url => new Uri(_bare).AbsoluteUri; + + public async Task SeedBaseAsync(Dictionary files) + { + await Git(_root, "init", "--bare", "-b", "main", _bare); + Directory.CreateDirectory(_seed); + await Git(_seed, "clone", _bare, _seed); + await Git(_seed, "config", "user.email", "test@codespace.dev"); + await Git(_seed, "config", "user.name", "Test"); + await Git(_seed, "config", "commit.gpgsign", "false"); + await CommitAsync(files, "seed"); + await Git(_seed, "push", "origin", "main"); + } + + public async Task HeadShaAsync() => (await Git(_seed, "rev-parse", "HEAD")).Trim(); + + public async Task CommitOnBranchAsync(string branch, Dictionary files) + { + await Git(_seed, "checkout", "-b", branch); + await CommitAsync(files, "candidate work"); + await Git(_seed, "push", "origin", branch); + } + + private async Task CommitAsync(Dictionary files, string message) + { + foreach (var (name, content) in files) await File.WriteAllTextAsync(Path.Combine(_seed, name), content); + + await Git(_seed, "add", "-A"); + await Git(_seed, "commit", "-m", message); + } + + private static async Task Git(string workdir, params string[] args) + { + var result = await new LocalProcessRunner().RunAsync(new SandboxSpec { Command = "git", Args = args, WorkingDirectory = workdir, TimeoutSeconds = 60 }, CancellationToken.None); + + if (result.Status != SandboxStatus.Success || result.ExitCode != 0) + throw new InvalidOperationException($"git {string.Join(' ', args)} failed (exit {result.ExitCode}): {result.Stderr}"); + + return result.Stdout; + } + + public void Dispose() + { + try { Directory.Delete(_root, recursive: true); } catch { /* best-effort */ } + } + } +} diff --git a/backend/tests/CodeSpace.SandboxTests/GoalChannelE2ETests.cs b/backend/tests/CodeSpace.SandboxTests/GoalChannelE2ETests.cs index 12d1a75f1..66b50ed3b 100644 --- a/backend/tests/CodeSpace.SandboxTests/GoalChannelE2ETests.cs +++ b/backend/tests/CodeSpace.SandboxTests/GoalChannelE2ETests.cs @@ -58,6 +58,9 @@ public sealed partial class GoalChannelE2ETests(ITestOutputHelper output) : IDis [InlineData("/security-review")] public Task A_claude_goal_that_opens_with_a_slash_word_reaches_the_model_as_text(string word) => SlashWordReachesTheModelAsync(word, AgentAutonomyLevel.Confined, lane: "root"); + [Fact] + public Task A_claude_session_masked_as_the_executor_keeps_it_still_resumes() => MaskedSessionResumesAsync(AgentAutonomyLevel.Confined, lane: "root"); + /// /// The mention arm, for either lane: a goal naming a secret in every place a mention reaches — the config home by /// ~ and by its absolute path, a directory outside the workspace, a workspace symlink out of it, a workspace @@ -117,6 +120,47 @@ internal async Task ResumedPromptReadsNothingAsync(AgentAutonomyLevel tier, stri output.WriteLine($"{RanMarker} {LanePrefix(lane)}resume claude-code {tier} uid={NonRootWorker.EffectiveUid()} confined={Confined} controlRead={expected.Count}/{expected.Count}"); } + /// + /// The MASKED-RESUME arm: a session the executor keeps is masked with the run's secrets + /// () before a later attempt — a respawn, a checkpoint resume — restores it, + /// so the pinned CLI must still load the masked file and continue. The first run's goal carries a canary, which the + /// CLI writes into its session as the user's turn: that restored turn reaches the model masked. The continuation's + /// own goal carries the canary again, as a respawn re-sends its sealed goal, so the agent still has the value it was + /// given. (A warm revise round in the same process is handed the conversation unmasked, and never stores it.) + /// + internal async Task MaskedSessionResumesAsync(AgentAutonomyLevel tier, string lane) + { + if (!ReviewerReadsItsDiffE2ETests.Armed(ClaudeCodeHarness.HarnessKind) || OperatingSystem.IsWindows()) return; + + await ReviewerReadsItsDiffE2ETests.RequirePinnedBinaryAsync(Harness, ClaudeCodeHarness.HarnessKind); + + var fixture = PlantFixture(); + var canary = $"CANARY-masked-{fixture.Nonce}"; + var first = await RunAsync(fixture, tier, _ => $"Say hello. GOAL-first-{fixture.Nonce} deploy key {canary}", Structured); + var session = SessionOf(first, fixture); + + session.Transcript.ShouldContain(canary, Case.Sensitive, $"fixture check: the CLI must have written the goal into its session, or masking it proves nothing. {Diagnosis(first)}"); + + var masked = session with { Transcript = SessionTranscriptRedaction.Redact(session.Transcript, new SecretRedactor([canary])) }; + masked.Transcript.ShouldNotContain(canary); + + var run = await RunAsync(fixture, tier, _ => $"Say hello again. GOAL-again-{fixture.Nonce} deploy key {canary}", Structured, masked); + + run.Spec.Args.ShouldContain("--resume", "fixture check: the continuation must ride --resume"); + (string.Join('\n', run.Lines) + run.Result.Stderr).ShouldNotContain("No conversation found", Case.Insensitive, $"the CLI must load the masked session. {Diagnosis(run)}"); + + var requests = MainRequests(run); + requests.ShouldNotBeEmpty($"the model must have been asked. {Diagnosis(run)}"); + var request = requests[0].GetRawText(); + request.ShouldContain($"GOAL-first-{fixture.Nonce}", Case.Sensitive, $"the continuation must carry the restored conversation, or it started cold. {Diagnosis(run)}"); + request.ShouldContain($"GOAL-first-{fixture.Nonce} deploy key {SecretRedactor.Placeholder}", Case.Sensitive, $"the restored turn reaches the model masked. {Diagnosis(run)}"); + request.ShouldNotContain($"GOAL-first-{fixture.Nonce} deploy key {canary}", Case.Sensitive, $"the masked value never reaches the model from the restored turn. {Diagnosis(run)}"); + request.ShouldContain($"GOAL-again-{fixture.Nonce} deploy key {canary}", Case.Sensitive, $"the goal the continuation re-sends still hands the agent the value. {Diagnosis(run)}"); + AssertTheModelAnsweredIt(run, fixture); + + output.WriteLine($"{RanMarker} {LanePrefix(lane)}masked-resume claude-code {tier} uid={NonRootWorker.EffectiveUid()} confined={Confined}"); + } + /// The session a finished run left: its id off the result line, and its transcript read where the executor captures it (). private static Session SessionOf(GoalRun run, Fixture fixture) { diff --git a/backend/tests/CodeSpace.SandboxTests/NonRootWorkerE2ETests.cs b/backend/tests/CodeSpace.SandboxTests/NonRootWorkerE2ETests.cs index b23d47f66..4568e1c15 100644 --- a/backend/tests/CodeSpace.SandboxTests/NonRootWorkerE2ETests.cs +++ b/backend/tests/CodeSpace.SandboxTests/NonRootWorkerE2ETests.cs @@ -187,6 +187,16 @@ public async Task A_standard_claude_goal_that_opens_with_a_slash_word_reaches_th await arms.SlashWordReachesTheModelAsync(word, AgentAutonomyLevel.Standard, Lane); } + [Fact] + public async Task A_confined_run_spools_owner_only_as_the_worker_uid_and_still_uses_its_config_home() + { + // The spool and its raw logs are owner-only now: the shipped uid must still bind its config home out of it under + // bubblewrap, write there, and read the run back. + if (!NonRootWorker.Require()) return; + + await new SpoolPermissionsE2ETests(output).OwnerOnlySpoolAsync(Lane); + } + [Fact] public async Task A_standard_claude_agent_that_plants_push_vectors_in_its_git_cannot_reach_the_sink_when_the_platform_publishes() { diff --git a/backend/tests/CodeSpace.SandboxTests/SpoolPermissionsE2ETests.cs b/backend/tests/CodeSpace.SandboxTests/SpoolPermissionsE2ETests.cs new file mode 100644 index 000000000..290cb23c7 --- /dev/null +++ b/backend/tests/CodeSpace.SandboxTests/SpoolPermissionsE2ETests.cs @@ -0,0 +1,78 @@ +using CodeSpace.Core.Services.Agents.Sandbox.Isolation; +using CodeSpace.Core.Services.Agents.Sandbox.Runners; +using CodeSpace.Messages.Agents; +using Shouldly; +using Xunit.Abstractions; + +namespace CodeSpace.SandboxTests; + +/// +/// 🟢 Sandbox E2E (high fidelity, Rule 12): the per-run spool is owner-only on the real kernel, under real bubblewrap, +/// and a confined run still works inside it. The spool holds out.log/err.log — the agent's output before +/// redaction — and sat at the worker's umask (a 0755 directory around a 0644 log) beside the 0700 config home. It is +/// now created 0700 at launch and its logs 0600. What has to stay true under that: bubblewrap still binds the config +/// home out of the 0700 spool, the agent still writes there, and the worker still reads the run back. +/// +/// The root lane runs the arm here; the non-root lane runs the same arm as the shipped worker's uid through +/// . Each arm that ran prints , which the lanes require. +/// +[Trait("Category", "Sandbox")] +public sealed class SpoolPermissionsE2ETests(ITestOutputHelper output) +{ + /// Printed by every arm that actually ran. + public const string RanMarker = "[spool-perms-e2e] ran"; + + private const UnixFileMode OwnerOnlyDirectory = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute; + private const UnixFileMode OwnerOnlyFile = UnixFileMode.UserRead | UnixFileMode.UserWrite; + + [Fact] + public async Task A_confined_run_spools_owner_only_and_still_uses_its_config_home() + { + if (!OperatingSystem.IsLinux() || BubblewrapSandbox.Available is null && !BubblewrapSandbox.IsRequired) return; + + await OwnerOnlySpoolAsync("root"); + } + + /// The arm, for either lane: a confined durable run that writes to stdout, stderr and its config home. + internal async Task OwnerOnlySpoolAsync(string lane) + { + if (!OperatingSystem.IsLinux()) return; // bubblewrap is Linux-only (Rule 12.1); each caller already gates on its lane's posture + + BubblewrapSandbox.Available.ShouldNotBeNull($"this arm asserts the spool under REAL confinement ({BubblewrapSandbox.UnavailableReason})"); + + var workspace = Directory.CreateTempSubdirectory("cs-spool-perms-").FullName; + var key = Guid.NewGuid().ToString("N"); + var runner = new LocalProcessRunner(); + + try + { + var spec = new SandboxSpec + { + Command = "/bin/sh", Args = ["-c", "printf 'raw out\\n'; printf 'raw err\\n' >&2; touch \"$CS_HOME/probe\" && printf 'home-ok\\n'"], + WorkingDirectory = workspace, TimeoutSeconds = 60, ConfigHomeEnvVars = ["CS_HOME"], + }; + + var handle = await runner.LaunchAsync(spec, key, CancellationToken.None); + var lines = new List(); + var result = await runner.AttachAsync(handle, (frame, _) => { lines.Add(frame.Text.Trim()); return Task.CompletedTask; }, CancellationToken.None); + + handle.Confinement.ShouldNotBeNull("fixture check: the run was confined by bubblewrap, or this proves nothing about binding out of a 0700 spool"); + result.Status.ShouldBe(SandboxStatus.Success, $"the confined run must still complete: stderr '{result.Stderr}'"); + lines.ShouldContain("raw out", "the worker still reads the run's stdout back out of its owner-only spool"); + lines.ShouldContain("home-ok", "bubblewrap still binds the config home out of the 0700 spool, and the agent writes there"); + File.Exists(Path.Combine(LocalProcessRunner.ConfigHomePath(handle.SpoolDirectory), "probe")).ShouldBeTrue("the agent's config-home write landed in the bound directory"); + + File.GetUnixFileMode(handle.SpoolDirectory).ShouldBe(OwnerOnlyDirectory, $"the spool {handle.SpoolDirectory} must be owner-only — check `stat -c %a` on it"); + File.GetUnixFileMode(Path.Combine(handle.SpoolDirectory, "out.log")).ShouldBe(OwnerOnlyFile, "out.log is the agent's raw stdout"); + File.GetUnixFileMode(Path.Combine(handle.SpoolDirectory, "err.log")).ShouldBe(OwnerOnlyFile, "err.log is the agent's raw stderr"); + File.GetUnixFileMode(LocalProcessRunner.ConfigHomePath(handle.SpoolDirectory)).ShouldBe(OwnerOnlyDirectory, "the config home stays owner-only"); + + output.WriteLine($"{RanMarker} {(lane == "root" ? "" : lane + " ")}uid={NonRootWorker.EffectiveUid()} confined=True"); + } + finally + { + try { Directory.Delete(LocalProcessRunner.SpoolDirectoryFor(key), recursive: true); } catch { /* best-effort */ } + try { Directory.Delete(workspace, recursive: true); } catch { /* best-effort */ } + } + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Agents/AgentSessionTranscriptCheckpointerTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/AgentSessionTranscriptCheckpointerTests.cs index 1fa34dcaa..89261de7e 100644 --- a/backend/tests/CodeSpace.UnitTests/Agents/AgentSessionTranscriptCheckpointerTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Agents/AgentSessionTranscriptCheckpointerTests.cs @@ -1,4 +1,5 @@ using System.Text; +using System.Text.Json; using CodeSpace.Core.Persistence.Entities; using CodeSpace.Core.Services.Agents; using CodeSpace.Core.Services.Agents.Recovery; @@ -277,11 +278,30 @@ public async Task A_stamp_that_loses_its_fence_takes_no_checkpoint() (await checkpointer.CheckpointAsync(Request(path), CancellationToken.None)).ShouldNotBeNull("the retry is immediate — a refused stamp must not advance the growth watermark it never earned"); } + [Fact] + public async Task A_checkpoint_is_masked_with_the_runs_secrets_and_still_resumes() + { + // SX-13: the live session file is the agent's own, so a key it printed or a workflow secret its goal carried + // is in it verbatim — and a checkpoint is a team artifact the next attempt restores for --resume. + // MUTATION this pins: store buffer[..complete] unmasked (drop Masked) → the canary is in the stored bytes → red. + const string key = "CANARY-G2-checkpoint-sk-ant-5d6e7f80"; + var whole = "{\"type\":\"init\",\"session_id\":\"s-1\"}\n" + JsonSerializer.Serialize(new { type = "user", tool_result = $"key={key}" }) + "\n"; + var path = WriteTranscript(whole + "{\"type\":\"assis"); + + var taken = (await NewCheckpointer().CheckpointAsync(Request(path, redactor: new SecretRedactor([key])), CancellationToken.None)).ShouldNotBeNull(); + + var stored = Encoding.UTF8.GetString(_store.Writes.Single().Bytes.ToArray()); + stored.ShouldNotContain(key, Case.Sensitive, "the stored checkpoint must not carry the secret the agent echoed into its session"); + foreach (var line in stored.Split('\n', StringSplitOptions.RemoveEmptyEntries)) JsonDocument.Parse(line).Dispose(); // every restored line must still parse + stored.ShouldEndWith("\n", Case.Sensitive, "still only the complete lines"); + taken.Bytes.ShouldBe(Encoding.UTF8.GetByteCount(whole), "the watermark is the FILE's complete prefix — what the next tick measures the file against — not the masked copy's length"); + } + private ArtifactSessionTranscriptCheckpointer NewCheckpointer() => new(_store, _runs, _clock, NullLogger.Instance); - private static SessionTranscriptCheckpointRequest Request(string path, long? previousBytes = null) => - new(Guid.NewGuid(), new AgentRunOwnerToken(Guid.NewGuid(), Guid.NewGuid(), 7), path, "s-1", previousBytes); + private static SessionTranscriptCheckpointRequest Request(string path, long? previousBytes = null, SecretRedactor? redactor = null) => + new(Guid.NewGuid(), new AgentRunOwnerToken(Guid.NewGuid(), Guid.NewGuid(), 7), path, "s-1", previousBytes) { Redactor = redactor ?? SecretRedactor.None }; private string WriteTranscript(string content, string name = "session.jsonl") => WriteTranscript(Encoding.UTF8.GetBytes(content), name); diff --git a/backend/tests/CodeSpace.UnitTests/Agents/SessionTranscriptRedactionTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/SessionTranscriptRedactionTests.cs new file mode 100644 index 000000000..518e46097 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Agents/SessionTranscriptRedactionTests.cs @@ -0,0 +1,173 @@ +using System.Text.Json; +using CodeSpace.Core.Services.Agents; +using Shouldly; + +namespace CodeSpace.UnitTests.Agents; + +/// +/// The masking a CLI session transcript gets before it is stored (the run's result, its mid-run checkpoints, the resume +/// payload a retry is handed). The transcript is newline-delimited JSON that a harness restores VERBATIM for +/// --resume, so the mask has two jobs and both are pinned here: no secret survives, and every line that parsed +/// before still parses — a transcript the CLI cannot read costs the continuation it exists for. +/// +[Trait("Category", "Unit")] +public sealed class SessionTranscriptRedactionTests +{ + private const string Key = "CANARY-G2-sk-ant-transcript-7f3e9a1c"; + + [Fact] + public void A_masked_transcript_is_still_valid_jsonl_with_no_secret_left() + { + var transcript = string.Join('\n', + Line(new { type = "system", subtype = "init", session_id = "s-1" }), + Line(new { type = "user", message = new { role = "user", content = new object[] { new { type = "tool_result", tool_use_id = "t-1", content = $"ANTHROPIC_API_KEY={Key}\n" } } } }), + Line(new { type = "assistant", message = new { content = new object[] { new { type = "text", text = $"The key is {Key}, and again {Key}." } } } }), + ""); + + var masked = SessionTranscriptRedaction.Redact(transcript, new SecretRedactor([Key])); + + masked.ShouldNotContain(Key); + masked.Split('\n').Length.ShouldBe(transcript.Split('\n').Length, "a line is masked in place; the CLI's line structure is the session's structure"); + AssertValidJsonLines(masked); + masked.ShouldContain(SecretRedactor.Placeholder); + } + + [Fact] + public void A_line_without_a_secret_keeps_its_exact_bytes() + { + // The CLI wrote these lines; anything the mask did not need to touch must reach --resume byte for byte. + var untouched = "{\"type\":\"system\", \"session_id\":\"s-1\",\"cwd\":\"/tmp/w\\/x\"}"; + var transcript = untouched + "\n" + Line(new { text = Key }) + "\n"; + + var masked = SessionTranscriptRedaction.Redact(transcript, new SecretRedactor([Key])); + + masked.Split('\n')[0].ShouldBe(untouched); + } + + [Theory] + [InlineData("p4ss-w0rd-backslash-tail\\")] // the file holds `…tail\\`: a raw match leaves a lone `\` that escapes what follows + [InlineData("pa\"ss-quoted-secret-value")] // the file holds `pa\"ss…`, so the raw value is never there to match + [InlineData("tab\tseparated-secret-value")] // a control character the CLI's encoder escapes + public void A_secret_the_cli_escaped_is_masked_and_the_line_still_parses(string secret) + { + // MUTATION this pins: replace the value-aware mask with plain text replacement over the line. The first case + // then produces invalid JSON and the other two leave the secret readable once the line is decoded. + var transcript = Line(new { type = "user", tool_result = $"value={secret};" }) + "\n"; + + var masked = SessionTranscriptRedaction.Redact(transcript, new SecretRedactor([secret])); + + AssertValidJsonLines(masked); + DecodedStrings(masked).ShouldAllBe(text => !text.Contains(secret), $"the decoded transcript still carries the secret: {masked}"); + } + + [Fact] + public void A_secret_in_a_property_name_is_masked_too() + { + var transcript = "{\"" + Key + "\":true}\n"; + + var masked = SessionTranscriptRedaction.Redact(transcript, new SecretRedactor([Key])); + + masked.ShouldNotContain(Key); + AssertValidJsonLines(masked); + } + + [Fact] + public void A_numeric_secret_becomes_a_masked_string_and_the_line_still_parses() + { + const string pin = "8031975264"; + var transcript = "{\"account\":" + pin + ",\"n\":42}\n"; + + var masked = SessionTranscriptRedaction.Redact(transcript, new SecretRedactor([pin])); + + masked.ShouldNotContain(pin); + AssertValidJsonLines(masked); + JsonDocument.Parse(masked.TrimEnd('\n')).RootElement.GetProperty("n").GetInt32().ShouldBe(42, "a number that is no secret keeps its value"); + } + + [Fact] + public void A_line_that_is_not_json_gets_the_plain_mask() + { + // A torn tail, or a log line a CLI interleaved: there is no value to mask inside, and nothing to keep valid. + var transcript = Line(new { ok = true }) + "\n" + $"{{\"type\":\"assistant\",\"text\":\"{Key}"; + + var masked = SessionTranscriptRedaction.Redact(transcript, new SecretRedactor([Key])); + + masked.ShouldNotContain(Key); + masked.Split('\n')[0].ShouldBe(Line(new { ok = true })); + } + + [Fact] + public void A_complete_line_holding_a_lone_surrogate_escape_is_still_masked_inside_its_values() + { + // Node's JSON.stringify writes a lone surrogate as an escape when a truncated tool output split an emoji: the line + // is valid JSON, but the reader's own string decoding refuses it. The secret here is one only the value-aware mask + // can find — its ESC is escaped in lowercase hex, as Node writes it, which no fixed escaped form matches. + // MUTATION this pins: decode with Utf8JsonReader.GetString → the line falls back and keeps the secret. + const string secret = "\u001b[0mCANARY-G2-esc-token-41c7"; + var line = "{\"type\":\"user\",\"tool_result\":\"\\u001b[0mCANARY-G2-esc-token-41c7 ... output truncated \\ud83d\"}"; + + Should.NotThrow(() => JsonDocument.Parse(line).Dispose(), "fixture check: the line is valid JSON as the CLI wrote it"); + + var masked = SessionTranscriptRedaction.Redact(line + "\n", new SecretRedactor([secret])); + + masked.ShouldNotContain("CANARY-G2-esc-token-41c7"); + AssertValidJsonLines(masked); + masked.ShouldContain("output truncated", Case.Sensitive, "the rest of the value is kept"); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_torn_tail_holding_a_secret_the_writer_escaped_is_masked(bool asciiOnlyWriter) + { + // A line the reader cannot take — the tail a killed CLI left half written — gets no value-aware mask, so a secret + // the writer escaped (a multi-line key, a quote, a backslash) must be matched in its escaped forms: a writer's + // short escapes (\n, \", \\), and an ASCII-only writer's \uXXXX ones. + // MUTATION this pins: drop the escaped forms from the fallback → both bodies survive. + const string pem = "-----BEGIN KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASC\n-----END KEY-----"; + const string quoted = "pa\"ss\\word+CANARY-G2-1234"; + var encoder = asciiOnlyWriter ? System.Text.Encodings.Web.JavaScriptEncoder.Default : System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping; + var torn = "{\"type\":\"user\",\"tool_result\":\"" + JsonEncodedText.Encode($"{pem} then {quoted} and the CLI was kil", encoder); + + var masked = SessionTranscriptRedaction.Redact(Line(new { ok = true }) + "\n" + torn, new SecretRedactor([pem, quoted])); + + var tail = masked.Split('\n')[1]; + tail.ShouldNotContain("MIIEvQIBADANBgkqhkiG9w0BAQEFAASC"); + tail.ShouldNotContain("CANARY-G2-1234"); + tail.ShouldContain("and the CLI was kil", Case.Sensitive, "only the secrets are masked"); + } + + [Fact] + public void A_redactor_with_no_secrets_returns_the_transcript_itself() + { + var transcript = Line(new { text = Key }) + "\n"; + + SessionTranscriptRedaction.Redact(transcript, SecretRedactor.None).ShouldBeSameAs(transcript); + } + + private static string Line(object value) => JsonSerializer.Serialize(value); + + private static void AssertValidJsonLines(string transcript) + { + foreach (var line in transcript.Split('\n').Where(l => l.Length > 0)) + Should.NotThrow(() => JsonDocument.Parse(line).Dispose(), $"every line --resume reads must still parse: {line}"); + } + + private static IEnumerable DecodedStrings(string transcript) + { + foreach (var line in transcript.Split('\n').Where(l => l.Length > 0)) + { + using var document = JsonDocument.Parse(line); + + foreach (var text in Strings(document.RootElement)) yield return text; + } + } + + private static IEnumerable Strings(JsonElement element) => element.ValueKind switch + { + JsonValueKind.String => [element.GetString()!], + JsonValueKind.Object => element.EnumerateObject().SelectMany(p => Strings(p.Value).Prepend(p.Name)), + JsonValueKind.Array => element.EnumerateArray().SelectMany(Strings), + _ => [], + }; +} diff --git a/backend/tests/CodeSpace.UnitTests/Agents/SupervisorBranchlessStopGradeTests.cs b/backend/tests/CodeSpace.UnitTests/Agents/SupervisorBranchlessStopGradeTests.cs index 42281316d..ebaccd67a 100644 --- a/backend/tests/CodeSpace.UnitTests/Agents/SupervisorBranchlessStopGradeTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Agents/SupervisorBranchlessStopGradeTests.cs @@ -238,19 +238,64 @@ public async Task A_missing_rubric_judge_is_not_marked_judgedSummary() SupervisorOutcome.ReadAcceptanceGradeJudgedSummary(outcome).ShouldBeFalse(); } + // ── F1 residual: what a supervisor grade keeps never carries a workflow secret ── + + private const string UnitSecret = "CANARY-G2-unit-goal-4b5c6d7e"; + private const string SpecSecret = "CANARY-G2-floor-argv-8f9a0b1c"; + + [Fact] + public async Task A_branchless_gate_masks_the_units_sealed_secrets_and_the_scope_secrets_its_spec_carries() + { + // The unit's sealed task names the value its goal embedded; the node scope resolved another into the + // model-authored acceptance argv. Both run exactly, and the grade repeats the argv and its output into a team + // artifact — so the request must name both, and nothing else of the scope. + // MUTATIONS this pins: drop WorkflowSecrets from the branchless lane's request; read the unit's secrets without + // opening its sealed task; or read the spec's without the ambient scope → one of the two canaries goes missing → red. + var sealer = new CodeSpace.Core.Services.Workflows.Runtime.WorkflowSensitivePayloadSealer(new CodeSpace.Core.Services.Workflows.Runtime.DataProtectionWorkflowSensitivePayloadProtector(new Microsoft.AspNetCore.DataProtection.EphemeralDataProtectionProvider())); + var db = Infrastructure.EmptyTestDb.New(); + var task = new AgentTask { Goal = $"write the report, deploy key {UnitSecret}", Harness = "codex-cli", WorkflowSecrets = [UnitSecret] }; + var sealedTask = AgentTaskSecrets.Seal(task, sealer); + db.AgentRun.Add(new CodeSpace.Core.Persistence.Entities.AgentRun { Id = UnitId, TeamId = TeamId, Harness = task.Harness, Status = AgentRunStatus.Succeeded, TaskJson = sealedTask.PublicJson, SealedTaskCiphertext = sealedTask.Ciphertext }); + await db.SaveChangesAsync(); + var grader = new CapturingGrader(new BenchmarkGrade { Passed = true, Detail = "artifact-present" }); + + using (InNodeScope(SpecSecret, "a-scope-secret-no-spec-carries")) + await GradeAsync(grader, StopWith(BenchmarkGradingKind.ArtifactPresent, command: $"REPORT-{SpecSecret}.md"), db: db, sealer: sealer); + + grader.WorkflowSecrets.ShouldHaveSingleItem().ShouldBe([UnitSecret, SpecSecret], ignoreOrder: true, "the grade masks what the unit was handed and what its acceptance argv carries — and no other scope secret"); + } + + [Fact] + public async Task A_branchless_gate_that_faults_masks_the_fault_it_quotes() + { + // MUTATION this pins: record `grade-error: {ex.Message}` without MaskedDetail → the decision tape keeps the canary → red. + var grader = new CapturingGrader(new InvalidOperationException($"judge endpoint refused bearer {SpecSecret}")); + + string outcome; + using (InNodeScope(SpecSecret)) + outcome = await GradeAsync(grader, StopWith(BenchmarkGradingKind.ArtifactPresent)); + + outcome.ShouldContain("grade-error: "); + outcome.ShouldNotContain(SpecSecret); + } + + /// The ambient call scope the engine pushes for the supervisor node: its capture redactor holds the node scope's secrets. + private static IDisposable InNodeScope(params string[] secrets) => + CodeSpace.Core.Services.Workflows.Llm.LlmCallContext.Push(new CodeSpace.Core.Services.Workflows.Llm.LlmCallScope(Guid.NewGuid(), TeamId, "sup", "", "agent.supervisor", null!, null!, CaptureRedactor: new CodeSpace.Core.Services.Workflows.Runtime.PersistenceSecretRedactor(secrets))); + // ── fixture ── private static readonly AcceptanceRubric WithRubric = new() { Criteria = new[] { new AcceptanceRubricCriterion { Id = "sources", Requirement = "names at least one source" } } }; /// Drive the stop grade directly with a branchless context (the fake resolver finds no published branch when the tape carries none). - private static async Task GradeAsync(CapturingGrader grader, string stopPayloadJson, StubRubricJudge? rubricJudge = null, IReadOnlyList? acceptanceChecks = null, ProducerFixture? producers = null, CodeSpace.Core.Persistence.Db.CodeSpaceDbContext? db = null) + private static async Task GradeAsync(CapturingGrader grader, string stopPayloadJson, StubRubricJudge? rubricJudge = null, IReadOnlyList? acceptanceChecks = null, ProducerFixture? producers = null, CodeSpace.Core.Persistence.Db.CodeSpaceDbContext? db = null, CodeSpace.Core.Services.Workflows.Runtime.WorkflowSensitivePayloadSealer? sealer = null) { // Only the stop-grade path's own collaborators are real here: the grader under test, the branch resolver (the // source of the branchless world), the manifest store the oracle anchor reads, and the budget ledger the call // scope carries. Every other seam is untouched by ApplyStopAcceptanceGradeAsync. var service = new SupervisorTurnService(null!, null!, null!, db: db ?? Infrastructure.EmptyTestDb.New(), grader, null!, null!, null!, null!, null!, null!, new NoManifests(), new FakeSupervisorPublishedBranchResolver(), null!, new AdmitAllBudgetLedger(), - null!, null!, NullLogger.Instance, rubricJudge); + null!, null!, NullLogger.Instance, rubricJudge, sealer: sealer); var context = new SupervisorTurnContext { @@ -287,11 +332,11 @@ private static async Task GradeAsync(CapturingGrader grader, string stop private sealed record ProducerFixture(ReviewModelIdentity? Unit, ReviewModelIdentity? Stop); - private static string StopWith(BenchmarkGradingKind? kind, AcceptanceRubric? rubric = null) => JsonSerializer.Serialize(new SupervisorStopPayload + private static string StopWith(BenchmarkGradingKind? kind, AcceptanceRubric? rubric = null, string command = "REPORT.md") => JsonSerializer.Serialize(new SupervisorStopPayload { Outcome = "completed", Summary = "The two languages differ mainly in their memory model.", - Acceptance = new SupervisorAcceptanceSpec { Command = new[] { "REPORT.md" }, Kind = kind, Rubric = rubric }, + Acceptance = new SupervisorAcceptanceSpec { Command = new[] { command }, Kind = kind, Rubric = rubric }, }, AgentJson.Options); /// The oracle-anchor read: this run published nothing, so it has no manifest rows. @@ -323,10 +368,13 @@ private sealed class CapturingGrader : ISupervisorAcceptanceGrader public List Postures { get; } = new(); + public List?> WorkflowSecrets { get; } = new(); + public Task GradeCapturedAsync(CapturedAcceptanceGradeRequest request, CancellationToken cancellationToken) { ProducerModels.Add(request.ProducerModel); Postures.Add(request.Posture); + WorkflowSecrets.Add(request.WorkflowSecrets); return GradeCapturedAsync(request.AgentRunId, request.TeamId, request.Spec, request.TimeoutSeconds, cancellationToken); } diff --git a/backend/tests/CodeSpace.UnitTests/Architecture/AcceptanceGradePostureInventoryTests.cs b/backend/tests/CodeSpace.UnitTests/Architecture/AcceptanceGradePostureInventoryTests.cs index e5bb35c2e..6e2e6bd1c 100644 --- a/backend/tests/CodeSpace.UnitTests/Architecture/AcceptanceGradePostureInventoryTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Architecture/AcceptanceGradePostureInventoryTests.cs @@ -57,6 +57,20 @@ public void Every_production_grade_passes_a_posture_carrying_request() $"{nameof(AcceptanceGradingPosturePolicy)}.{nameof(AcceptanceGradingPosturePolicy.For)}(the producing run's task):\n " + string.Join("\n ", offenders)); } + [Fact] + public void Every_production_grade_request_names_the_secrets_its_evidence_must_mask() + { + // The other half of what a grade keeps. An acceptance argv can carry a workflow secret (an operator floor, or + // one a model authored from a goal that resolved it), the check runs it exactly, and the grade repeats the argv + // and the check's output into a team artifact, its tail and its detail. Only a request that names the + // producer's secrets gets them masked — a lane that leaves WorkflowSecrets out stores them in plaintext, and + // compiles. The supervisor's nine lanes did, until this was pinned. + // MUTATION this pins: drop `WorkflowSecrets = …` from any lane's request → that lane is named here → red. + var unmasked = GradeCalls().Where(call => !call.Text.Contains("WorkflowSecrets = ", StringComparison.Ordinal)).Select(call => $"{call.File}:{call.Line}").ToList(); + + unmasked.ShouldBeEmpty("these grade requests carry no WorkflowSecrets, so the grader cannot mask a secret their argv or output carries: pass the producing run's (its opened task's AgentTask.WorkflowSecrets, plus the scope secrets the spec carries):\n " + string.Join("\n ", unmasked)); + } + [Fact] public void The_set_of_grade_lanes_is_pinned() { @@ -114,9 +128,9 @@ public void Every_oracle_runner_is_bound_to_a_producing_run_posture() } /// Every grade call in a production file that holds the acceptance grader, minus the declarations and the local-verifier hand-off. Comment lines are text about a call, not a call. - private static List<(string File, int Line, string Argument)> GradeCalls() + private static List<(string File, int Line, string Argument, string Text)> GradeCalls() { - var calls = new List<(string, int, string)>(); + var calls = new List<(string, int, string, string)>(); foreach (var file in Directory.EnumerateFiles(Path.Combine(RepositoryRoot(), ScannedRoot), "*.cs", SearchOption.AllDirectories)) { @@ -133,7 +147,7 @@ public void Every_oracle_runner_is_bound_to_a_producing_run_posture() if (line.StartsWith("//", StringComparison.Ordinal) || line.StartsWith('*') || line.Contains(LocalVerifierHandOff, StringComparison.Ordinal)) continue; - foreach (Match match in GradeCall.Matches(line)) calls.Add((relative, i + 1, match.Groups["argument"].Value.TrimStart())); + foreach (Match match in GradeCall.Matches(line)) calls.Add((relative, i + 1, match.Groups["argument"].Value.TrimStart(), line)); } } diff --git a/backend/tests/CodeSpace.UnitTests/Supervisor/AcceptanceEvidenceSecretMaskingTests.cs b/backend/tests/CodeSpace.UnitTests/Supervisor/AcceptanceEvidenceSecretMaskingTests.cs index b3176671d..1a0f3fa9a 100644 --- a/backend/tests/CodeSpace.UnitTests/Supervisor/AcceptanceEvidenceSecretMaskingTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Supervisor/AcceptanceEvidenceSecretMaskingTests.cs @@ -1,6 +1,7 @@ using CodeSpace.Core.Services.Agents.Eval.Benchmark; using CodeSpace.Core.Services.Agents.Eval.Benchmark.Graders; using CodeSpace.Core.Services.Agents.Sandbox; +using CodeSpace.Core.Services.Agents.Workspace; using CodeSpace.Core.Services.Supervisor; using CodeSpace.Core.Services.Workflows.Runtime; using CodeSpace.Messages.Agents; @@ -82,6 +83,35 @@ public void Every_kept_field_of_a_grade_is_masked_and_a_grade_with_no_secrets_is SupervisorAcceptanceGrader.MaskedGrade(grade, null).ShouldBeSameAs(grade); } + [Theory] + [InlineData("repository", "clone-failed: ")] + [InlineData("patch", "clone-failed: ")] + [InlineData("base", "clone-failed: ")] + [InlineData("repository-fault", "grade-error: ")] + public async Task A_grade_that_fails_before_its_check_runs_keeps_its_prefix_with_the_secret_masked(string lane, string prefix) + { + // The fail-closed arms quote the failure they caught, and that detail is kept: on the result, the supervisor's + // decision tape, the Room. Its prefix is what the infra classifier reads, so only the secret may change. + // MUTATION this pins: return a lane's grade without MaskedGrade (the shape before this fix, where only the + // evidence capture masked) → the canary survives in the detail → red. + var failure = $"fatal: could not read from https://ci:{Secret}@git.test/team/repo"; + var grader = new SupervisorAcceptanceGrader(new FailingResolver(lane.EndsWith("-fault", StringComparison.Ordinal) ? new InvalidOperationException(failure) : new WorkspaceException(failure)), null!, new RecordingRunner(), new BenchmarkGraderRegistry(new IBenchmarkGrader[] { new TestsPassGrader() }), null!, new SupervisorAcceptanceGraderTests.FakeArtifactStore(), null!, NullLogger.Instance); + var spec = new SupervisorAcceptanceSpec { Command = new[] { "make", "test" } }; + IReadOnlyList secrets = [Secret]; + + var grade = lane switch + { + "patch" => await grader.GradePatchAsync(new PatchAcceptanceGradeRequest { RepositoryId = Guid.NewGuid(), TeamId = Guid.NewGuid(), BaseSha = "abc123", Spec = spec, TimeoutSeconds = 30, Posture = null, WorkflowSecrets = secrets }, CancellationToken.None), + "base" => await grader.GradeBaseAsync(new BaseAcceptanceGradeRequest { RepositoryId = Guid.NewGuid(), TeamId = Guid.NewGuid(), BaseSha = "abc123", Spec = spec, TimeoutSeconds = 30, Posture = null, WorkflowSecrets = secrets }, CancellationToken.None), + _ => await grader.GradeAsync(new RepositoryAcceptanceGradeRequest { RepositoryId = Guid.NewGuid(), TeamId = Guid.NewGuid(), Branch = "agent/x", Spec = spec, TimeoutSeconds = 30, Posture = null, WorkflowSecrets = secrets }, CancellationToken.None), + }; + + grade.Passed.ShouldBeFalse(); + grade.Detail.ShouldStartWith(prefix, Case.Sensitive, "the classifier reads the prefix; masking may not move it"); + grade.Detail.ShouldNotContain(Secret, Case.Sensitive, $"the {lane} lane kept the secret in its detail: {grade.Detail}"); + grade.Detail.ShouldContain(PersistenceSecretRedactor.Marker); + } + private static async Task GradeAsync(RecordingRunner runner, SupervisorAcceptanceGraderTests.FakeArtifactStore artifacts, string[] command, string[]? setup, IReadOnlyList? secrets) { var directory = Directory.CreateTempSubdirectory("cs-grade-mask-").FullName; @@ -97,6 +127,14 @@ private static async Task GradeAsync(RecordingRunner runner, Sup finally { Directory.Delete(directory, recursive: true); } } + /// A resolver whose every resolve fails the way a clone does (or with any other fault), quoting the failure. + private sealed class FailingResolver(Exception failure) : IAgentWorkspaceResolver + { + public Task ResolveAsync(AgentTask task, Guid teamId, CancellationToken cancellationToken) => throw failure; + + public Task ResolveByRepositoryIdAsync(Guid repositoryId, Guid teamId, CancellationToken cancellationToken, string? @ref = null, bool softFallback = false, string? pinnedSha = null) => throw failure; + } + /// Records every spec it runs; a spec carrying the setup argv answers , any other . private sealed class RecordingRunner : ISandboxRunnerRegistry, ISandboxRunner { diff --git a/backend/tests/CodeSpace.UnitTests/Supervisor/CapturedDeliverableGradeTests.cs b/backend/tests/CodeSpace.UnitTests/Supervisor/CapturedDeliverableGradeTests.cs index 8f8ded205..670be269f 100644 --- a/backend/tests/CodeSpace.UnitTests/Supervisor/CapturedDeliverableGradeTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Supervisor/CapturedDeliverableGradeTests.cs @@ -160,6 +160,43 @@ public async Task A_logical_path_that_collides_with_a_captured_directory_costs_o oracle.SeenFiles.ShouldNotBeEmpty("whichever of the pair landed first is still graded"); } + [Fact] + public async Task A_captured_grade_masks_the_producers_secrets_in_everything_it_keeps() + { + // F1 residual: the supervisor's repo-less lane graded a unit with no secrets at all, so a model-authored + // acceptance that resolved one — or an oracle that quoted it back — kept it in the evidence artifact a Viewer + // downloads, its tail and the detail on the decision tape. + // MUTATION this pins: drop WorkflowSecrets from the captured lane's WorkspaceGradeRequest → the stored evidence + // carries the canary → red. + const string secret = "CANARY-G2-captured-9e8d7c6b"; + var runId = Guid.NewGuid(); + var teamId = Guid.NewGuid(); + var (grader, _, store) = New(new BenchmarkGrade { Passed = false, Detail = $"missing: report.md (looked with {secret})", EvidenceText = $"$ check {secret}\nnot found" }, Row(runId, teamId, "notes.md", "# notes\n")); + + var grade = await grader.GradeCapturedAsync(new CapturedAcceptanceGradeRequest { AgentRunId = runId, TeamId = teamId, Spec = Spec("report.md"), TimeoutSeconds = 60, Posture = null, WorkflowSecrets = [secret] }, CancellationToken.None); + + store.PutTexts.ShouldHaveSingleItem("the evidence is stored as a team artifact").ShouldNotContain(secret); + grade.Detail.ShouldNotContain(secret); + (grade.EvidenceTail ?? "").ShouldNotContain(secret); + } + + [Fact] + public async Task A_captured_grade_whose_rebuild_faulted_masks_the_quoted_fault() + { + // MUTATION this pins: return the captured lane's grade without MaskedGrade → the `grade-error:` arm quotes the + // exception verbatim → red. + const string secret = "CANARY-G2-captured-fault-1a2b3c"; + var runId = Guid.NewGuid(); + var teamId = Guid.NewGuid(); + var (grader, oracle, _) = New(new BenchmarkGrade { Passed = true, Detail = "unused" }, Row(runId, teamId, "report.md", "# r\n")); + oracle.Throw = new InvalidOperationException($"judge refused: token {secret} expired"); + + var grade = await grader.GradeCapturedAsync(new CapturedAcceptanceGradeRequest { AgentRunId = runId, TeamId = teamId, Spec = Spec("report.md"), TimeoutSeconds = 60, Posture = null, WorkflowSecrets = [secret] }, CancellationToken.None); + + grade.Detail.ShouldStartWith("grade-error: "); + grade.Detail.ShouldNotContain(secret); + } + // ─── Fixtures ─── private static SupervisorAcceptanceSpec Spec(params string[] paths) => @@ -217,7 +254,14 @@ private sealed class FakeArtifactStore : IArtifactStore ? new ArtifactBytes { Id = artifactId, Sha256 = "", ContentType = "text/plain", Bytes = bytes } : null); - public Task PutAsync(Guid teamId, ReadOnlyMemory bytes, string contentType, CancellationToken cancellationToken) => Task.FromResult(Guid.NewGuid()); + public List PutTexts { get; } = new(); + + public Task PutAsync(Guid teamId, ReadOnlyMemory bytes, string contentType, CancellationToken cancellationToken) + { + PutTexts.Add(Encoding.UTF8.GetString(bytes.Span)); + return Task.FromResult(Guid.NewGuid()); + } + public Task GetMetadataAsync(Guid teamId, Guid artifactId, CancellationToken cancellationToken) => throw new NotSupportedException(); } diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorReviseTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorReviseTests.cs index 8ae5a75af..e9966b7a3 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorReviseTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorReviseTests.cs @@ -135,6 +135,20 @@ public void A_captured_session_makes_the_revision_warm() revise.Goal.ShouldNotContain("Original goal", customMessage: "a warm resume already holds the goal — only the delta is sent"); } + [Fact] + public void A_warm_revision_continues_the_exact_conversation_it_is_handed_rather_than_the_masked_kept_copy() + { + // The kept transcript is masked for storage; the warm goal only says "continue". The executor hands the round + // the session as the agent wrote it, so a secret the original goal carried is still in the conversation. + var result = AcceptanceFailed("exit 1") with { SessionId = "sess-1", SessionTranscript = "{\"text\":\"deploy with ***\"}" }; + + var revise = AgentRunExecutor.BuildReviseTask(TaskWith(), result, "the check failed", conversation: "{\"text\":\"deploy with tok-123\"}"); + + revise.RestoredTranscript.ShouldBe("{\"text\":\"deploy with tok-123\"}"); + revise.ResumeFromSessionId.ShouldBe("sess-1"); + AgentRunExecutor.BuildReviseTask(TaskWith(), result, "the check failed", mayResume: false, conversation: "{\"text\":\"deploy with tok-123\"}").RestoredTranscript.ShouldBeNull("a round the pipe cannot carry warm restores nothing"); + } + [Theory] [InlineData("sess-1", null)] // id captured but no transcript — --resume would error on an empty config home [InlineData(null, "bytes")] // transcript without an id — nothing to hand --resume diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorSessionTranscriptTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorSessionTranscriptTests.cs index 0598d3343..cffab9aec 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorSessionTranscriptTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorSessionTranscriptTests.cs @@ -1,4 +1,5 @@ using System.Text; +using System.Text.Json; using CodeSpace.Core.Services.Agents; using CodeSpace.Core.Services.Agents.Harnesses.Claude; using CodeSpace.Core.Services.Agents.Sandbox.Runners; @@ -187,13 +188,45 @@ public async Task An_under_cap_session_is_captured_so_the_skip_is_not_a_dead_pat Encoding.UTF8.GetBytes(captured.SessionTranscript).ShouldBe(spool.SessionBytes, "under the cap the session is captured byte-for-byte; without this the skip test would pass on a capture that never works"); } + // ─── SX-13: what the capture KEEPS is masked, and still resumes ───────────────────────────────── + // + // The session file is the agent's own: a tool call that printed its environment, or a goal that carried a + // workflow secret, puts the value there verbatim. The capture is the result's sessionTranscript, the offloaded + // artifact and the resume payload a retry restores, so it is masked with the run's redactor — and masked inside + // the JSON values, because the next attempt hands these exact lines to `--resume`. + + [Fact] + public async Task A_captured_session_is_masked_with_the_runs_secrets_and_every_line_still_parses() + { + // MUTATION this pins: read the file into SessionTranscript without SessionTranscriptRedaction (the shape before + // this fix) → the key and the workflow secret are in the captured transcript → red. + const string key = "CANARY-G2-capture-sk-ant-91b2c3d4e5"; + const string workflowSecret = "CANARY-G2-capture-wf-\"quoted\"-7a8b"; + var session = JsonSerializer.Serialize(new { type = "user", tool_result = $"ANTHROPIC_API_KEY={key}" }) + "\n" + + JsonSerializer.Serialize(new { type = "assistant", text = $"deploying with {workflowSecret}" }) + "\n"; + using var spool = new SessionSpool(Encoding.UTF8.GetBytes(session)); + + var captured = await CaptureAsync(spool, Result(), new SecretRedactor([key, workflowSecret])); + + var lines = captured.SessionTranscript.Split('\n', StringSplitOptions.RemoveEmptyEntries); + lines.Length.ShouldBe(2, "the mask works line by line; the session keeps its turns"); + foreach (var line in lines) + { + using var parsed = JsonDocument.Parse(line); // throws on a line the mask broke — --resume would refuse it + parsed.RootElement.ToString().ShouldNotContain(key); + } + + captured.SessionTranscript.ShouldNotContain(key, Case.Sensitive, "the model key the agent echoed must not be kept"); + JsonDocument.Parse(lines[1]).RootElement.GetProperty("text").GetString().ShouldBe("deploying with ***", "the workflow secret is masked inside its value, escaped quotes and all"); + } + /// Run the production capture over a staged spool and return the result it produced. - private static async Task CaptureAsync(SessionSpool spool, AgentRunResult result) + private static async Task CaptureAsync(SessionSpool spool, AgentRunResult result, SecretRedactor? redactor = null) { var executor = new AgentRunExecutor(null!, null!, null!, null!, null!, null!, null!, null!, null!, null!, null!, null!, null!, null!, null!, null!, null!, NullLogger.Instance); var task = new AgentTask { Goal = "Fix the failing billing tests", Harness = ClaudeCodeHarness.HarnessKind, WorkspaceDirectory = SessionSpool.WorkspaceDirectory }; - return await executor.CaptureSessionTranscriptAsync(new AgentRunExecutor.SessionCapture(Guid.NewGuid(), task, new ClaudeCodeHarness(), spool.Handle), result, CancellationToken.None); + return await executor.CaptureSessionTranscriptAsync(new AgentRunExecutor.SessionCapture(Guid.NewGuid(), task, new ClaudeCodeHarness(), spool.Handle, redactor ?? SecretRedactor.None), result, CancellationToken.None); } private static AgentRunResult Result() => new() { Status = AgentRunStatus.Succeeded, ExitReason = "completed", SessionId = "sess-cap-boundary" }; @@ -205,7 +238,9 @@ private sealed class SessionSpool : IDisposable private readonly string _root = Path.Combine(Path.GetTempPath(), $"cs-spool-{Guid.NewGuid():N}"); - public SessionSpool(long atLeastBytes) + public SessionSpool(long atLeastBytes) : this(BuildSession(atLeastBytes)) { } + + public SessionSpool(byte[] session) { var configHome = LocalProcessRunner.ConfigHomePath(_root); var relative = ((IAgentSessionTranscript)new ClaudeCodeHarness()).SessionTranscriptRelativePath(configHome, WorkspaceDirectory, "sess-cap-boundary") @@ -213,7 +248,7 @@ public SessionSpool(long atLeastBytes) var path = Path.Combine(configHome, relative.Replace('/', Path.DirectorySeparatorChar)); Directory.CreateDirectory(Path.GetDirectoryName(path)!); - SessionBytes = BuildSession(atLeastBytes); + SessionBytes = session; File.WriteAllBytes(path, SessionBytes); Handle = new SandboxHandle { Kind = "local", ProcessId = 4242, SpoolDirectory = _root, Deadline = DateTimeOffset.UtcNow.AddHours(1) }; diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/LocalProcessDurableRunnerTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/LocalProcessDurableRunnerTests.cs index 49462b0bb..50780a455 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/LocalProcessDurableRunnerTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/LocalProcessDurableRunnerTests.cs @@ -682,6 +682,55 @@ private static async Task RunSupervisorScriptAsync(string spoolDirectory, l return process.ExitCode; } + [Fact] + public async Task The_spool_and_the_raw_output_logs_it_holds_are_owner_only() + { + if (OperatingSystem.IsWindows()) return; // unix modes (Rule 12.1) + + // SX-16: out.log is the agent's stdout BEFORE redaction — an echoed model key, a broker token, a workflow secret + // in a tool result — and the spool sat at the default umask: a 0755 directory around a 0644 out.log, readable by + // every local account on the worker, beside the 0700 agent-home that was already locked down. + // MUTATION this pins: drop the spool's owner-only create at launch → the directory reads 0755 → red; drop the + // copiers' umask → out.log/err.log read 0644 → red. + var handle = await LaunchAsync(ContractSpecs.MultiLine("alpha", "beta")); + + var (result, lines) = await AttachCollectAsync(handle); + + result.Status.ShouldBe(SandboxStatus.Success, Why(handle, result)); + string.Join(",", lines).ShouldBe("alpha,beta", "the run still spools and is still read back — the modes changed, the capture did not"); + + File.GetUnixFileMode(handle.SpoolDirectory).ShouldBe(UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute, $"the per-run spool {handle.SpoolDirectory} must be owner-only, as its agent-home is"); + + foreach (var log in new[] { "out.log", "err.log" }) + File.GetUnixFileMode(Path.Combine(handle.SpoolDirectory, log)).ShouldBe(UnixFileMode.UserRead | UnixFileMode.UserWrite, $"{log} holds the agent's raw, unredacted output and must be owner-only (0600)"); + } + + [Fact] + public async Task The_agent_keeps_the_workers_umask_for_the_files_it_writes() + { + if (OperatingSystem.IsWindows()) return; + + // The other half of the copiers' owner-only umask: it is THEIRS. The agent's own files — a build artifact, a + // report a later step or a reviewer reads — keep the mode the worker would give them. + // MUTATION this pins: set `umask 077` at the top of the supervisor script instead of inside copy_spool → the + // agent reports 0077 → red. + var (worker, _) = await RunShellAsync("umask"); + var (result, lines) = await AttachCollectAsync(await LaunchAsync(ContractSpecs.EchoStdin(null) with { Args = ["-c", "umask"], TimeoutSeconds = 20 })); + + result.Status.ShouldBe(SandboxStatus.Success); + lines.ShouldHaveSingleItem().ShouldBe(worker.Trim(), "the agent runs with the worker's umask, not the copiers' 077"); + } + + private static async Task<(string Stdout, int ExitCode)> RunShellAsync(string command) + { + using var process = Process.Start(new ProcessStartInfo("/bin/sh", ["-c", command]) { RedirectStandardOutput = true, UseShellExecute = false })!; + var stdout = await process.StandardOutput.ReadToEndAsync(); + + await process.WaitForExitAsync(); + + return (stdout, process.ExitCode); + } + [Fact] public async Task A_run_under_the_spool_cap_spools_byte_identical_output() {