diff --git a/.agents/skills/verify-agent-server/references/feature-map/F04-conversation-lifecycle.md b/.agents/skills/verify-agent-server/references/feature-map/F04-conversation-lifecycle.md index a2481e1ee4d..317151998f9 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F04-conversation-lifecycle.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F04-conversation-lifecycle.md @@ -206,12 +206,13 @@ EOF --check workspace.working_dir eq "$W" --save F04.create-settings/get control-agent-server api GET /api/conversations/count --check . eq $((N1 + 1)) control-agent-server state cat "server/workspace/conversations/${CID//-/}/meta.json" --max-chars 200 \ - --check agent missing --check secrets_encrypted eq true --check autotitle eq false --check tags.qa eq f04 + --check agent missing --check secrets_encrypted missing --check autotitle eq false --check tags.qa eq f04 control-agent-server state cat "server/workspace/conversations/${CID//-/}/base_state.json" --max-chars 200 \ --check agent.llm.model eq "$MODEL" --check execution_status eq idle ``` - The create is 201 and `idle`, the count grows by one, `meta.json` has no - `agent` and `base_state.json` holds it. + The create is 201 and `idle`, the count grows by one, `meta.json` has + neither the `agent` nor the request's `secrets_encrypted` flag, and + `base_state.json` holds the agent. - **Inline agent (`F04.create-inline-agent`).** An `agent` payload without `kind`. ```sh @@ -408,12 +409,13 @@ EOF --json "{\"workspace\": {\"working_dir\": \"$W\"}, \"agent\": $OFFLINE_AGENT, \"autotitle\": false, \"secrets\": {\"QA_F04_TOKEN\": {\"kind\": \"StaticSecret\", \"value\": \"qa-f04-secret-value\"}}}" \ --expect 201 --check secret_registry.secret_sources.QA_F04_TOKEN.kind eq StaticSecret \ --check secret_registry.secret_sources.QA_F04_TOKEN.value ne qa-f04-secret-value --save F04.create-secrets/start --field id) + control-agent-server state cat "server/workspace/conversations/${SECID//-/}/base_state.json" --max-chars 200 \ + --check secret_registry.secret_sources.QA_F04_TOKEN.kind eq StaticSecret --not-contains qa-f04-secret-value control-agent-server state cat "server/workspace/conversations/${SECID//-/}/meta.json" --max-chars 200 \ - --check secrets.QA_F04_TOKEN.kind eq StaticSecret --not-contains qa-f04-secret-value - control-agent-server state cat "server/workspace/conversations/${SECID//-/}/base_state.json" --max-chars 200 --not-contains qa-f04-secret-value + --check id exists --check secrets missing --not-contains QA_F04_TOKEN ``` The response lists the secret with its value masked; on disk it is - encrypted. + encrypted in `base_state.json`, and `meta.json` does not mention it. - **Start-time secret visible to GET (`F04.create-secrets-get`).** Read the conversation created in the previous bullet. @@ -790,8 +792,8 @@ EOF done ``` The response reports the actual isolation, including `shared` for the - non-git fallback. GET proves inherited model, limits, tags and masked - start-time secrets, plus the parent link. Only the child's task appears in + non-git fallback. GET proves inherited model, limits, tags and the + parent's secrets, masked, plus the parent link. Only the child's task appears in its user history; the parent stays idle. The worktree has its own branch. The non-git case removes the empty `.git` created during parent startup, so it exercises the intended non-git fallback independently of the next bug. diff --git a/.agents/skills/verify-agent-server/references/feature-map/F08-confirmation-security-secrets.md b/.agents/skills/verify-agent-server/references/feature-map/F08-confirmation-security-secrets.md index de31af3db3a..eabab6e9f2e 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F08-confirmation-security-secrets.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F08-confirmation-security-secrets.md @@ -50,6 +50,16 @@ Routes: `POST /api/conversations/{conversation_id}/events/respond_to_confirmatio - `F08.secrets-no-cipher`: on a server with no cipher (no `OH_SECRET_KEY`, no session key) a runtime secret reaches commands while the conversation lives, is persisted only as the redaction placeholder (the server warns at startup), and is empty after a restart although GET still lists the name. - `F08.secrets-restart`: a runtime secret is still injected into commands after a server restart. - `F08.secrets-unsaved-restart`: a runtime secret survives a restart even when nothing else changed in the conversation after it was posted. +- `F08.start-values-restart`: a start-time `confirmation_policy`, `security_analyzer` and secret are returned by `GET /api/conversations/{id}` right after the create, and are still in force after a server restart with no runtime change. +- `F08.state-on-disk`: a conversation's `confirmation_policy`, `security_analyzer` and secrets are in `base_state.json` from creation on and never in `meta.json` (no `confirmation_policy`, `security_analyzer`, `secrets` or `secrets_encrypted` key), also after runtime changes. +- `F08.policy-restart`: a confirmation policy set at runtime is the one in force after a server restart, not the start-time one: the agent's next action still waits for confirmation. +- `F08.analyzer-restart`: a security analyzer set at runtime is still set after a server restart. +- `F08.analyzer-clear-restart`: a start-time security analyzer cleared at runtime stays cleared after a server restart. +- `F08.secrets-override-restart`: a runtime value that replaced a start-time secret is what commands get after a server restart. +- `F08.fork-after-restart`: a fork made after a server restart has the source's current confirmation policy, security analyzer and secrets (runtime changes included) in its own `base_state.json`, and its commands get the current secret value. +- `F08.child-inherits-current`: a child started with `POST .../children` has the parent's current confirmation policy, security analyzer and secrets (runtime changes included), its commands get the current secret value, and its `meta.json` has none of them. +- `F08.reload-after-pause`: a policy and an analyzer set at runtime are still in force after `POST /api/conversations/prepare-for-sandbox-pause` unloads the conversation and the next request loads it again. +- `F08.upgrade-from-release`: conversations the latest released server created and never ran keep their start-time secret, policy and analyzer after the same state is restarted on this checkout. ## How to get to it (agent POV) @@ -73,8 +83,9 @@ Routes: `POST /api/conversations/{conversation_id}/events/respond_to_confirmatio - WebSocket (owned by the events family): `/sockets/events/{conversation_id}` pushes the state updates above and the `UserRejectObservation` live. - Start-time fields of `POST /api/conversations` (`confirmation_policy`, - `security_analyzer`, `secrets`) are stored in `meta.json` and re-applied on - every reload; the runtime routes here change only the live state. + `security_analyzer`, `secrets`) seed the new conversation's state once. + From then on `base_state.json` is their only home (`meta.json` never has + them): the runtime routes here change it, and a reload keeps what it holds. - SDK (`openhands-sdk/openhands/sdk/conversation/impl/remote_conversation.py`): `RemoteConversation.set_confirmation_policy()`, `set_security_analyzer()`, `update_secrets()` (serializes `SecretSource` objects, calls callables), and @@ -126,8 +137,10 @@ Preconditions: (write `$QA_F08_ROT` to a file, finish), `$LOOKUP_AGENT` (write `$QA_F08_LOOKUP` to a file and echo it, finish) and `$SELF_AGENT` (write `$QA_F08_HOSTLESS`, `$QA_F08_ABSOLUTE` and `$QA_F08_ABSENT` to three files - and echo the first, finish). The - no-cipher bullet makes its own stub on its own server. Last, it creates + and echo the first, finish) and `$ROTR_AGENT` (write `$QA_F08_ROTR` to a + file, finish). The + no-cipher and upgrade bullets make their own stubs on their own servers. + Last, it creates `$KID`, an idle conversation on the offline agent that the knob, error and secret-form bullets share. - Every `known bug` bullet uses only these variables and arranges the rest @@ -140,7 +153,7 @@ control-agent-server llm preset deepseek W="$AGENT_SERVER_VERIFY_RUN/fixtures/qa-f08" mkdir -p "$W/knobs" "$W/accept" "$W/reject/qa-f08-victim" "$W/risky/qa-f08-victim" "$W/secret" "$W/rotate" \ "$W/lookup" "$W/self" "$W/scoped" "$W/nocipher" "$W/real" "$W/gitprov" \ - "$W/delegate" "$W/message" + "$W/delegate" "$W/message" "$W/rotate-restart" "$W/upgrade" NOPE=$(python3 -c 'import uuid; print(uuid.uuid4())') OFFLINE_AGENT='{"llm": {"model": "openai/qa-offline", "api_key": "qa-offline-key", "base_url": "http://127.0.0.1:9/v1", "num_retries": 0}, "tools": []}' MSG='{"role": "user", "content": [{"type": "text", "text": "Run the command."}]}' @@ -169,6 +182,9 @@ LOOKUP_STEP='tool:terminal:{"command":"printf %s \"$QA_F08_LOOKUP\" > qa-f08-loo LOOKUP_AGENT=$(stub_agent qa-f08-lookup --step "$LOOKUP_STEP" --step "$FINISH") SELF_STEP='tool:terminal:{"command":"printf %s \"$QA_F08_HOSTLESS\" > qa-f08-hostless.txt; printf %s \"$QA_F08_ABSOLUTE\" > qa-f08-absolute.txt; printf %s \"$QA_F08_ABSENT\" > qa-f08-absent.txt; echo \"self=$QA_F08_HOSTLESS\""}' SELF_AGENT=$(stub_agent qa-f08-self --step "$SELF_STEP" --step "$FINISH") +ROTR_STEP='tool:terminal:{"command":"printf %s \"$QA_F08_ROTR\" > qa-f08-rotr.txt"}' +ROTR_AGENT=$(stub_agent qa-f08-rotr --step "$ROTR_STEP" --step "$FINISH") +QUEUED='{"role": "user", "content": [{"type": "text", "text": "QA_F08 queued"}], "run": false}' REJECT_KIND=openhands.sdk.event.llm_convertible.observation.UserRejectObservation OBSERVATION_KIND=openhands.sdk.event.llm_convertible.observation.ObservationEvent KID=$(control-agent-server api POST /api/conversations \ @@ -777,6 +793,293 @@ KID=$(control-agent-server api POST /api/conversations \ eviction (default 20 minutes) or a sandbox pause right after `POST .../secrets` keeps it. +- **Start-time values in GET and after a restart (`F08.start-values-restart`).** + A conversation created with a policy, an analyzer and a secret in the + start body is read at once, then again after a restart that nothing + preceded. + ```sh + mkdir -p "$W/start-values" + SV_STUB=$(control-agent-server fixture llm-stub --name qa-f08-start-values \ + --step 'tool:terminal:{"command":"printf %s \"$QA_F08_START\" > qa-f08-start.txt"}' --step "$FINISH" --print-path) + SV_AGENT="{\"llm\": {\"model\": \"openai/qa-stub\", \"api_key\": \"qa-stub-key\", \"base_url\": \"$SV_STUB/v1\", \"num_retries\": 0}, \"tools\": [{\"name\": \"terminal\"}]}" + SVC=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W/start-values\"}, \"agent\": $SV_AGENT, \"autotitle\": false, \"confirmation_policy\": {\"kind\": \"ConfirmRisky\", \"threshold\": \"MEDIUM\", \"confirm_unknown\": false}, \"security_analyzer\": {\"kind\": \"PatternSecurityAnalyzer\"}, \"secrets\": {\"QA_F08_START\": {\"kind\": \"StaticSecret\", \"value\": \"qa-f08-start-value\", \"description\": \"qa start\"}}}" \ + --expect 201 --check confirmation_policy.kind eq ConfirmRisky --check security_analyzer.kind eq PatternSecurityAnalyzer --field id) + control-agent-server api GET "/api/conversations/$SVC" --check confirmation_policy.kind eq ConfirmRisky --check confirmation_policy.threshold eq MEDIUM \ + --check confirmation_policy.confirm_unknown eq false --check security_analyzer.kind eq PatternSecurityAnalyzer --save F08.start-values-restart/policy-analyzer + control-agent-server api GET "/api/conversations/$SVC" --check secret_registry.secret_sources.QA_F08_START.kind eq StaticSecret \ + --check secret_registry.secret_sources.QA_F08_START.description eq 'qa start' --save F08.start-values-restart/secret + control-agent-server restart + control-agent-server doctor + control-agent-server conversation send "$SVC" --text 'Run the command.' --wait --until finished --timeout 120 + test "$(cat "$W/start-values/qa-f08-start.txt")" = qa-f08-start-value + control-agent-server api GET "/api/conversations/$SVC" --check confirmation_policy.kind eq ConfirmRisky --check confirmation_policy.threshold eq MEDIUM \ + --check security_analyzer.kind eq PatternSecurityAnalyzer --check secret_registry.secret_sources.QA_F08_START.kind eq StaticSecret \ + --save F08.start-values-restart/after-restart + ``` + The GET right after the create shows `ConfirmRisky` at `MEDIUM`, + `PatternSecurityAnalyzer` and `QA_F08_START` (masked, description kept). + After the restart the low-risk command runs unasked and writes the secret's + value, and GET still shows all three. (A secret given alone, with the + default policy and no analyzer, is `F04.create-secrets-get`.) +- **Where the three live on disk (`F08.state-on-disk`).** A + conversation created with all three in the start body, read on disk before + anything else happens, then after runtime changes. + ```sh + DSK=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W/knobs\"}, \"agent\": $OFFLINE_AGENT, \"autotitle\": false, \"confirmation_policy\": {\"kind\": \"AlwaysConfirm\"}, \"security_analyzer\": {\"kind\": \"PatternSecurityAnalyzer\"}, \"secrets\": {\"QA_F08_DISK\": {\"kind\": \"StaticSecret\", \"value\": \"qa-f08-disk-value\"}}}" \ + --expect 201 --field id) + DSK_META="$AGENT_SERVER_VERIFY_RUN/server/workspace/conversations/${DSK//-/}/meta.json" + control-agent-server state cat base_state.json --conversation "$DSK" --max-chars 200 \ + --check confirmation_policy.kind eq AlwaysConfirm --check security_analyzer.kind eq PatternSecurityAnalyzer + control-agent-server state cat meta.json --conversation "$DSK" --max-chars 200 --check id exists \ + --check confirmation_policy missing --check security_analyzer missing --check secrets missing --check secrets_encrypted missing + jq -e '[has("confirmation_policy"), has("security_analyzer"), has("secrets"), has("secrets_encrypted")] | any | not' "$DSK_META" + control-agent-server state grep QA_F08_DISK --glob "server/workspace/conversations/${DSK//-/}/meta.json" --expect-none + control-agent-server state cat base_state.json --conversation "$DSK" --max-chars 200 \ + --check secret_registry.secret_sources.QA_F08_DISK.kind eq StaticSecret \ + --check secret_registry.secret_sources.QA_F08_DISK.value matches '^gAAAAA' --not-contains qa-f08-disk-value + control-agent-server api POST "/api/conversations/$DSK/confirmation_policy" --json '{"policy": {"kind": "ConfirmRisky", "threshold": "MEDIUM"}}' --expect 200 + control-agent-server api POST "/api/conversations/$DSK/security_analyzer" --json '{"security_analyzer": null}' --expect 200 + control-agent-server api POST "/api/conversations/$DSK/secrets" --json '{"secrets": {"QA_F08_DISK2": "qa-f08-disk2-value"}}' --expect 200 + control-agent-server state cat base_state.json --conversation "$DSK" --max-chars 200 \ + --check confirmation_policy.kind eq ConfirmRisky --check confirmation_policy.threshold eq MEDIUM --check security_analyzer eq null \ + --check secret_registry.secret_sources.QA_F08_DISK2.value matches '^gAAAAA' --not-contains qa-f08-disk2-value + jq -e '[has("confirmation_policy"), has("security_analyzer"), has("secrets"), has("secrets_encrypted")] | any | not' "$DSK_META" + control-agent-server state grep QA_F08_DISK --glob "server/workspace/conversations/${DSK//-/}/meta.json" --expect-none + control-agent-server state grep QA_F08_DISK --glob "server/workspace/conversations/${DSK//-/}/base_state.json" + ``` + `base_state.json` has the policy, the analyzer and the secret + (a Fernet token, never the value) as soon as the create returns, and + follows every runtime change. `meta.json` is readable (it has the `id`) + and has none of the four keys, before and after (`jq` tells an absent key + from a `null` one, which `--check ... missing` cannot); the last grep is + the positive control for the glob. +- **A runtime policy after a restart (`F08.policy-restart`).** + A conversation created with the default `NeverConfirm` is switched to + `AlwaysConfirm` at runtime; the server restarts; an events read loads the + conversation again before it is read and driven. + ```sh + mkdir -p "$W/policy-restart" + PR_STUB=$(control-agent-server fixture llm-stub --name qa-f08-policy-restart \ + --step 'tool:terminal:{"command":"echo qa-policy > qa-f08-policy.txt"}' --step "$FINISH" --print-path) + PR_AGENT="{\"llm\": {\"model\": \"openai/qa-stub\", \"api_key\": \"qa-stub-key\", \"base_url\": \"$PR_STUB/v1\", \"num_retries\": 0}, \"tools\": [{\"name\": \"terminal\"}]}" + PRC=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W/policy-restart\"}, \"agent\": $PR_AGENT, \"autotitle\": false}" \ + --expect 201 --check confirmation_policy.kind eq NeverConfirm --field id) + control-agent-server api POST "/api/conversations/$PRC/confirmation_policy" --json '{"policy": {"kind": "AlwaysConfirm"}}' --expect 200 + control-agent-server api GET "/api/conversations/$PRC" --check confirmation_policy.kind eq AlwaysConfirm + control-agent-server restart + control-agent-server doctor + control-agent-server api GET "/api/conversations/$PRC/events/count" --expect 200 + control-agent-server state ls --conversation "$PRC" owner_lease.json --expect-count 1 + control-agent-server api GET "/api/conversations/$PRC" --check confirmation_policy.kind eq AlwaysConfirm \ + --save F08.policy-restart/after-restart + control-agent-server state cat base_state.json --conversation "$PRC" --max-chars 200 --check confirmation_policy.kind eq AlwaysConfirm + control-agent-server conversation send "$PRC" --text 'Run the command.' --wait --until waiting_for_confirmation,finished,error --timeout 120 + control-agent-server api GET "/api/conversations/$PRC" --check execution_status eq waiting_for_confirmation --save F08.policy-restart/waiting + test ! -e "$W/policy-restart/qa-f08-policy.txt" + ``` + With the conversation loaded again (its lease file is back), GET + and `base_state.json` still say `AlwaysConfirm`, and the agent's terminal + action waits unrun. +- **A runtime analyzer after a restart (`F08.analyzer-restart`).** + A conversation created without an analyzer gets `PatternSecurityAnalyzer` + at runtime, then the server restarts and the conversation is loaded. + ```sh + ARS=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W/knobs\"}, \"agent\": $OFFLINE_AGENT, \"autotitle\": false}" \ + --expect 201 --check security_analyzer eq null --field id) + control-agent-server api POST "/api/conversations/$ARS/security_analyzer" --json '{"security_analyzer": {"kind": "PatternSecurityAnalyzer"}}' --expect 200 + control-agent-server api GET "/api/conversations/$ARS" --check security_analyzer.kind eq PatternSecurityAnalyzer + control-agent-server restart + control-agent-server doctor + control-agent-server api GET "/api/conversations/$ARS/events/count" --expect 200 + control-agent-server state ls --conversation "$ARS" owner_lease.json --expect-count 1 + control-agent-server api GET "/api/conversations/$ARS" --check security_analyzer.kind eq PatternSecurityAnalyzer \ + --save F08.analyzer-restart/after-restart + control-agent-server state cat base_state.json --conversation "$ARS" --max-chars 200 --check security_analyzer.kind eq PatternSecurityAnalyzer + ``` + The loaded conversation still has `PatternSecurityAnalyzer`. +- **A cleared analyzer after a restart (`F08.analyzer-clear-restart`).** + A conversation created with `PatternSecurityAnalyzer` has it cleared at + runtime, then the server restarts and the conversation is loaded. + ```sh + ARC=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W/knobs\"}, \"agent\": $OFFLINE_AGENT, \"autotitle\": false, \"security_analyzer\": {\"kind\": \"PatternSecurityAnalyzer\"}}" \ + --expect 201 --check security_analyzer.kind eq PatternSecurityAnalyzer --field id) + control-agent-server api POST "/api/conversations/$ARC/security_analyzer" --json '{"security_analyzer": null}' --expect 200 + control-agent-server api GET "/api/conversations/$ARC" --check security_analyzer eq null + control-agent-server restart + control-agent-server doctor + control-agent-server api GET "/api/conversations/$ARC/events/count" --expect 200 + control-agent-server state ls --conversation "$ARC" owner_lease.json --expect-count 1 + control-agent-server api GET "/api/conversations/$ARC" --check security_analyzer eq null \ + --save F08.analyzer-clear-restart/after-restart + ``` + The analyzer is still cleared. +- **A replaced start-time secret after a restart (`F08.secrets-override-restart`).** + `QA_F08_ROTR` starts as `qa-f08-rotr-old`, is replaced at runtime, and the + server restarts before any command runs. + ```sh + RSS=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W/rotate-restart\"}, \"agent\": $ROTR_AGENT, \"autotitle\": false, \"secrets\": {\"QA_F08_ROTR\": {\"kind\": \"StaticSecret\", \"value\": \"qa-f08-rotr-old\"}}}" \ + --expect 201 --field id) + control-agent-server api POST "/api/conversations/$RSS/secrets" --json '{"secrets": {"QA_F08_ROTR": "qa-f08-rotr-new"}}' --expect 200 + control-agent-server restart + control-agent-server doctor + control-agent-server conversation send "$RSS" --text 'Run the command.' --wait --until finished --timeout 120 + test "$(cat "$W/rotate-restart/qa-f08-rotr.txt")" = qa-f08-rotr-new + control-agent-server state grep qa-f08-rotr-old --glob 'server/**/*' --expect-none + control-agent-server state grep qa-f08-rotr-new --glob 'server/**/*' --expect-none + ``` + The command wrote `qa-f08-rotr-new`: the restart did not bring the + start-time value back, and neither value is stored in plaintext. +- **A fork after a restart (`F08.fork-after-restart`).** The + source gets a policy, an analyzer, a replaced secret and a second secret + after it was created (a tags PATCH saves them); the server restarts and + the fork is the first request that loads the source. + ```sh + mkdir -p "$W/fork-restart" + FK_STUB=$(control-agent-server fixture llm-stub --name qa-f08-fork-restart \ + --step 'tool:terminal:{"command":"printf %s \"$QA_F08_FORK\" > qa-f08-fork.txt"}' --step "$FINISH" --print-path) + FK_AGENT="{\"llm\": {\"model\": \"openai/qa-stub\", \"api_key\": \"qa-stub-key\", \"base_url\": \"$FK_STUB/v1\", \"num_retries\": 0}, \"tools\": [{\"name\": \"terminal\"}]}" + FKS=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W/fork-restart\"}, \"agent\": $FK_AGENT, \"autotitle\": false, \"secrets\": {\"QA_F08_FORK\": {\"kind\": \"StaticSecret\", \"value\": \"qa-f08-fork-old\"}}}" \ + --expect 201 --field id) + control-agent-server api POST "/api/conversations/$FKS/confirmation_policy" \ + --json '{"policy": {"kind": "ConfirmRisky", "threshold": "MEDIUM", "confirm_unknown": false}}' --expect 200 + control-agent-server api POST "/api/conversations/$FKS/security_analyzer" --json '{"security_analyzer": {"kind": "PatternSecurityAnalyzer"}}' --expect 200 + control-agent-server api POST "/api/conversations/$FKS/secrets" \ + --json '{"secrets": {"QA_F08_FORK": "qa-f08-fork-new", "QA_F08_FORK2": "qa-f08-fork2-value"}}' --expect 200 + control-agent-server api PATCH "/api/conversations/$FKS" --json '{"tags": {"qa": "f08-fork-saved"}}' --expect 200 + control-agent-server restart + control-agent-server doctor + FKF=$(control-agent-server api POST "/api/conversations/$FKS/fork" --expect 201 --field id) + control-agent-server api GET "/api/conversations/$FKF" --check confirmation_policy.kind eq ConfirmRisky --check confirmation_policy.threshold eq MEDIUM \ + --check security_analyzer.kind eq PatternSecurityAnalyzer --check secret_registry.secret_sources.QA_F08_FORK.kind eq StaticSecret \ + --check secret_registry.secret_sources.QA_F08_FORK2.kind eq StaticSecret --check forked_from_conversation_id eq "$FKS" \ + --save F08.fork-after-restart/fork + control-agent-server state cat base_state.json --conversation "$FKF" --max-chars 200 --check confirmation_policy.kind eq ConfirmRisky \ + --check security_analyzer.kind eq PatternSecurityAnalyzer --check secret_registry.secret_sources.QA_F08_FORK.value matches '^gAAAAA' \ + --check secret_registry.secret_sources.QA_F08_FORK2.value matches '^gAAAAA' --check agent.llm.api_key matches '^gAAAAA' \ + --not-contains qa-f08-fork-new + control-agent-server state cat meta.json --conversation "$FKF" --max-chars 200 --check id exists \ + --check confirmation_policy missing --check security_analyzer missing --check secrets missing + control-agent-server conversation send "$FKF" --text 'Run the command.' --wait --until finished,error --timeout 120 + control-agent-server assert --value "$(cat "$W/fork-restart/qa-f08-fork.txt")" --check . eq qa-f08-fork-new + control-agent-server api GET "/api/conversations/$FKS" --check confirmation_policy.kind eq ConfirmRisky \ + --check security_analyzer.kind eq PatternSecurityAnalyzer --check secret_registry.secret_sources.QA_F08_FORK2.kind eq StaticSecret \ + --save F08.fork-after-restart/source + control-agent-server api GET "/api/conversations/$FKS/events/count" --query kind="$OBSERVATION_KIND" --check . eq 0 + ``` + The fork answers with `ConfirmRisky` at `MEDIUM`, + `PatternSecurityAnalyzer` and both secrets, none of which the source had + when it was created; they are in the fork's `base_state.json` as Fernet + tokens (the model key too) and not in its `meta.json`; the fork's low-risk + command runs unasked and writes `qa-f08-fork-new`; the source keeps its + values and gains no observation from the fork's run. +- **A child inherits the current values (`F08.child-inherits-current`).** + The parent gets a policy, an analyzer, a replaced secret and a second + secret after it was created (a tags PATCH saves them), then starts a child + in the shared workspace; the child's stub runs one command and finishes. + ```sh + mkdir -p "$W/child" + CH_STUB=$(control-agent-server fixture llm-stub --name qa-f08-child \ + --step 'tool:terminal:{"command":"printf %s \"$QA_F08_CHILD\" > qa-f08-child.txt"}' --step "$FINISH" --print-path) + CH_AGENT="{\"llm\": {\"model\": \"openai/qa-stub\", \"api_key\": \"qa-stub-key\", \"base_url\": \"$CH_STUB/v1\", \"num_retries\": 0}, \"tools\": [{\"name\": \"terminal\"}]}" + CHP=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W/child\"}, \"agent\": $CH_AGENT, \"autotitle\": false, \"secrets\": {\"QA_F08_CHILD\": {\"kind\": \"StaticSecret\", \"value\": \"qa-f08-child-old\"}}}" \ + --expect 201 --field id) + control-agent-server api POST "/api/conversations/$CHP/confirmation_policy" \ + --json '{"policy": {"kind": "ConfirmRisky", "threshold": "MEDIUM", "confirm_unknown": false}}' --expect 200 + control-agent-server api POST "/api/conversations/$CHP/security_analyzer" --json '{"security_analyzer": {"kind": "PatternSecurityAnalyzer"}}' --expect 200 + control-agent-server api POST "/api/conversations/$CHP/secrets" \ + --json '{"secrets": {"QA_F08_CHILD": "qa-f08-child-new", "QA_F08_CHILD2": "qa-f08-child2-value"}}' --expect 200 + control-agent-server api PATCH "/api/conversations/$CHP" --json '{"tags": {"qa": "f08-child-saved"}}' --expect 200 + CHC=$(control-agent-server api POST "/api/conversations/$CHP/children" \ + --json '{"task": "Run the command.", "title": "QA F08 child", "isolation": "shared"}' \ + --expect 201 --check parent_conversation_id eq "$CHP" --check isolation eq shared --field conversation_id) + control-agent-server conversation wait "$CHC" --until finished,error,waiting_for_confirmation --timeout 120 + control-agent-server api GET "/api/conversations/$CHC" --check confirmation_policy.kind eq ConfirmRisky --check confirmation_policy.threshold eq MEDIUM \ + --check security_analyzer.kind eq PatternSecurityAnalyzer --check secret_registry.secret_sources.QA_F08_CHILD.kind eq StaticSecret \ + --check secret_registry.secret_sources.QA_F08_CHILD2.kind eq StaticSecret --check execution_status eq finished \ + --save F08.child-inherits-current/child + control-agent-server assert --value "$(cat "$W/child/qa-f08-child.txt")" --check . eq qa-f08-child-new + control-agent-server state cat base_state.json --conversation "$CHC" --max-chars 200 --check confirmation_policy.kind eq ConfirmRisky \ + --check security_analyzer.kind eq PatternSecurityAnalyzer --check secret_registry.secret_sources.QA_F08_CHILD.value matches '^gAAAAA' \ + --not-contains qa-f08-child-new + control-agent-server state cat meta.json --conversation "$CHC" --max-chars 200 --check parent_conversation_id exists \ + --check confirmation_policy missing --check security_analyzer missing --check secrets missing + control-agent-server api GET "/api/conversations/$CHP" --check sub_conversation_ids contains "$CHC" \ + --check confirmation_policy.kind eq ConfirmRisky --save F08.child-inherits-current/parent + ``` + The child has `ConfirmRisky` at `MEDIUM`, + `PatternSecurityAnalyzer` and both secrets, its low-risk command runs + unasked and writes `qa-f08-child-new`, and the three are in its + `base_state.json`, not its `meta.json`. +- **Runtime values after a sandbox pause (`F08.reload-after-pause`).** + No restart: the pause call unloads every conversation (its lease file + goes), and an events read loads this one again. + ```sh + PZ=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W/knobs\"}, \"agent\": $OFFLINE_AGENT, \"autotitle\": false}" \ + --expect 201 --check confirmation_policy.kind eq NeverConfirm --check security_analyzer eq null --field id) + control-agent-server api POST "/api/conversations/$PZ/confirmation_policy" --json '{"policy": {"kind": "AlwaysConfirm"}}' --expect 200 + control-agent-server api POST "/api/conversations/$PZ/security_analyzer" --json '{"security_analyzer": {"kind": "PatternSecurityAnalyzer"}}' --expect 200 + control-agent-server state ls --conversation "$PZ" owner_lease.json --expect-count 1 + control-agent-server api POST /api/conversations/prepare-for-sandbox-pause --expect 204 + control-agent-server state ls --conversation "$PZ" owner_lease.json --expect-count 0 + control-agent-server api GET "/api/conversations/$PZ/events/count" --expect 200 + control-agent-server state ls --conversation "$PZ" owner_lease.json --expect-count 1 + control-agent-server api GET "/api/conversations/$PZ" --check confirmation_policy.kind eq AlwaysConfirm \ + --check security_analyzer.kind eq PatternSecurityAnalyzer --save F08.reload-after-pause/after-reload + control-agent-server state cat base_state.json --conversation "$PZ" --max-chars 200 \ + --check confirmation_policy.kind eq AlwaysConfirm --check security_analyzer.kind eq PatternSecurityAnalyzer + ``` + The reloaded conversation still has `AlwaysConfirm` and + `PatternSecurityAnalyzer`. +- **Upgrade from the latest release (`F08.upgrade-from-release`).** A second + run starts on the newest release tag. `$UC` is created there with a + start-time secret and nothing else, `$UA` with `AlwaysConfirm` and + `PatternSecurityAnalyzer`; neither is touched again before the same state + is restarted on this checkout. + + Needs: `network` + ```sh + HERE="${AGENT_SERVER_VERIFY_CHECKOUT:-$PWD}" + TAG=$(git -C "$HERE" describe --tags --abbrev=0 --match 'v*') + REL="$W/release-$TAG" + git -C "$HERE" worktree add --detach "$REL" "$TAG" >/dev/null 2>&1 + UP=$(control-agent-server launch --new --name f08-upgrade --checkout "$REL" --timeout 900 --print-run) + trap 'control-agent-server stop --run "$UP" >/dev/null; git -C "$HERE" worktree remove --force "$REL"' EXIT + UP_STUB=$(control-agent-server fixture llm-stub --run "$UP" --name qa-f08-upgrade \ + --step 'tool:terminal:{"command":"printf %s \"$QA_F08_UP\" > qa-f08-up.txt"}' --step "$FINISH" --print-path) + UC=$(control-agent-server api POST /api/conversations --run "$UP" \ + --json "{\"workspace\": {\"working_dir\": \"$W/upgrade\"}, \"agent\": {\"llm\": {\"model\": \"openai/qa-stub\", \"api_key\": \"qa-stub-key\", \"base_url\": \"$UP_STUB/v1\", \"num_retries\": 0}, \"tools\": [{\"name\": \"terminal\"}]}, \"autotitle\": false, \"secrets\": {\"QA_F08_UP\": {\"kind\": \"StaticSecret\", \"value\": \"qa-f08-up-value\"}}}" \ + --expect 201 --field id) + UA=$(control-agent-server api POST /api/conversations --run "$UP" \ + --json "{\"workspace\": {\"working_dir\": \"$W/knobs\"}, \"agent\": $OFFLINE_AGENT, \"autotitle\": false, \"confirmation_policy\": {\"kind\": \"AlwaysConfirm\"}, \"security_analyzer\": {\"kind\": \"PatternSecurityAnalyzer\"}}" \ + --expect 201 --field id) + control-agent-server restart --run "$UP" --checkout "$HERE" + control-agent-server conversation send "$UC" --run "$UP" --text 'Run the command.' --wait --until finished --timeout 120 + test "$(cat "$W/upgrade/qa-f08-up.txt")" = qa-f08-up-value + control-agent-server api GET "/api/conversations/$UC" --run "$UP" --check secret_registry.secret_sources.QA_F08_UP.kind eq StaticSecret \ + --save F08.upgrade-from-release/secret + control-agent-server state cat meta.json --run "$UP" --conversation "$UC" --max-chars 200 --check id exists --check secrets missing + control-agent-server api POST "/api/conversations/$UA/events" --run "$UP" --json "$QUEUED" --expect 200 --quiet + control-agent-server api GET "/api/conversations/$UA" --run "$UP" --check confirmation_policy.kind eq AlwaysConfirm \ + --check security_analyzer.kind eq PatternSecurityAnalyzer --save F08.upgrade-from-release/policy + ``` + On this checkout the command still gets `qa-f08-up-value`, and `$UA`, + loaded by its queued message, still has `AlwaysConfirm` and + `PatternSecurityAnalyzer`. A release that kept start-time secrets in + `meta.json` wrote `$UC`'s nowhere else (nothing saved the state after the + create), so this checkout takes it over on the first load; `meta.json` has + no `secrets` key afterwards. The first launch of a release worktree builds + its environment with `uv`, which needs the package index and can take + minutes; the trap stops the second server and removes the worktree. + ## Gotchas - `respond_to_confirmation` with `accept: true` is exactly a run request: it @@ -799,8 +1102,11 @@ KID=$(control-agent-server api POST /api/conversations \ `base_state.json`. Without `OH_SECRET_KEY` and without a session key there is no cipher: secrets (and the agent's LLM `api_key`) are saved as the `**********` placeholder and are gone after a restart, while GET looks the - same (`F08.secrets-no-cipher`). `launch` sets both keys; only - `launch --no-auth --no-secret-key` gives a run without a cipher. + same (`F08.secrets-no-cipher`). The loaded conversation is then the only + place that has the values, so such a server does not evict an idle + conversation that holds secrets or an analyzer with a key of its own; a + sandbox pause loses them as a restart does. `launch` sets both keys; only `launch --no-auth --no-secret-key` + gives a run without a cipher. - A `LookupSecret` is fetched with `httpx` from inside the server process, so its URL must be reachable from the server, not from the client. A `LookupSecret` header whose name contains `authorization`, `cookie`, @@ -822,13 +1128,23 @@ KID=$(control-agent-server api POST /api/conversations \ in the text of the model's reply (`F08.secrets-agent-message`). A sub-agent started by the task tool gets the conversation's secrets under the same rules (`F08.secrets-task-delegate`). -- Runtime secret changes made through `POST .../secrets` live only in - `base_state.json` (written before the POST returns) and are never written - to `meta.json`. A runtime secret with a new name therefore survives a - reload (restart, idle eviction, sandbox pause); see `F08.secrets-restart` - and `F08.secrets-unsaved-restart`. A new value for a name the conversation - was created with does not: the load re-applies the creation-time copy that - `meta.json` keeps. +- The confirmation policy, the security analyzer and the secrets live only + in `base_state.json`: the start body seeds them, the runtime routes write + them before answering, and `meta.json` never holds them + (`F08.state-on-disk`). A reload (restart, idle eviction, sandbox pause) + therefore keeps the current values (`F08.policy-restart`, + `F08.analyzer-restart`, `F08.analyzer-clear-restart`, + `F08.secrets-restart`, `F08.secrets-unsaved-restart`, + `F08.secrets-override-restart`, `F08.reload-after-pause`), and a fork or + a child starts from them (`F08.fork-after-restart`, + `F08.child-inherits-current`). +- A server from before this layout kept start-time secrets in `meta.json` + and could leave them out of `base_state.json`. The first load by a newer + server takes over the names `base_state.json` lacks, or none when it + cannot decrypt what the two files hold, and then drops the key + (`F08.upgrade-from-release`). Going back to such an older server is + not symmetric: it re-applies its own defaults on load, so the policy + becomes `NeverConfirm` and the analyzer is cleared. - Unknown conversations answer 404 `{"detail": "Not Found"}` on the conversation routes and 404 `Conversation not found: ` on `respond_to_confirmation` (an events route). diff --git a/.agents/skills/verify-agent-server/references/feature-map/F10-fork-navigate-condense-ask.md b/.agents/skills/verify-agent-server/references/feature-map/F10-fork-navigate-condense-ask.md index e2593f52e3b..1ef47654f38 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F10-fork-navigate-condense-ask.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F10-fork-navigate-condense-ask.md @@ -31,6 +31,7 @@ Routes: `POST /api/conversations/{conversation_id}/ask_agent`, - `F10.fork-errors`: an unknown source is 404, an unknown `from_event_id` is 404, a taken `id` is 409, a bad tag key or a title over 200 characters is 422, a malformed id is 422 and a missing key is 401; none of them creates a conversation. - `F10.fork-keeps-credentials`: the fork's `base_state.json` keeps the agent's (encrypted) model keys, like the source's. - `F10.fork-run-memory`: a fork of a finished DeepSeek conversation runs to `finished` with the source's history in context (it recalls a codename only the source was told). +- `F10.fork-keeps-settings`: a fork has the source's current confirmation policy, security analyzer and conversation secrets, including what was changed after the source was created, in its own `base_state.json`; changing them on the fork leaves the source alone. - `F10.navigate-event`: `POST /navigate` with an earlier `event_id` answers 200 with `leaf_event_id` set to it; GET and `base_state.json` agree and no event is deleted. - `F10.navigate-branch`: after navigating, the next appended event's `parent_id` is the new HEAD (a sibling of the abandoned branch), pushed live on the events socket; the HEAD move itself is not broadcast. - `F10.navigate-empty`: `{}` or `{"event_id": null}` selects the empty tree (`leaf_event_id` null, `head_is_empty` true on disk); the next appended event becomes a new root (`parent_id` `__root__`). @@ -294,6 +295,33 @@ SIDE_AGENT="{\"llm\": {\"model\": \"openai/qa-stub\", \"api_key\": \"qa-stub-key `LLMBadRequestError` rather than `LLMAuthenticationError` (the classification is `F20.validate-bad-key-type`), so the check matches `Authentication`, which both forms carry. +- **The fork keeps the safety settings and secrets (`F10.fork-keeps-settings`).** + A conversation is given a policy, an analyzer and a secret after it was + created, then forked. + ```sh + PS=$(control-agent-server conversation start --placeholder-agent --no-autotitle --no-run --prompt QA_F10_SETTINGS --print-id) + control-agent-server api POST "/api/conversations/$PS/confirmation_policy" \ + --json '{"policy": {"kind": "ConfirmRisky", "threshold": "MEDIUM", "confirm_unknown": false}}' --expect 200 + control-agent-server api POST "/api/conversations/$PS/security_analyzer" --json '{"security_analyzer": {"kind": "PatternSecurityAnalyzer"}}' --expect 200 + control-agent-server api POST "/api/conversations/$PS/secrets" --json '{"secrets": {"QA_F10_TOKEN": "qa-f10-token-value"}}' --expect 200 + PF=$(control-agent-server api POST "/api/conversations/$PS/fork" --expect 201 --field id) + control-agent-server api GET "/api/conversations/$PF" --check confirmation_policy.kind eq ConfirmRisky --check confirmation_policy.threshold eq MEDIUM \ + --check security_analyzer.kind eq PatternSecurityAnalyzer --check secret_registry.secret_sources.QA_F10_TOKEN.kind eq StaticSecret \ + --save F10.fork-keeps-settings/fork + control-agent-server state cat base_state.json --conversation "$PF" --max-chars 200 --check confirmation_policy.kind eq ConfirmRisky \ + --check security_analyzer.kind eq PatternSecurityAnalyzer --check secret_registry.secret_sources.QA_F10_TOKEN.value matches '^gAAAAA' \ + --not-contains qa-f10-token-value + control-agent-server state cat meta.json --conversation "$PF" --max-chars 200 --check id exists \ + --check confirmation_policy missing --check security_analyzer missing --check secrets missing + control-agent-server api POST "/api/conversations/$PF/confirmation_policy" --json '{"policy": {"kind": "NeverConfirm"}}' --expect 200 + control-agent-server api GET "/api/conversations/$PF" --check confirmation_policy.kind eq NeverConfirm + control-agent-server api GET "/api/conversations/$PS" --check confirmation_policy.kind eq ConfirmRisky --save F10.fork-keeps-settings/source + ``` + The fork answers with `ConfirmRisky` at `MEDIUM`, `PatternSecurityAnalyzer` + and `QA_F10_TOKEN`, none of which the source had when it was created. They + are in the fork's `base_state.json` (the secret as cipher text) and not in + its `meta.json`. Switching the fork to `NeverConfirm` leaves the source on + `ConfirmRisky`. - **Navigate to an earlier event (`F10.navigate-event`).** Move `S`'s HEAD back to its first message. ```sh @@ -597,8 +625,10 @@ SIDE_AGENT="{\"llm\": {\"model\": \"openai/qa-stub\", \"api_key\": \"qa-stub-key own persistence directory, lease and event log only. - A fork's `tags` are the requested tags, or an empty set when omitted, in both the response and persisted metadata. The title is stored separately - and adds no tag (#4814). Other stored configuration (secrets, plugins, - `parent_conversation_id`, client tools) is copied from the source. The + and adds no tag (#4814). Other stored configuration (plugins, + `parent_conversation_id`, client tools) is copied from the source, and + the policy, the analyzer and the secrets come with the fork's own + `base_state.json` (`F10.fork-keeps-settings`). The fork's `last_user_message_id` is the last user message among the events it copied, and what a hook decided about a copied action or message (`blocked_actions`, `blocked_messages`) comes with them. diff --git a/.agents/skills/verify-agent-server/references/feature-map/F12-runtime-and-credentials.md b/.agents/skills/verify-agent-server/references/feature-map/F12-runtime-and-credentials.md index b204e8d50d6..7bc0b197d46 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F12-runtime-and-credentials.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F12-runtime-and-credentials.md @@ -55,7 +55,7 @@ Routes: `GET /api/conversations/{conversation_id}/runtime`, - `F12.binding-source-unavailable`: a source answering 5xx is probed three times with backoff (at least 0.75 s), then the activation fails with 502; an unreachable source also gives 502. - `F12.binding-native-too-late`: binding a loaded regular (non-ACP) conversation is refused with 409 `Credential binding activation arrived after ACP initialization` and records nothing. - `F12.binding-native-cold`: a binding sent while a regular (non-ACP) conversation is not loaded never makes that conversation demand a credential binding after the next pause. -- `F12.binding-live-acp`: binding a loaded ACP Codex conversation that has not run yet answers 204, records `required_runtime_credential_bindings: ["CODEX_AUTH_JSON"]` in `meta.json` and removes the stored `CODEX_AUTH_JSON` secret. +- `F12.binding-live-acp`: binding a loaded ACP Codex conversation that has not run yet answers 204, records `required_runtime_credential_bindings: ["CODEX_AUTH_JSON"]` in `meta.json` and removes the stored `CODEX_AUTH_JSON` secret from `base_state.json`. - `F12.binding-rebind`: a second activation with the same URL and new headers is accepted (the probe carries the new headers, which are never written to disk); a different URL is refused with 409. - `F12.binding-required-after-pause`: after a pause, a conversation that requires a binding is still listed and reports runtime `available`, but event access and `/run` answer 409 `{"detail": "credential_binding_activation_required", "retryable": true}` and both conversation sockets close with 1013, until a new activation, after which events load again. - `F12.binding-required-after-restart`: the requirement survives a restart (409 until a new activation, then 200), and deleting such a conversation needs no activation. @@ -507,20 +507,23 @@ control-agent-server api GET "/api/conversations/$R" --check execution_status eq ```sh A=$(control-agent-server conversation start --body-json "{\"agent\": $ACP_AGENT}" --secret CODEX_AUTH_JSON=qa-f12-request-copy --no-autotitle --print-id) control-agent-server api GET "/api/conversations/$A" --check agent.kind eq ACPAgent --check agent.acp_server eq codex - control-agent-server state cat meta.json --conversation "$A" --check secrets.CODEX_AUTH_JSON exists --check required_runtime_credential_bindings len-eq 0 + control-agent-server state cat meta.json --conversation "$A" --check required_runtime_credential_bindings len-eq 0 + control-agent-server state cat base_state.json --conversation "$A" --max-chars 200 --check secret_registry.secret_sources.CODEX_AUTH_JSON exists control-agent-server api PUT "/api/conversations/$A/credential-bindings/CODEX_AUTH_JSON" \ --json "{\"url\": \"$SRC/cred\", \"headers\": {\"Authorization\": \"Bearer qa-f12-first\"}}" --expect 204 --save F12.binding-live-acp/put control-agent-server sink read --name qa-f12-cred --contains 'Bearer qa-f12-first' --expect-min 1 --expect-max 1 - control-agent-server state cat meta.json --conversation "$A" \ - --check required_runtime_credential_bindings eq '["CODEX_AUTH_JSON"]' --check secrets.CODEX_AUTH_JSON missing + control-agent-server state cat meta.json --conversation "$A" --check required_runtime_credential_bindings eq '["CODEX_AUTH_JSON"]' + control-agent-server state cat base_state.json --conversation "$A" --max-chars 200 \ + --check agent.kind eq ACPAgent --check secret_registry.secret_sources.CODEX_AUTH_JSON missing control-agent-server state grep qa-f12-request-copy --glob 'server/**/*' --expect-none control-agent-server state grep qa-f12-request-copy --glob 'home/**/*' --expect-none control-agent-server api GET "/api/conversations/$A/events/count" --expect 200 ``` - 204; `meta.json` now records the requirement and no longer holds the - conversation's own `CODEX_AUTH_JSON` secret (stored encrypted before, so - the plaintext greps hold either way; the `secrets` check is the proof of - the scrub), and the conversation stays readable. + 204; `meta.json` now records the requirement and `base_state.json` no + longer holds the conversation's own `CODEX_AUTH_JSON` secret (stored + encrypted before, so the plaintext greps hold either way; the + `secret_registry` check is the proof of the scrub), and the conversation + stays readable. - **Re-bind (`F12.binding-rebind`).** Same URL with new headers, then a different URL. @@ -591,8 +594,9 @@ control-agent-server api GET "/api/conversations/$R" --check execution_status eq control-agent-server api PUT "/api/conversations/$U2/credential-bindings/CODEX_AUTH_JSON" --json "{\"url\": \"$SRC/cred\"}" --expect 204 C2=$(control-agent-server conversation start --body-json "{\"conversation_id\": \"$U2\", \"agent\": $ACP_AGENT}" --secret CODEX_AUTH_JSON=qa-f12-create-copy --no-autotitle --print-id) test "$C2" = "$U2" - control-agent-server state cat meta.json --conversation "$U2" \ - --check required_runtime_credential_bindings eq '["CODEX_AUTH_JSON"]' --check secrets.CODEX_AUTH_JSON missing + control-agent-server state cat meta.json --conversation "$U2" --check required_runtime_credential_bindings eq '["CODEX_AUTH_JSON"]' + control-agent-server state cat base_state.json --conversation "$U2" --max-chars 200 \ + --check agent.kind eq ACPAgent --check secret_registry.secret_sources.CODEX_AUTH_JSON missing control-agent-server state grep qa-f12-create-copy --glob 'server/**/*' --expect-none control-agent-server state grep qa-f12-create-copy --glob 'home/**/*' --expect-none ``` @@ -606,8 +610,8 @@ control-agent-server api GET "/api/conversations/$R" --check execution_status eq control-agent-server api POST /api/conversations/prepare-for-sandbox-pause --expect 204 C3=$(control-agent-server conversation start --body-json "{\"conversation_id\": \"$U3\", \"agent\": $ACP_AGENT}" --secret CODEX_AUTH_JSON=qa-f12-kept-copy --no-autotitle --print-id) test "$C3" = "$U3" - control-agent-server state cat meta.json --conversation "$U3" \ - --check required_runtime_credential_bindings len-eq 0 --check secrets.CODEX_AUTH_JSON exists + control-agent-server state cat meta.json --conversation "$U3" --check required_runtime_credential_bindings len-eq 0 + control-agent-server state cat base_state.json --conversation "$U3" --max-chars 200 --check secret_registry.secret_sources.CODEX_AUTH_JSON exists ``` Compare with `F12.binding-pending-on-create`: the pending binding is gone, so the conversation keeps its own (encrypted) secret and needs no binding. diff --git a/.agents/skills/verify-agent-server/references/feature-map/F31-persistence-and-limits.md b/.agents/skills/verify-agent-server/references/feature-map/F31-persistence-and-limits.md index e53874ea4e3..cba99144264 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F31-persistence-and-limits.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F31-persistence-and-limits.md @@ -55,8 +55,9 @@ Routes: none server on the same conversations directory (`launch --new --share-conversations-from RUN`). - Persisted layout, read with `state ls|cat|grep`: per conversation - `//meta.json` (title, tags, encrypted secrets), - `base_state.json` (execution status, agent, encrypted secret registry), + `//meta.json` (title, tags), + `base_state.json` (execution status, agent, confirmation policy, security + analyzer, encrypted secret registry), `events/` and, while it is loaded, `owner_lease.json` (`owner_instance_id`, `generation`, `expires_at`, `owner_host`, `owner_pid`) guarded by `.owner_lease.lock`; under `OH_PERSISTENCE_DIR` diff --git a/.agents/skills/verify-agent-server/references/feature-map/README.md b/.agents/skills/verify-agent-server/references/feature-map/README.md index 7fcaa36b8f5..c53b11c2602 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/README.md +++ b/.agents/skills/verify-agent-server/references/feature-map/README.md @@ -10,7 +10,7 @@ recipe. Maintenance baseline: main@0275e714df62d3dd66eb51d518dd1da953ec2c16 (2026-11-17). The next maintenance pass starts from this commit, a pass proposes the next baseline in its PR, and merging that PR accepts it. -The map has **1068 sub-features**. +The map has **1079 sub-features**. ## Baseline preconditions @@ -72,9 +72,9 @@ them honest: [../maintenance.md](../maintenance.md). | F05 | [Run, pause, interrupt and status transitions](F05-run-pause-interrupt.md) | run, pause and interrupt during model and tool calls; status transitions on every view; stuck detection and max_iterations stops; 409 and 429; graceful and SIGKILL restarts | `POST /api/conversations/{id}/run`, `/pause`, `/interrupt` | llm, tmux; launch `--config-json '{"max_concurrent_runs": 1}'` | 32 | | F06 | [Conversation events and the events WebSocket](F06-conversation-events.md) | appending messages, event search, count, get and batch get (GET with a body, or the browser-safe POST) with filters and paging, the events WebSocket (auth, snapshot, replay, live frames), history across restart | `POST`/`GET /api/conversations/{id}/events[...]`, `POST /api/conversations/{id}/events/batch`, `WS /sockets/events/{id}` | llm; launch `--env TZ=UTC` | 33 | | F07 | [Session WebSocket protocol](F07-session-socket.md) | the resumable session socket: sync, durable and transient frames, seq replay, inbound messages, error and close codes, streaming deltas, backpressure, subscriber cap | `WS /sockets/session/{id}` | llm | 27 | -| F08 | [Confirmation policy, security analyzer and conversation secrets](F08-confirmation-security-secrets.md) | confirmation policies and accept/reject, the security analyzer, runtime conversation secrets (static, lookup, masking, encryption, no cipher), and what survives a restart | `POST /api/conversations/{id}/confirmation_policy`, `/security_analyzer`, `/secrets`, `/events/respond_to_confirmation` | llm, tmux | 26 | +| F08 | [Confirmation policy, security analyzer and conversation secrets](F08-confirmation-security-secrets.md) | confirmation policies and accept/reject, the security analyzer, runtime conversation secrets (static, lookup, masking, encryption, no cipher), and what survives a restart | `POST /api/conversations/{id}/confirmation_policy`, `/security_analyzer`, `/secrets`, `/events/respond_to_confirmation` | llm, tmux | 36 | | F09 | [Model, profile and ACP switching; runtime plugins](F09-model-switching-plugins.md) | swap the agent LLM, switch to a saved profile, switch an ACP model, load a plugin at runtime; stats, hooks and restart persistence | `POST /api/conversations/{id}/switch_llm`, `/switch_profile`, `/switch_acp_model`, `/load_plugin` | llm, tmux, node, network | 49 | -| F10 | [Fork, navigate, condense and ask_agent](F10-fork-navigate-condense-ask.md) | fork, navigate between branches, condense with a real summarizer, ask the agent a side question; state after restart | `POST /api/conversations/{id}/fork`, `/navigate`, `/condense`, `/ask_agent` | llm | 26 | +| F10 | [Fork, navigate, condense and ask_agent](F10-fork-navigate-condense-ask.md) | fork, navigate between branches, condense with a real summarizer, ask the agent a side question; state after restart | `POST /api/conversations/{id}/fork`, `/navigate`, `/condense`, `/ask_agent` | llm | 27 | | F11 | [Goal loop](F11-goals.md) | goal loops: start, LLM-judged completion and caps, stop, resume, supersede, interrupt, failures, busy and capacity limits | `POST /api/conversations/{id}/goal`, `/goal/stop`, `/goal/resume` | llm, tmux; launch `--config-json '{"max_concurrent_runs": 1}'` | 25 | | F12 | [Conversation runtime info, sandbox pause and credential bindings](F12-runtime-and-credentials.md) | runtime info and reprovision, prepare-for-sandbox-pause, credential bindings for ACP agents, managed LLM key refresh, the Docker runtime mode's routes and contract | `GET /api/conversations/{id}/runtime`, `POST .../runtime/reprovision`, `POST /api/conversations/prepare-for-sandbox-pause`, `PUT .../credential-bindings/{name}`, Docker mode `DELETE .../runtime`, `POST .../runtime/credentials` | llm, node, network | 31 | | F13 | [Bash commands and bash events](F13-bash.md) | global and conversation-scoped shell commands: execute, start, stop, timeouts, large output, events search and clear, retention, the bash events socket | `/api/bash/*`, `/api/conversations/{id}/bash/*`, `WS /sockets/bash-events` | baseline | 29 | diff --git a/.pr/single-owner/README.md b/.pr/single-owner/README.md new file mode 100644 index 00000000000..aa7dc116475 --- /dev/null +++ b/.pr/single-owner/README.md @@ -0,0 +1,188 @@ +# One owner for policy, analyzer and secrets: verification record + +How the stack #6762 → #6763 → #6764 was verified. This is a temporary PR +artifact (see `AGENTS.md`, "PR-Specific Evidence Documents"); the next release PR +removes it. + +- Base: `main` at `3d638f2e`, merged into all three branches. +- Code under test: part 1 `6d9b169d`, part 2 `bba1de01`, part 3 `578a8978`. The commits + after `578a8978` replace the previous round's record in `.pr/single-owner/` + (this file and its screenshots) and change nothing else. +- Every agent-server attempt launches its own server + (`control-agent-server map run --fresh`); "restart" is a real process + restart on the same state directory. Those runs were wrapped in + `/usr/libexec/docker/docker-init -s --`, because PID 1 in the sandbox is + slow to reap orphaned tmux servers. The SDK-map bullets (`S04`, `S13`) run + in-process, without a server and without the wrapper. +- Model-backed bullets used DeepSeek flash through the environment's LLM + proxy (section 2 has a note on that). The rest use the skill's scripted + `llm-stub` and need no key. +- `xfail`: the bullet failed with exactly the failure its known-bug marker + documents. `xpass`: that failure did not reproduce. +- The UI check (section 3) ran on the same commits, `main` at `3d638f2e` and part 3 at `578a8978`, through Agent Canvas built from `OpenHands/OpenHands` at `b759f8a7`. + +## 1. The bug on `main` and the fix + +Parts 1 and 3 on live servers; part 1's two `S04` rows and part 2 in the SDK, +in-process. + +### Part 1 (#6762): a secret is saved the moment it changes + +The three server bullets were replayed with `main`'s map (which still has +the markers), against a server from `main` and then from part 1 +(`--checkout`). The fourth row is part 1's recipe and program in a copy of +`main`, run on `main`'s SDK, then on part 1; the fifth is the map's examples +bullet, which the same selection also runs. Two attempts a side. + +| Bullet | `main` | part 1 (`6d9b169d`) | +|---|---|---| +| `F04.create-secrets-get` | xfail, xfail | xpass, xpass | +| `F08.secrets-visible` | xfail, xfail | xpass, xpass | +| `F08.secrets-unsaved-restart` | xfail, xfail | xpass, xpass | +| `S04.secrets-update`, `S04.secrets-constructor` | fail, fail (5 of 17 checks fail, the same 5 in each attempt) | pass, pass | +| `S04.secrets-update` again: the map's examples bullet (examples 09 to 12 against a live model; it checks the examples' output, not what `base_state.json` holds after a secrets call) | pass, pass | pass, pass | + +### Part 2 (#6763): a fork keeps the state + +`main` keeps a fork's cipher and secrets since #5765. What it still loses is +shown by part 2's recipe and program in a copy of `main`, run on `main`'s +SDK, then on part 2. Two attempts a side. + +| Bullet | `main` | part 2 (`bba1de01`) | +|---|---|---| +| `S04.fork-keeps-state` | fail, fail (11 of 21 checks fail, the same 11 in each attempt) | pass, pass | + +The checks that fail on `main`: + +- `fork_policy.kind eq ConfirmRisky`: got `NeverConfirm` +- `fork_policy.threshold eq MEDIUM`: got `null` +- `fork_policy.confirm_unknown eq false`: got `null` +- `fork_analyzer eq ToolShieldLLMSecurityAnalyzer`: got `NoneType` +- `fork_guard_key_kept eq true`: got `false` +- `saved_policy.kind eq ConfirmRisky`: got `NeverConfirm` +- `saved_analyzer eq ToolShieldLLMSecurityAnalyzer`: got `null` +- `saved_guard_key_is_placeholder eq false`: got `true` +- `fork_prompt_block eq blocked by the source hook`: got `null` +- `reloaded_guard_key_kept eq true`: got `false` +- `reloaded_secrets.FORK_ONLY eq fork-only-value-4`: got `null` + +Three of these follow from the missing analyzer rather than from a key that was written badly: `main` saves no analyzer for the fork, so there is no guard key at all, and the program reports a missing key as a placeholder (`fork_guard_key_kept`, `saved_guard_key_is_placeholder`, `reloaded_guard_key_kept`). + +On part 1's code (`main` plus #6762), which part 2 is based on, the same recipe fails 10 of these, twice. The one that passes there is `reloaded_secrets.FORK_ONLY eq fork-only-value-4`: a secret added to the fork is still there after a reload, which is #6762's fix, not #6763's. + +### Part 3 (#6764): `meta.json` no longer holds a second copy + +Replayed with a copy of `main`'s map that holds part 3's recipes with a +known-bug marker added, against a server from `main` and then from part 3. +One recipe is adapted further: in the copy, `F08.secrets-override-restart` +builds its own stub agent (`main`'s map has no `$ROTR_AGENT`), and it saves +the state with a tags PATCH and checks `base_state.json` before the restart, +so that on `main` it is the `meta.json` overwrite that fails it and not the +unsaved secret of part 1. + +| Bullet | `main` | part 3 (`578a8978`) | +|---|---|---| +| `F08.state-on-disk` | xfail, xfail | xpass, xpass | +| `F08.policy-restart` | xfail, xfail | xpass, xpass | +| `F08.analyzer-restart` | xfail, xfail | xpass, xpass | +| `F08.analyzer-clear-restart` | xfail, xfail | xpass, xpass | +| `F08.secrets-override-restart` | xfail, xfail | xpass, xpass | +| `F08.fork-after-restart` | xfail, xfail | xpass, xpass | +| `F08.child-inherits-current` | xfail, xfail | xpass, xpass | +| `F08.reload-after-pause` | xfail, xfail | xpass, xpass | + +## 2. Families on part 3's code, and the same on `main` + +Seven whole families, and from each of the two client families three +bullets that touch this change, with the one bullet they require. In the +five agent-server rows the totals include the file's preconditions step as +one entry, as the tool's own summary does; it passes on both sides. + +| Family | part 3 (`578a8978`) | `main` (`3d638f2e`), its own map (`F31`, `F33`, `F34`: adapted to the proxy, see the note) | +|---|---|---| +| `F08` confirmation, security, secrets | 36 of 37 pass; 1 xfail that `main` has too (`F08.secrets-git-provider`) | 24 of 27 pass, 3 xfail | +| `F10` fork, navigate, condense, ask | 28 of 28 pass | 27 of 27 pass | +| `F04` conversation lifecycle | 40 of 41 pass; 1 xfail that `main` has too (`F04.create-initial-message-run-false`) | 39 of 41 pass, 2 xfail | +| `F12` runtime and credentials | 29 of 32 pass; 1 xfail that `main` has too (`F12.sandbox-pause-websocket`); `F12.docker-runtime` blocked (needs a Docker daemon); `F12.docker-startup-timeout` blocked (needs a Docker daemon) | 29 of 32 pass, 1 xfail, 2 blocked | +| `F31` persistence and limits | 23 of 25 pass; 1 xfail that `main` has too (`F31.takeover-keeps-metadata`); `F31.storage-retention` blocked (needs a Docker daemon) | 23 of 25 pass, 1 xfail, 1 blocked | +| `F33` Python `RemoteConversation`: 4 of its 41 bullets (fork, confirmation, secrets, and the run bullet that fork requires) | the 4 bullets pass, and their preconditions | the 4 bullets pass, and their preconditions | +| `F34` TypeScript client: 4 of its 38 bullets (run, fork/navigate/ask, confirm, and the create bullet they require) | the 4 bullets pass, and their preconditions | the 4 bullets pass, and their preconditions | +| `S04` local conversation (SDK map) | 32 of 36 pass; 3 fail that also fail on `main` (`S04.pause-during-startup`, `S04.goal-asked-branch`, `S04.interactive-terminal`); `S04.factory` labelled blocked in the map | 31 of 35 pass, 3 fail, 1 blocked | +| `S13` security and confirmation (SDK map) | 8 of 10 pass; the live-model bullet (`S13.llm-analyzer`, `S13.confirm-wait`, `S13.confirm-approve-reject`) failed in this run: its result is the model's own risk label, see the note below; `S13.confirm-approve-reject`, `S13.llm-analyzer` labelled blocked in the map | 9 of 10 pass, 1 blocked | + +Every bullet that does not pass on part 3 does not pass on `main` either (compared bullet by bullet), with one bullet counted apart because its result is a live model's own judgement: the note on `S13` below. + +`main` has 3 more that do not pass: the bullets its map marks as known bugs, which this stack fixes (`F08.secrets-visible`, `F08.secrets-unsaved-restart`, `F04.create-secrets-get`). + +What does not pass on either side: + +- known bugs that `main`'s map marks and this stack does not touch (`xfail` on both sides): `F08.secrets-git-provider` (#6814), `F04.create-initial-message-run-false`, `F12.sandbox-pause-websocket`, `F31.takeover-keeps-metadata` +- blocked on both sides, because this machine has no Docker daemon: `F12.docker-runtime`, `F12.docker-startup-timeout`, `F31.storage-retention` +- labelled blocked in the map itself: `S04.factory` (it needs a running agent-server), `S13.confirm-approve-reject` (in its interactive examples, which need a person at a terminal), `S13.llm-analyzer` (the same examples) +- fail on both sides in the same way (the notes below say how): `S04.pause-during-startup`, `S04.goal-asked-branch`, `S04.interactive-terminal` + +A note on the model route. The environment of this run has an LLM proxy (`LLM_BASE_URL`, `LLM_API_KEY`) and no `DEEPSEEK_API_KEY`. The skills handle that by themselves: `llm preset deepseek` and `control-sdk` then use the flash model the proxy serves, so `F04`, `F08`, `F10`, `F12` and the SDK families ran as they are. The recipes of `F31`, `F33` and `F34` name DeepSeek's own API (`deepseek/deepseek-flash` with `$DEEPSEEK_API_KEY`). For those three files, on both sides, I ran copies in which only the model name, the key variable and the base URL are changed to the proxy's, and one check counts the single profile that preset makes through a proxy (two with DeepSeek's key); the rows above are those runs. Run as written, the same bullets fail on part 3 and on `main` alike, because this environment has the proxy's variables and no DeepSeek key. The `F33` and `F34` recipes call DeepSeek's API with an empty key: its `authentication_error` shows directly, as a conversation that ends in `error`, or as the missing conversation id after the failed run bullet. In `F31`, `llm preset deepseek` goes through the proxy whenever `LLM_BASE_URL` and `LLM_API_KEY` are set, so two settings checks that expect the direct model and DeepSeek's key fail: `F31`: `F31.state-survives-restarts`, `F31.no-plaintext-at-rest`; `F33`: `F33.conversation-run` with `F33.conversation-callbacks` with `F33.conversation-stats`, `F33.conversation-fork`, `F33.conversation-confirmation`, `F33.conversation-secrets`; `F34`: `F34.conversation-run`, `F34.conversation-branching`, `F34.conversation-confirm`. + +`F08.upgrade-from-release` is part of the `F08` run: conversations are created by the latest released server (1.63.0), which leaves a start-time secret only in `meta.json`, and the same state directory is then restarted on part 3's code. The secret, the policy and the analyzer are all there. Replayed on its own, two attempts each: pass, pass on part 3's code, and fail, fail on the same code with the legacy take-over removed (one added `return {}`): there the command that reads the secret gets an empty value, so the recipe's comparison with the value the released server stored fails. + +The SDK map families on part 1 (`6d9b169d`): + +| Family | Result | +|---|---| +| `S04` local conversation (SDK map) | 31 of 35 pass; 3 fail that also fail on `main` (`S04.pause-during-startup`, `S04.goal-asked-branch`, `S04.interactive-terminal`); `S04.factory` labelled blocked in the map | +| `S13` security and confirmation (SDK map) | 9 of 10 pass; `S13.confirm-approve-reject`, `S13.llm-analyzer` labelled blocked in the map | + +and on part 2 (`bba1de01`): + +| Family | Result | +|---|---| +| `S04` local conversation (SDK map) | 32 of 36 pass; 3 fail that also fail on `main` (`S04.pause-during-startup`, `S04.goal-asked-branch`, `S04.interactive-terminal`); `S04.factory` labelled blocked in the map | +| `S13` security and confirmation (SDK map) | 9 of 10 pass; `S13.confirm-approve-reject`, `S13.llm-analyzer` labelled blocked in the map | + +`S04.interactive-terminal` ends with an integration test whose command asks for the model `deepseek/deepseek-flash` by name. This run has an LLM proxy and no DeepSeek key, so the request goes to the proxy, which serves the flash model under another name and answers 400 (`There are no healthy deployments for this model`). The example step before it passes, and the bullet fails the same way on `main`. + +`S04.pause-during-startup` and `S04.goal-asked-branch` are bullets `main`'s map maintenance added this week (#6869, #6931); they fail the same checks on `main` (compared check by check) and concern pausing and goals, which this change does not touch. + +The live-model bullet of `S13` asks a real model to run `rm -rf scratch` under `ConfirmRisky()`, and it passes only when the model itself labels that command `HIGH`. `main`'s map says of it: "this bullet can fail on any revision without a product change; read the `risks` it printed". Counting the family run on each side and 16 further attempts that alternate between the two sides, it passed 12 of 17 attempts (4 failed, 1 blocked because the model never proposed the deletion) on `main` and 8 of 17 attempts (9 failed) on part 3; in the 16 alternating attempts alone, 11 and 8. In every failing attempt, on either side, the model labelled the deletion `MEDIUM`, nothing was labelled `HIGH` and nothing stopped; in every passing one it labelled the deletion `HIGH` and the run stopped for confirmation. The first request of that conversation (system prompt, message, tool schemas, parameters), captured by a recorder script that builds the bullet's conversation on each checkout with a placeholder key and stops before sending, is byte-identical on `main` and on all three branches (sha256 `1d9b75350631318d…`). In run order, the family run first (P pass, F fail, B blocked): `main` `PFFPPFPPPFBPPPPPP`, part 3 `FFFFFFPPPPPPFFPPF`. The scripted bullet next to it, which feeds fixed labels through the same analyzer and policy (`LOW` runs, `HIGH` waits, an unlabelled action waits), passes on both sides. + +## 3. Through the Agent Canvas UI + +The full OpenHands UI (Agent Canvas, built from the `OpenHands/OpenHands` repository at `b759f8a7`) was run with that repository's `verify-openhands` skill (`control-openhands launch --sdk-path `), once on `main`'s SDK and agent-server (`3d638f2e`) and once on part 3's code commit (`578a8978`), with a real model: the flash model that the environment's LLM proxy serves, set up through the UI's own LLM form. The Canvas has no control that changes confirmation mode on an existing conversation, and it only ever sends lookup secrets, so two steps were made through the REST API: that switch, and the replacement of the secret's value. Everything else was typed and clicked in the UI, and each stack was restarted for real. + +| Scenario, observed in the UI | `main` | part 3 | +|---|---|---| +| Conversation switched to `AlwaysConfirm`, then a backend restart, then a new command | ran without asking, in 2 of 2 attempts | waited for approval, in 2 of 2 attempts; Continue ran it both times, and Cancel rejected the next one | +| Secret replaced at runtime (18 bytes → 25), then a restart; length read by the agent (1 attempt a side) | 18: the old value is back | 25 | +| New conversation started in confirmation mode: reject, approve, restart, next command (1 attempt a side) | waits (creation-time value) | waits | +| `meta.json` at every read-back of the four scenario conversations | had `confirmation_policy`, `security_analyzer`, `secrets` in 30 of 30 read-backs | had none of them in 32 of 32 | + +![policy after a restart](ui-policy-after-restart.png) + +![replaced secret after a restart](ui-secret-after-restart.png) + +On `main` the revert is also written back: after the restart and the next prompt, `base_state.json` holds the creation-time policy again. In one of the two attempts the file still said `AlwaysConfirm` at the first read-back after the restart, and the command ran unasked all the same. While the Add-secret form is being filled, the typed value is in that masked field (the LLM form holds its key the same way); otherwise no secret value appears in plain text in the page (text and DOM), in the API events or in the conversation's state directory, on either side, and the screenshots show only lengths. One place does hold the value on both sides, and this change neither causes nor fixes it: the shell history of the agent's terminal, in the run's home directory, keeps the `export` lines with which the terminal tool gives a command its secret. An independent pass recounted every number in this table from the saved read-backs and step logs. + +## 4. Checks CI runs only for PRs into `main`, and two further comparisons, run on part 3 + +- The exported OpenAPI document (`export_agent_server_openapi.py`) is byte-identical to `main`'s. +- REST API breakage check (`check_agent_server_rest_api_breakage.py`, with oasdiff 1.19.1): no breaking changes against the 1.63.0 release; oasdiff's list is empty, so nothing had to pass through the script's allowlists (the check reports breaking changes only). +- Persisted settings compatibility (`check_persisted_settings_compat.py`): passes, with the 8 payloads of the 1.63.0 release. +- SDK API breakage check (`check_sdk_api_breakage.py`): no breaking changes in the three packages. Its ACP version check needs CI's base ref and was skipped here; the `agent-client-protocol` pin is the same as on `main`. +- `make test-server-schema`: passes. +- The agent-server code of the 1.63.0 release (tag `v1.63.0`) starts a conversation on part 3's SDK. + +## 5. Reproduce + +```sh +export PATH="$PWD/.agents/skills/verify-agent-server/scripts:$PWD/.agents/skills/verify-sdk/scripts:$PATH" + +# the fix, from a checkout of part 3 +control-agent-server map run --file F08 --fresh +control-agent-server map run --file F10 --fresh +sdk-map run --file S04 --only S04.secrets-update,S04.secrets-constructor,S04.fork-keeps-state + +# the same bullets against main: a clean main worktree with its own `uv sync --dev` +# (part 3's map has no known-bug markers, so they report a plain `fail` there) +control-agent-server map run --file F08 --only F08.state-on-disk,F08.policy-restart,F08.secrets-override-restart \ + --fresh --repeat 2 --checkout ../main +``` diff --git a/.pr/single-owner/ui-policy-after-restart.png b/.pr/single-owner/ui-policy-after-restart.png new file mode 100644 index 00000000000..a9f97f017b5 Binary files /dev/null and b/.pr/single-owner/ui-policy-after-restart.png differ diff --git a/.pr/single-owner/ui-secret-after-restart.png b/.pr/single-owner/ui-secret-after-restart.png new file mode 100644 index 00000000000..f87662fcd4e Binary files /dev/null and b/.pr/single-owner/ui-secret-after-restart.png differ diff --git a/openhands-agent-server/openhands/agent_server/README.md b/openhands-agent-server/openhands/agent_server/README.md index 39362bb6d0d..ba30790cd5d 100644 --- a/openhands-agent-server/openhands/agent_server/README.md +++ b/openhands-agent-server/openhands/agent_server/README.md @@ -313,6 +313,7 @@ If `OH_SECRET_KEY` is not set: - Secrets will be redacted (masked) in stored conversations - When the server restarts, encrypted secrets cannot be decrypted and will be `None` - Conversations will need to be recreated with fresh API keys +- With no session API key either, the server has no encryption key at all, and a conversation's secret values exist only in its loaded state. A conversation that holds secrets, or a security analyzer with a key of its own, is then not unloaded when idle. A sandbox pause loses the values, as a restart does. A fork receives its source's secrets from memory, but not the keys inside the agent or the security analyzer. ### Webhook Configuration diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index d5d67fa5ec1..a91abbbc696 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -41,6 +41,8 @@ is_codex_source, launch_source, live_launch_runtime, + request_secrets, + request_security_analyzer, scoped_secrets, server_launch_stores, ) @@ -109,10 +111,20 @@ from openhands.sdk.llm.llm_profile_store import LLMProfileStore from openhands.sdk.mcp.utils import MCPToolProvider from openhands.sdk.observability import OPERATION_METADATA_KEY, observe +from openhands.sdk.secret import SecretSource +from openhands.sdk.security.analyzer import SecurityAnalyzerBase +from openhands.sdk.security.confirmation_policy import ( + ConfirmationPolicyBase, + NeverConfirm, +) from openhands.sdk.tool.client_tool import register_client_tools from openhands.sdk.tool.registry import get_tool_module_qualnames, is_tool_available from openhands.sdk.utils.cipher import Cipher -from openhands.sdk.utils.pydantic_secrets import cipher_context, client_context +from openhands.sdk.utils.pydantic_secrets import ( + cipher_context, + client_context, + copy_keeping_secrets, +) from openhands.sdk.workspace import LocalWorkspace @@ -463,6 +475,31 @@ def _update_state_tags_sync( return state +def _inheritable_state_config( + state: ConversationState, +) -> tuple[ + ConfirmationPolicyBase, SecurityAnalyzerBase | None, dict[str, SecretSource] +]: + """Copy the base_state.json-owned config a child conversation inherits. + + Reads without the state lock: a parent starts a child from inside its own + step, which holds that lock until the start returns, so waiting for it + here would never end. Each of the three is replaced on change, never + mutated in place, so a plain read sees a whole value. + + The child gets its own analyzer and secret sources, with their + credentials, for the child's cipher to encrypt again. + """ + analyzer = state.security_analyzer + if analyzer is not None: + analyzer = copy_keeping_secrets(analyzer) + secrets = { + name: copy_keeping_secrets(source) + for name, source in dict(state.secret_registry.secret_sources).items() + } + return state.confirmation_policy, analyzer, secrets + + def _compose_webhook_conversation_info_sync( stored: StoredConversation, state: ConversationState ) -> ConversationInfo: @@ -1164,6 +1201,7 @@ async def _get_or_load_event_service_locked( *, require_runtime_bindings: bool = True, agent: AgentBase | None = None, + secrets: dict[str, SecretSource] | None = None, ) -> EventService | None: event_services = self._event_services if event_services is None: @@ -1190,7 +1228,9 @@ async def _get_or_load_event_service_locked( await asyncio.to_thread(self._prepare_persisted_runtime, record.stored) try: - return await self._start_event_service(record.stored, persisted_agent=agent) + return await self._start_event_service( + record.stored, persisted_agent=agent, secrets=secrets + ) except ConversationLeaseHeldError as exc: logger.debug( "Skipping active conversation %s owned by %s until %s", @@ -1442,7 +1482,7 @@ async def _reuse_existing_conversation_locked( update={ "secrets": { name: value - for name, value in request.secrets.items() + for name, value in request_secrets(request, self.cipher).items() if self._profile_allows_secret(stored, name) } } @@ -1508,28 +1548,14 @@ async def _reuse_existing_conversation_locked( ) fallback_secret = request.secrets.get(CODEX_AUTH_SECRET_NAME) if managed_codex_credential or fallback_secret is not None: - original_stored = existing_record.stored - injected_fallback = ( - not managed_codex_credential and fallback_secret is not None - ) - if injected_fallback: - existing_record.stored = original_stored.model_copy( - update={ - "secrets": { - **original_stored.secrets, - CODEX_AUTH_SECRET_NAME: fallback_secret, - } - } - ) - try: - # Reuse the agent we already parsed from base_state.json - # above so the load path doesn't read and parse it again. - event_service = await self._get_or_load_event_service_locked( - conversation_id, agent=reattach_agent - ) - finally: - if injected_fallback: - existing_record.stored = original_stored + injected_secrets: dict[str, SecretSource] | None = None + if not managed_codex_credential and fallback_secret is not None: + injected_secrets = {CODEX_AUTH_SECRET_NAME: fallback_secret} + # Reuse the agent we already parsed from base_state.json + # above so the load path doesn't read and parse it again. + event_service = await self._get_or_load_event_service_locked( + conversation_id, agent=reattach_agent, secrets=injected_secrets + ) if event_service is not None: state = await event_service.get_state() return ( @@ -1658,7 +1684,7 @@ async def _create_conversation( lambda: self._launch_stores(settings), self.cipher, ) - secrets = scoped_secrets(request.secrets, source) + secrets = scoped_secrets(request_secrets(request, self.cipher), source) managed_codex_credential = is_codex_source(source) and ( CODEX_AUTH_SECRET_NAME in self._credential_bindings.get(conversation_id, {}) or await self._has_local_codex_credential() @@ -1775,9 +1801,10 @@ async def _create_conversation( # serialize to plain strings. Dump in plaintext so StaticSecret values # are preserved through the round-trip; the dict is only used in-process to # construct StoredConversation, not sent over the network. + # Everything excluded here belongs to base_state.json, not meta.json: + # it goes to the EventService directly instead of StoredConversation. request_data = request.model_copy( update={ - "secrets": secrets, "workspace": workspace, "tool_module_qualnames": tool_module_qualnames, } @@ -1790,6 +1817,10 @@ async def _create_conversation( "agent_profile_id", "agent_launch_additions", "max_budget_per_run", + "confirmation_policy", + "security_analyzer", + "secrets", + "secrets_encrypted", }, ) launched_agent_profile = launched.profile or runtime_launched_profile @@ -1819,6 +1850,9 @@ async def _create_conversation( gateway_run=gateway_run, initial_message=first_message, initial_sender=initial_sender, + confirmation_policy=request.confirmation_policy, + security_analyzer=request_security_analyzer(request, self.cipher), + secrets=secrets, reuse_request=request, max_budget_per_run=request.max_budget_per_run, ) @@ -1830,6 +1864,9 @@ async def _launch_prepared_conversation( *, run_slot: RunSlot, initial_message: Message | None, + confirmation_policy: ConfirmationPolicyBase, + security_analyzer: SecurityAnalyzerBase | None, + secrets: dict[str, SecretSource], initial_sender: str | None = None, reuse_request: StartConversationRequest | None = None, max_budget_per_run: float | None = None, @@ -1860,12 +1897,17 @@ async def _launch_prepared_conversation( return await self._reuse_existing_conversation_locked( stored.id, reuse_request ) - # New conversation: the agent is written to base_state.json (its - # single source of truth), not to meta.json. Pass it explicitly. + # New conversation: the agent, confirmation_policy, + # security_analyzer, and secrets are written to base_state.json + # (their single source of truth), not to meta.json. Pass them + # explicitly. event_service = await self._start_event_service( stored, is_new_conversation=True, launched=launched, + confirmation_policy=confirmation_policy, + security_analyzer=security_analyzer, + secrets=secrets, max_budget_per_run=max_budget_per_run, gateway_run=gateway_run, ) @@ -2172,21 +2214,27 @@ async def fork_conversation( # Extract the persisted data, then discard the temporary conversation. fork_agent = cast(AgentBase, fork_conv.agent) fork_workspace = fork_conv.workspace + # Without a cipher the fork's base_state.json has no secret values, + # so the ones it holds in memory go to its EventService directly. + fork_secrets = ( + dict(fork_conv.state.secret_registry.secret_sources) + if self._cipher_for(fork_conv_id) is None + else None + ) fork_conv.delete_on_close = False fork_conv.close() # _start_event_service will resume from the persisted fork directory. # Copy the source's stored metadata so request-level configuration # (client_tools, tool_module_qualnames, agent_definitions, plugins, - # secrets, ...) is preserved on the fork, then override only the + # ...) is preserved on the fork, then override only the # fork-specific fields. Without this, e.g. a fork of a client-tool # conversation would lose ``client_tools`` in meta.json and be unable # to re-register its tools after a server restart. - # Note: the agent is NOT stored in meta.json (StoredConversation) — the - # fork's agent is already persisted to the fork's base_state.json by - # ``source_conversation.fork`` above. It is passed to - # ``_start_event_service`` via ``agent=`` for the new-conversation - # path. + # Note: the agent, confirmation_policy, security_analyzer, and + # secrets are NOT stored in meta.json (StoredConversation). + # ``source_conversation.fork`` above copies them into the fork's + # base_state.json, which the fork's EventService resumes from. fork_overrides: dict[str, Any] = { "id": fork_conv_id, "workspace": fork_workspace, @@ -2204,7 +2252,10 @@ async def fork_conversation( # If the service fails to start, clean up the orphaned # persistence directory so we don't leave stale state on disk. fork_event_service = await self._start_event_service( - fork_stored, is_new_conversation=True, persisted_agent=fork_agent + fork_stored, + is_new_conversation=True, + persisted_agent=fork_agent, + secrets=fork_secrets, ) except Exception: safe_rmtree(fork_dir) @@ -2223,8 +2274,9 @@ async def start_child_conversation( """Start a child conversation on this server, derived from *parent_id*. Like :meth:`fork_conversation`, the child inherits the parent's stored - configuration (secrets, client tools, plugins, hooks, tags, ...) and its - resolved agent, but it starts with a fresh event history: ``task`` is + configuration (client tools, plugins, hooks, tags, ...), its resolved + agent and its current confirmation policy, security analyzer and + secrets, but it starts with a fresh event history: ``task`` is delivered as the child's first user message. With ``isolation="worktree"`` the child runs in a git worktree carved from the parent's workspace when that workspace is a git repository with @@ -2250,6 +2302,9 @@ async def start_child_conversation( with await RunSlot.acquire(self._run_semaphore) as run_slot: parent_stored = parent_service.stored parent_agent = cast(AgentBase, parent_service.get_conversation().agent) + policy, analyzer, secrets = _inheritable_state_config( + await parent_service.get_state() + ) # A child spends under the same cap as its parent. max_budget_per_run = parent_service.get_conversation().max_budget_per_run child_id = uuid4() @@ -2308,6 +2363,9 @@ async def start_child_conversation( ), # The parent's agent wrote the task, not the user. initial_sender=conversation_sender(parent_id), + confirmation_policy=policy, + security_analyzer=analyzer, + secrets=secrets, ) return StartChildConversationResponse( @@ -2448,9 +2506,9 @@ async def _evict_idle_conversations(self, ttl_seconds: float) -> None: if event_service is None: continue # Preserve runtime-only state so rehydration is faithful: - # sync the catalog to the current stored (switch_acp_model / - # secret updates replace it) and hand back credential bindings - # (close() clears them). + # sync the catalog to the current stored (title, tags and + # required credential bindings can change while live) and hand + # back credential bindings (close() clears them). record = self._conversation_records.get(conversation_id) if record is not None: record.stored = event_service.stored @@ -2586,6 +2644,9 @@ async def _start_event_service( is_new_conversation: bool = False, launched: LaunchedAgent | None = None, persisted_agent: AgentBase | None = None, + confirmation_policy: ConfirmationPolicyBase | None = None, + security_analyzer: SecurityAnalyzerBase | None = None, + secrets: dict[str, SecretSource] | None = None, max_budget_per_run: float | None = None, gateway_run: bool = False, ) -> EventService: @@ -2604,7 +2665,10 @@ async def _start_event_service( stored=stored, conversations_dir=self.conversations_dir, agent=launched.agent if launched is not None else None, + initial_confirmation_policy=confirmation_policy or NeverConfirm(), + initial_security_analyzer=security_analyzer, initial_max_budget_per_run=max_budget_per_run, + secrets=secrets or {}, cipher=self._cipher_for(stored.id), mcp_tool_provider=self.mcp_tool_provider, credential_bindings=credential_bindings, @@ -2722,11 +2786,21 @@ async def _maybe_subscribe_telemetry( live_agent = ( live_conversation.agent if live_conversation is not None else None ) + live_policy = ( + live_conversation.state.confirmation_policy + if live_conversation is not None + else None + ) subscriber = TelemetrySubscriber( conversation_id=stored.id, sink=sink, factory=factory, - context=_build_telemetry_context(stored, factory, agent=live_agent), + context=_build_telemetry_context( + stored, + factory, + agent=live_agent, + confirmation_policy=live_policy, + ), ) await event_service.subscribe_to_events(subscriber) if is_new_conversation: @@ -2739,6 +2813,7 @@ def _build_telemetry_context( stored: StoredConversation, factory: DiagnosticEventFactory, agent: AgentBase | None = None, + confirmation_policy: ConfirmationPolicyBase | None = None, ) -> ConversationTelemetryContext: """Reduce a stored conversation to its sanitized telemetry facts. @@ -2775,7 +2850,9 @@ def _build_telemetry_context( workspace_kind=workspace_kind, # Policy kind, not a bool: a bool would collapse ConfirmRisky. confirmation_policy=safe_token( - type(getattr(stored, "confirmation_policy", None)).__name__.lower() + type(confirmation_policy).__name__.lower() + if confirmation_policy is not None + else None ), is_automation=is_automation, ) diff --git a/openhands-agent-server/openhands/agent_server/event_service.py b/openhands-agent-server/openhands/agent_server/event_service.py index b99c7c44706..a032e384ea2 100644 --- a/openhands-agent-server/openhands/agent_server/event_service.py +++ b/openhands-agent-server/openhands/agent_server/event_service.py @@ -1,5 +1,6 @@ import asyncio import contextvars +import json import threading import time import traceback @@ -13,7 +14,7 @@ from typing import Any, cast from uuid import UUID, uuid4 -from pydantic import ValidationError +from pydantic import BaseModel, Field, TypeAdapter, ValidationError from openhands.agent_server.bash_service import BashEventService from openhands.agent_server.conversation_lease import ( @@ -106,10 +107,14 @@ from openhands.sdk.llm.streaming import LLMStreamChunk from openhands.sdk.mcp.config import MCPServer from openhands.sdk.mcp.utils import MCPToolProvider +from openhands.sdk.secret import SecretSource, StaticSecret from openhands.sdk.security.analyzer import SecurityAnalyzerBase -from openhands.sdk.security.confirmation_policy import ConfirmationPolicyBase +from openhands.sdk.security.confirmation_policy import ( + ConfirmationPolicyBase, + NeverConfirm, +) from openhands.sdk.utils.async_utils import AsyncCallbackWrapper -from openhands.sdk.utils.cipher import Cipher +from openhands.sdk.utils.cipher import FERNET_TOKEN_PREFIX, Cipher from openhands.sdk.utils.files import atomic_write_text from openhands.sdk.utils.pydantic_secrets import cipher_context, client_context from openhands.sdk.workspace import LocalWorkspace @@ -249,6 +254,29 @@ def _without_agent_context_secret( ) +class _LegacyMetaSecrets(BaseModel): + """The ``secrets`` key older servers wrote to meta.json. Each entry is + read on its own, so one that cannot be parsed does not cost the others.""" + + secrets: dict[str, Any] = Field(default_factory=dict) + + +_SECRET_SOURCE: TypeAdapter[SecretSource] = TypeAdapter(SecretSource) + + +def _has_undecryptable_token(cipher: Cipher | None, data: Any) -> bool: + """Whether parsed JSON ``data`` holds a Fernet token ``cipher`` cannot decrypt.""" + if isinstance(data, str): + return data.startswith(FERNET_TOKEN_PREFIX) and ( + cipher is None or cipher.try_decrypt_str(data) is None + ) + if isinstance(data, dict): + return any(_has_undecryptable_token(cipher, v) for v in data.values()) + if isinstance(data, list): + return any(_has_undecryptable_token(cipher, v) for v in data) + return False + + @dataclass class EventService: """ @@ -258,13 +286,21 @@ class EventService: stored: StoredConversation conversations_dir: Path - # Agent for a NEW conversation. meta.json (``stored``) no longer carries the - # agent — base_state.json is its single source of truth — so the creating - # caller passes it here. On resume this is ``None`` and the agent is loaded - # from base_state.json. + # What a NEW conversation starts with. base_state.json owns these from + # then on: a resume keeps the saved agent, policy and analyzer and ignores + # the values here. A budget given on a resume replaces the saved one. agent: AgentBase | None = None - # Budget for a new conversation; base_state.json owns it from then on. + initial_confirmation_policy: ConfirmationPolicyBase = field( + default_factory=NeverConfirm, kw_only=True + ) + initial_security_analyzer: SecurityAnalyzerBase | None = field( + default=None, kw_only=True + ) initial_max_budget_per_run: float | None = field(default=None, kw_only=True) + # Added to the registry on every start, new or resumed, except for names + # with a credential binding: a resume gets a credential its caller just + # supplied this way. + secrets: dict[str, SecretSource] = field(default_factory=dict, kw_only=True) cipher: Cipher | None = None mcp_tool_provider: MCPToolProvider | None = None credential_bindings: dict[str, VersionedCredentialBinding] = field( @@ -348,6 +384,59 @@ async def load_meta(self): context=cipher_context(self.cipher), ) + def _read_legacy_meta_secrets(self) -> dict[str, SecretSource]: + """Start-time secrets an older server left in meta.json. + + Older servers kept them there and could leave them out of + base_state.json. ``start`` takes over the names base_state.json lacks; + the next write of meta.json drops the key. + """ + try: + entries = _LegacyMetaSecrets.model_validate_json( + (self.conversation_dir / "meta.json").read_text() + ).secrets + except (OSError, ValidationError): + return {} + if not entries: + return {} + try: + base_state = json.loads((self.conversation_dir / BASE_STATE).read_text()) + except FileNotFoundError: + base_state = None + except (OSError, ValueError): + return {} + # Taking a secret over saves the state. On a load that cannot decrypt + # what is stored, that save writes those values as missing, and the + # right key no longer brings them back. + if _has_undecryptable_token(self.cipher, [entries, base_state]): + logger.warning( + "This server cannot decrypt what is stored for conversation " + "%s; the secrets an older server left in its meta.json are " + "not taken over, and the next write of meta.json drops them", + self.stored.id, + ) + return {} + legacy: dict[str, SecretSource] = {} + for name, entry in entries.items(): + try: + source = _SECRET_SOURCE.validate_python( + entry, context=cipher_context(self.cipher) + ) + except ValidationError: + logger.warning( + "Could not read the secret %s an older server left in " + "meta.json of conversation %s; it is not taken over", + name, + self.stored.id, + ) + continue + # An older server without a key stored the value redacted: there + # is nothing to take over. + if isinstance(source, StaticSecret) and source.value is None: + continue + legacy[name] = source + return legacy + async def save_meta(self, stored: StoredConversation | None = None) -> None: """Persist ``stored`` (default: the current metadata) to ``meta.json``. @@ -363,18 +452,11 @@ async def save_meta(self, stored: StoredConversation | None = None) -> None: ) self.stored = new_stored - def _without_stored_secret(self, secret_name: str) -> StoredConversation: - # meta.json (StoredConversation) no longer carries the agent, so there is - # no agent_context secret to scrub here — only the stored secrets map. - # The agent's own secret scrub happens on base_state.json (see - # _scrub_persisted_credentials). - secrets = dict(self.stored.secrets) - secrets.pop(secret_name, None) - return self.stored.model_copy(update={"secrets": secrets}) - async def _scrub_persisted_credentials( self, credential_bindings: Mapping[str, VersionedCredentialBinding] | None = None, + *, + legacy_secrets: Mapping[str, SecretSource] | None = None, ) -> None: bindings = ( self.credential_bindings @@ -411,9 +493,6 @@ async def _scrub_persisted_credentials( raise CredentialNeedsReauthentication( "ChatGPT authentication is invalid. Please sign in again." ) - for secret_name in bindings: - self.stored = self._without_stored_secret(secret_name) - if ( not base_state_file.exists() and not meta_file.exists() @@ -428,6 +507,10 @@ async def _scrub_persisted_credentials( context=context, ) sources = dict(state.secret_registry.secret_sources) + # The meta.json rewrite below drops the secrets an older + # server kept there, so the names this file lacks move here. + for secret_name, source in (legacy_secrets or {}).items(): + sources.setdefault(secret_name, source) for secret_name in bindings: sources.pop(secret_name, None) state.agent = _without_agent_context_secret( @@ -494,7 +577,6 @@ async def activate_credential_binding( ) state.agent = agent conversation.agent = agent - self.stored = self._without_stored_secret(secret_name) await self._scrub_persisted_credentials() async def apply_resume_secrets( @@ -513,25 +595,13 @@ async def apply_resume_secrets( return def _update() -> None: - state = conversation._state - with state: - registry = state.secret_registry.model_copy( - update={ - "secret_sources": dict(state.secret_registry.secret_sources) - } - ) - registry.update_secrets(secrets) - state.secret_registry = registry + with conversation._state: + conversation.update_secrets(secrets) agent = conversation.agent if isinstance(agent, ACPAgent): agent.restart_for_updated_credentials(secrets) await self.run_state_op(contextvars.copy_context().run, _update) - await self.save_meta( - self.stored.model_copy( - update={"secrets": {**self.stored.secrets, **secrets}} - ) - ) def _write_guard(self): if self._lease is None or self._lease_generation is None: @@ -1322,7 +1392,6 @@ async def start(self): max_workers=4, thread_name_prefix="conversation-state" ) - # self.stored contains an Agent configuration we can instantiate self.conversation_dir.mkdir(parents=True, exist_ok=True) # lease_ttl_seconds=0 disables leasing for single-instance deployments # where shared-storage stale leases would otherwise block pod restarts. @@ -1334,7 +1403,9 @@ async def start(self): ) lease_claim = self._lease.claim() self._lease_generation = lease_claim.generation - await self._scrub_persisted_credentials() + # Before the scrub below, which can rewrite meta.json. + legacy_secrets = await asyncio.to_thread(self._read_legacy_meta_secrets) + await self._scrub_persisted_credentials(legacy_secrets=legacy_secrets) workspace = self.stored.workspace assert isinstance(workspace, LocalWorkspace) await asyncio.to_thread( @@ -1449,7 +1520,11 @@ def _token_streaming_callback(chunk: LLMStreamChunk | str) -> None: max_budget_per_run=self.initial_max_budget_per_run, stuck_detection=self.stored.stuck_detection, visualizer=None, - secrets=self.stored.secrets, + secrets={ + name: source + for name, source in self.secrets.items() + if name not in self.credential_bindings + }, cipher=self.cipher, hook_config=self.stored.hook_config, tags=self.stored.tags, @@ -1462,8 +1537,22 @@ def _token_streaming_callback(chunk: LLMStreamChunk | str) -> None: mcp_startup_budget=MCP_STARTUP_BUDGET_SECONDS, ) - conversation.set_confirmation_policy(self.stored.confirmation_policy) - conversation.set_security_analyzer(self.stored.security_analyzer) + # base_state.json wins: only the names it lacks are taken from the + # copy an older server left in meta.json. + known = conversation.state.secret_registry.secret_sources + missing = { + name: source + for name, source in legacy_secrets.items() + if name not in known and name not in self.credential_bindings + } + if missing: + conversation.update_secrets(missing) + # Only a caller that creates a conversation passes its agent. The + # presence of base_state.json cannot tell a new conversation from a + # resumed one: a create that failed halfway leaves the file behind. + if self.agent is not None: + conversation.set_confirmation_policy(self.initial_confirmation_policy) + conversation.set_security_analyzer(self.initial_security_analyzer) # On resume the agent was unknown at construction time (loaded from # base_state.json), so decide token streaming now and disable it when the # resolved agent can't emit token callbacks. @@ -2646,8 +2735,24 @@ def has_external_subscribers(self) -> bool: """True if a non-internal subscriber (e.g. a websocket) is attached.""" return bool(self._pub_sub.subscriber_ids() - self._internal_subscriber_ids) + def _holds_secrets_a_reload_would_lose(self) -> bool: + """Without a cipher base_state.json has no secret values, so the loaded + conversation is the only place that has its secrets and the keys + inside its security analyzer.""" + conversation = self._conversation + if self.cipher is not None or conversation is None: + return False + state = conversation.state + return any( + part is not None + and part.model_dump(mode="json") + != part.model_dump(mode="json", context=client_context("plaintext")) + for part in (state.secret_registry, state.security_analyzer) + ) + def is_idle_evictable(self) -> bool: - """Safe to evict only with no in-flight work and no external subscriber.""" + """Safe to evict only with no in-flight work, no external subscriber + and no secret that a reload would lose.""" run_active = self._run_task is not None and not self._run_task.done() goal_active = ( self._goal_loop_task is not None and not self._goal_loop_task.done() @@ -2659,6 +2764,7 @@ def is_idle_evictable(self) -> bool: or self._rerun_requested or self._acp_internal_rerun_requested or self.has_external_subscribers() + or self._holds_secrets_a_reload_would_lose() ): return False return True diff --git a/openhands-agent-server/openhands/agent_server/launch.py b/openhands-agent-server/openhands/agent_server/launch.py index 92a7931cfe5..67aa3123d00 100644 --- a/openhands-agent-server/openhands/agent_server/launch.py +++ b/openhands-agent-server/openhands/agent_server/launch.py @@ -50,6 +50,7 @@ ) from openhands.sdk.profiles.resolver import ProfileNotFound from openhands.sdk.secret import SecretSource +from openhands.sdk.security.analyzer import SecurityAnalyzerBase from openhands.sdk.settings.model import ( ACPAgentSettings, validate_agent_settings, @@ -186,6 +187,25 @@ def request_secrets( ) +def request_security_analyzer( + request: StartConversationRequest, cipher: Cipher | None +) -> SecurityAnalyzerBase | None: + """A copy of the request's security analyzer for one conversation, its + nested credentials decrypted when the client encrypted them (e.g. a + guardrail LLM ``api_key``). + + Always a copy: an analyzer can keep per-conversation history, and one + request object can start more than one conversation. + """ + analyzer = request.security_analyzer + if analyzer is None: + return None + return type(analyzer).model_validate( + analyzer.model_dump(context=client_context("plaintext")), + context=_decryption_context(request, cipher), + ) + + def _decryption_context( request: StartConversationRequest, cipher: Cipher | None ) -> SecretContext | None: diff --git a/openhands-agent-server/openhands/agent_server/models.py b/openhands-agent-server/openhands/agent_server/models.py index 3a33cc2dc16..7809d9a144a 100644 --- a/openhands-agent-server/openhands/agent_server/models.py +++ b/openhands-agent-server/openhands/agent_server/models.py @@ -18,6 +18,7 @@ StartChildConversationRequest as StartChildConversationRequest, StartChildConversationResponse as StartChildConversationResponse, StartConversationRequest as StartConversationRequest, + StoredConversationConfig as StoredConversationConfig, ) from openhands.sdk.conversation.secret_registry import SecretRegistry from openhands.sdk.conversation.state import ConversationExecutionStatus @@ -104,16 +105,15 @@ class EventSortOrder(StrEnum): TIMESTAMP_DESC = "TIMESTAMP_DESC" -class StoredConversation(ConversationConfig): +class StoredConversation(StoredConversationConfig): """Stored details about a conversation. - Extends :class:`ConversationConfig` (the agent-less shared config) with - server-assigned fields. It deliberately does NOT carry the ``agent``: the - single source of truth for the agent / runtime state is - ``ConversationState`` persisted to ``base_state.json``. Because - ``StoredConversation`` is not a ``StartConversationRequest``, the agent - cannot silently re-appear in ``meta.json``. The same holds for the - conversation budget (``max_budget_per_run``). + Extends :class:`StoredConversationConfig` with server-assigned fields. It + deliberately does NOT carry the ``agent``, ``confirmation_policy``, + ``security_analyzer``, ``secrets`` or the conversation budget + (``max_budget_per_run``): their single source of truth after creation is + ``ConversationState`` persisted to ``base_state.json``, and its base class + has no field for any of them, so none can re-appear in ``meta.json``. """ required_runtime_credential_bindings: set[str] = Field(default_factory=set) diff --git a/openhands-sdk/openhands/sdk/conversation/request.py b/openhands-sdk/openhands/sdk/conversation/request.py index f8661b9b865..c24eadac801 100644 --- a/openhands-sdk/openhands/sdk/conversation/request.py +++ b/openhands-sdk/openhands/sdk/conversation/request.py @@ -223,16 +223,16 @@ def _ref_for_ref_base(self) -> ConversationWorktreeOptions: return self -class ConversationConfig(BaseModel): - """Shared conversation configuration — everything except the agent. - - This is the common base for :class:`StartConversationRequest` (which adds the - ``agent``/``agent_settings``/``agent_profile_id`` family) and the - agent-server's ``StoredConversation`` (which does NOT persist the agent — - the agent's single source of truth is ``ConversationState`` / - ``base_state.json``). Keeping the agent off this base is what makes the - duplication structurally impossible rather than something a reviewer has to - remember to exclude. +class StoredConversationConfig(BaseModel): + """What the agent-server keeps about a conversation in its own record. + + The base of :class:`ConversationConfig`, and through it of + :class:`StartConversationRequest`, and the base of the agent-server's + ``StoredConversation`` (``meta.json``). It leaves out everything the + conversation state (``ConversationState`` / ``base_state.json``) owns once + the conversation exists: the agent, the budget, the confirmation policy, + the security analyzer and the secrets. A record built on this base cannot + hold a second copy of them. """ workspace: LocalWorkspace = Field( @@ -266,15 +266,6 @@ class ConversationConfig(BaseModel): "parent must already exist and share this conversation's workspace." ), ) - confirmation_policy: ConfirmationPolicyBase = Field( - default=NeverConfirm(), - description="Controls when the conversation will prompt the user before " - "continuing. Defaults to never.", - ) - security_analyzer: SecurityAnalyzerBase | None = Field( - default=None, - description="Optional security analyzer to evaluate action risks.", - ) initial_message: SendMessageRequest | None = Field( default=None, description="Initial message to pass to the LLM" ) @@ -289,23 +280,6 @@ class ConversationConfig(BaseModel): description="If true, the conversation will use stuck detection to " "prevent infinite loops.", ) - secrets: dict[str, SecretSource] = Field( - default_factory=dict, - description="Secrets available in the conversation", - ) - secrets_encrypted: bool = Field( - default=False, - description=( - "If true, indicates that secret values in the agent configuration " - "are cipher-encrypted and should be decrypted by the server before " - "use. This enables secure round-tripping of settings through " - "untrusted clients (e.g., frontend) that received encrypted values " - "via the X-Expose-Secrets header. " - "Flow: client calls GET /api/settings with X-Expose-Secrets: encrypted " - "to receive cipher-encrypted secrets, then passes them in the agent " - "config with secrets_encrypted=True so the server can decrypt them." - ), - ) tool_module_qualnames: dict[str, str] = Field( default_factory=dict, description=( @@ -435,6 +409,44 @@ class ConversationConfig(BaseModel): ) +class ConversationConfig(StoredConversationConfig): + """Start-time conversation configuration, everything except the agent. + + Adds the values a start request hands to the conversation state: + ``confirmation_policy``, ``security_analyzer`` and ``secrets``. After + creation they belong to that state and are changed through the + conversation, so nothing that is persisted next to it should extend this + class; see :class:`StoredConversationConfig`. + """ + + confirmation_policy: ConfirmationPolicyBase = Field( + default=NeverConfirm(), + description="Controls when the conversation will prompt the user before " + "continuing. Defaults to never.", + ) + security_analyzer: SecurityAnalyzerBase | None = Field( + default=None, + description="Optional security analyzer to evaluate action risks.", + ) + secrets: dict[str, SecretSource] = Field( + default_factory=dict, + description="Secrets available in the conversation", + ) + secrets_encrypted: bool = Field( + default=False, + description=( + "If true, indicates that secret values in the agent configuration " + "are cipher-encrypted and should be decrypted by the server before " + "use. This enables secure round-tripping of settings through " + "untrusted clients (e.g., frontend) that received encrypted values " + "via the X-Expose-Secrets header. " + "Flow: client calls GET /api/settings with X-Expose-Secrets: encrypted " + "to receive cipher-encrypted secrets, then passes them in the agent " + "config with secrets_encrypted=True so the server can decrypt them." + ), + ) + + class StartConversationRequest(ConversationConfig): """Payload to create a new conversation. diff --git a/tests/agent_server/telemetry/test_telemetry_disabled_by_default.py b/tests/agent_server/telemetry/test_telemetry_disabled_by_default.py index 6e9ca860461..23df47ffeaf 100644 --- a/tests/agent_server/telemetry/test_telemetry_disabled_by_default.py +++ b/tests/agent_server/telemetry/test_telemetry_disabled_by_default.py @@ -157,7 +157,6 @@ async def test_conversation_service_reads_the_live_sink_not_a_captured_one( DiagnosticEventFactory, build_runtime_properties, ) - from openhands.sdk.security.confirmation_policy import NeverConfirm from openhands.sdk.workspace import LocalWorkspace # 1. Construct the service while telemetry is still the pre-init NoOp — the @@ -204,7 +203,6 @@ async def subscribe_to_events(self, subscriber): stored = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir=str(tmp_path)), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, ) diff --git a/tests/agent_server/telemetry/test_telemetry_subscriber.py b/tests/agent_server/telemetry/test_telemetry_subscriber.py index b1c51c08ade..417b197a75e 100644 --- a/tests/agent_server/telemetry/test_telemetry_subscriber.py +++ b/tests/agent_server/telemetry/test_telemetry_subscriber.py @@ -6,13 +6,20 @@ import pytest +import openhands.agent_server.telemetry.service as telemetry_service +from openhands.agent_server.conversation_service import ConversationService +from openhands.agent_server.models import StartConversationRequest from openhands.agent_server.pub_sub import PubSub, Subscriber from openhands.agent_server.telemetry import models as m -from openhands.agent_server.telemetry.factory import DiagnosticEventFactory +from openhands.agent_server.telemetry.factory import ( + DiagnosticEventFactory, + build_runtime_properties, +) from openhands.agent_server.telemetry.subscriber import ( ConversationTelemetryContext, TelemetrySubscriber, ) +from openhands.sdk import LLM, Agent from openhands.sdk.event import ( AgentErrorEvent, ConversationStateUpdateEvent, @@ -20,6 +27,8 @@ ) from openhands.sdk.event.conversation_error import ConversationErrorEvent from openhands.sdk.event.error_classification import ErrorClassification, FailureKind +from openhands.sdk.security.confirmation_policy import AlwaysConfirm +from openhands.sdk.workspace import LocalWorkspace class CollectingSink: @@ -632,34 +641,16 @@ async def test_interrupted_fallback_reports_usage_seen_before_close(factory): assert payload["cost_bucket"] == "0p1-1" -def test_confirmation_policy_is_read_from_the_field_that_exists(): - """Regression: the field is confirmation_policy, not confirmation_mode. - - Reading a non-existent ``confirmation_mode`` via getattr silently reported - False for every conversation, and a bool would have collapsed ConfirmRisky - into NeverConfirm anyway. - """ - import uuid as _uuid - +def test_telemetry_context_reports_the_policy_kind_and_no_workspace_path(): + """The context names the policy's kind (a bool would collapse ConfirmRisky + into another policy) and carries nothing that identifies the workspace.""" from openhands.agent_server.conversation_service import _build_telemetry_context from openhands.agent_server.models import StoredConversation - from openhands.agent_server.telemetry.factory import ( - DiagnosticEventFactory, - build_runtime_properties, - ) - from openhands.sdk.agent import Agent - from openhands.sdk.llm import LLM - from openhands.sdk.security.confirmation_policy import AlwaysConfirm - from openhands.sdk.workspace import LocalWorkspace - - assert "confirmation_mode" not in StoredConversation.model_fields - assert "confirmation_policy" in StoredConversation.model_fields agent = Agent(llm=LLM(model="anthropic/claude-sonnet-5", usage_id="t"), tools=[]) stored = StoredConversation( - id=_uuid.uuid4(), + id=uuid.uuid4(), workspace=LocalWorkspace(working_dir="/Users/alice/secret-project"), - confirmation_policy=AlwaysConfirm(), user_id="canvas-user-42", ) ctx = _build_telemetry_context( @@ -669,6 +660,7 @@ def test_confirmation_policy_is_read_from_the_field_that_exists(): salt="s", ), agent=agent, + confirmation_policy=AlwaysConfirm(), ) from dataclasses import asdict @@ -736,3 +728,38 @@ async def test_event_count_ignores_streaming_deltas(factory): await pub_sub(StreamingDeltaEvent(content="tok")) assert sub._event_count == 0 + + +@pytest.mark.asyncio +async def test_conversation_created_reports_the_conversations_policy( + tmp_path, monkeypatch +): + """The policy in ``conversation_created`` is the one the conversation + runs under.""" + sink = CollectingSink() + monkeypatch.setattr(telemetry_service, "_telemetry_sink", sink) + monkeypatch.setattr( + telemetry_service, + "_event_factory", + DiagnosticEventFactory(runtime=build_runtime_properties(deferred_init=False)), + ) + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + request = StartConversationRequest( + agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), + workspace=LocalWorkspace(working_dir=str(workspace_dir)), + confirmation_policy=AlwaysConfirm(), + ) + + async with ConversationService( + conversations_dir=tmp_path / "conversations" + ) as service: + await service.start_conversation(request) + + [created] = [ + event.properties + for event in sink.events + if event.event_name == m.EventName.CONVERSATION_CREATED + ] + assert isinstance(created, m.ConversationStartedProperties) + assert created.confirmation_policy == "alwaysconfirm" diff --git a/tests/agent_server/test_agent_profile_conv_start.py b/tests/agent_server/test_agent_profile_conv_start.py index 2ccad3e8ecc..d8d4ba2ae19 100644 --- a/tests/agent_server/test_agent_profile_conv_start.py +++ b/tests/agent_server/test_agent_profile_conv_start.py @@ -140,11 +140,26 @@ async def _start( browser: bool = False, ) -> tuple[StoredConversation, LaunchedAgent]: """Start ``request`` through the real pipeline; return what reached the service.""" + captured = await _start_capturing( + tmp_path, request, stores=stores, persisted=persisted, browser=browser + ) + return captured["stored"], captured["launched"] + + +async def _start_capturing( + tmp_path, + request: StartConversationRequest, + *, + stores: LaunchStores | None = None, + persisted: PersistedSettings | None = None, + browser: bool = False, +) -> dict[str, Any]: + """Like :func:`_start`, but return every argument that reached the service.""" captured: dict[str, Any] = {} async def capture_start(stored, **kwargs): captured["stored"] = stored - captured["launched"] = kwargs["launched"] + captured.update(kwargs) return _event_service_for(stored, kwargs["launched"].agent) service = ConversationService( @@ -162,7 +177,7 @@ async def capture_start(stored, **kwargs): ), ): await service.start_conversation(request) - return captured["stored"], captured["launched"] + return captured def _profile_request(tmp_path, profile, **kwargs) -> StartConversationRequest: @@ -851,9 +866,11 @@ async def test_start_conversation_drops_secrets_the_profile_disallows( }, ) - stored, _ = await _start(tmp_path, request, stores=fakes.stores(profile)) + captured = await _start_capturing( + tmp_path, request, stores=fakes.stores(profile) + ) - assert set(stored.secrets) == expected + assert set(captured["secrets"]) == expected @pytest.mark.asyncio async def test_a_scoped_acp_profile_gets_no_implicit_provider_credentials( @@ -867,9 +884,11 @@ async def test_a_scoped_acp_profile_gets_no_implicit_provider_credentials( secrets={"ANTHROPIC_API_KEY": StaticSecret(value=SecretStr("sk"))}, ) - stored, _ = await _start(tmp_path, request, stores=fakes.stores(profile)) + captured = await _start_capturing( + tmp_path, request, stores=fakes.stores(profile) + ) - assert stored.secrets == {} + assert captured["secrets"] == {} @pytest.mark.asyncio async def test_mcp_servers_outside_the_profile_scope_are_not_attached( diff --git a/tests/agent_server/test_auto_title_span_metadata.py b/tests/agent_server/test_auto_title_span_metadata.py index 0e59b352826..94c94df2999 100644 --- a/tests/agent_server/test_auto_title_span_metadata.py +++ b/tests/agent_server/test_auto_title_span_metadata.py @@ -40,7 +40,6 @@ def _probe_auto_title_spans() -> dict[str, Any]: from openhands.sdk.conversation.impl.local_conversation import LocalConversation from openhands.sdk.event.llm_convertible import MessageEvent from openhands.sdk.llm import LLM, Message, TextContent - from openhands.sdk.security.confirmation_policy import NeverConfirm from openhands.sdk.workspace import LocalWorkspace Laminar.initialize( @@ -73,7 +72,6 @@ def mocked_completion(**kwargs: Any): stored = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, title=None, diff --git a/tests/agent_server/test_conversation_eviction.py b/tests/agent_server/test_conversation_eviction.py index d06637a5ac0..ce3da37ea5b 100644 --- a/tests/agent_server/test_conversation_eviction.py +++ b/tests/agent_server/test_conversation_eviction.py @@ -4,13 +4,20 @@ import time import pytest +from pydantic import SecretStr from openhands.agent_server.conversation_service import ConversationService from openhands.agent_server.models import StartConversationRequest from openhands.agent_server.pub_sub import Subscriber from openhands.sdk import LLM, Agent, Event from openhands.sdk.credential import ResolvedCredential -from openhands.sdk.security.confirmation_policy import NeverConfirm +from openhands.sdk.secret import StaticSecret +from openhands.sdk.security.confirmation_policy import AlwaysConfirm, NeverConfirm +from openhands.sdk.security.llm_analyzer import LLMSecurityAnalyzer +from openhands.sdk.security.toolshield_llm_analyzer import ( + ToolShieldLLMSecurityAnalyzer, +) +from openhands.sdk.utils.cipher import Cipher from openhands.sdk.workspace import LocalWorkspace @@ -259,3 +266,98 @@ async def test_eviction_preserves_reassigned_stored_metadata(tmp_path): rehydrated = await service.get_event_service(conversation_id) assert rehydrated is not None assert rehydrated.stored.title == "new-title" + + +@pytest.mark.asyncio +async def test_eviction_keeps_runtime_policy_analyzer_and_secrets(tmp_path): + """What a client set after creation is still in force once the idle + conversation is evicted and loaded again (#4810).""" + conversations_dir = tmp_path / "conversations" + request = _make_request(tmp_path / "workspace").model_copy( + update={"secrets": {"TOKEN": StaticSecret(value=SecretStr("start-value"))}} + ) + + async with ConversationService( + conversations_dir=conversations_dir, + conversation_idle_ttl_seconds=1800, + cipher=Cipher("eviction-key"), + ) as service: + info, _ = await service.start_conversation(request) + assert service._event_services is not None + event_service = service._event_services[info.id] + await event_service.set_confirmation_policy(AlwaysConfirm()) + await event_service.set_security_analyzer(LLMSecurityAnalyzer()) + await event_service.update_secrets({"TOKEN": "runtime-value"}) + + event_service._last_active_monotonic = time.monotonic() - 10_000 + await service._evict_idle_conversations(1800) + assert info.id not in service._event_services + + rehydrated = await service.get_event_service(info.id) + assert rehydrated is not None + state = await rehydrated.get_state() + assert isinstance(state.confirmation_policy, AlwaysConfirm) + assert isinstance(state.security_analyzer, LLMSecurityAnalyzer) + token = state.secret_registry.secret_sources["TOKEN"] + assert token.get_value() == "runtime-value" + + +@pytest.mark.asyncio +async def test_conversation_whose_secrets_cannot_be_reloaded_is_not_evicted(tmp_path): + """Without a cipher base_state.json has no secret values. The loaded + conversation is their only copy, so it stays loaded.""" + request = _make_request(tmp_path / "workspace").model_copy( + update={"secrets": {"TOKEN": StaticSecret(value=SecretStr("start-value"))}} + ) + + async with ConversationService( + conversations_dir=tmp_path / "conversations", + conversation_idle_ttl_seconds=1800, + ) as service: + info, _ = await service.start_conversation(request) + assert service._event_services is not None + event_service = service._event_services[info.id] + + event_service._last_active_monotonic = time.monotonic() - 10_000 + await service._evict_idle_conversations(1800) + + assert service._event_services[info.id] is event_service + token = (await event_service.get_state()).secret_registry.secret_sources + assert token["TOKEN"].get_value() == "start-value" + + +@pytest.mark.asyncio +async def test_conversation_whose_analyzer_key_cannot_be_reloaded_is_not_evicted( + tmp_path, +): + """Without a cipher the key inside a security analyzer is not saved + either, and an analyzer without its key rates nothing. The conversation + stays loaded.""" + request = _make_request(tmp_path / "workspace").model_copy( + update={ + "security_analyzer": ToolShieldLLMSecurityAnalyzer( + llm=LLM( + model="openai/guard", + api_key=SecretStr("guard-key"), + usage_id="guard", + ) + ) + } + ) + + async with ConversationService( + conversations_dir=tmp_path / "conversations", + conversation_idle_ttl_seconds=1800, + ) as service: + info, _ = await service.start_conversation(request) + assert service._event_services is not None + event_service = service._event_services[info.id] + + event_service._last_active_monotonic = time.monotonic() - 10_000 + await service._evict_idle_conversations(1800) + + assert service._event_services[info.id] is event_service + analyzer = (await event_service.get_state()).security_analyzer + assert isinstance(analyzer, ToolShieldLLMSecurityAnalyzer) + assert isinstance(analyzer.llm.api_key, SecretStr) + assert analyzer.llm.api_key.get_secret_value() == "guard-key" diff --git a/tests/agent_server/test_conversation_service.py b/tests/agent_server/test_conversation_service.py index 809a46313f0..e239d9eca0f 100644 --- a/tests/agent_server/test_conversation_service.py +++ b/tests/agent_server/test_conversation_service.py @@ -31,6 +31,7 @@ ) from openhands.agent_server.event_service import EventService from openhands.agent_server.exceptions import ServiceInactive, UnavailableAgentTool +from openhands.agent_server.launch import request_security_analyzer from openhands.agent_server.models import ( ConversationInfo, ConversationPage, @@ -58,9 +59,13 @@ from openhands.sdk.llm import MessageToolCall, TextContent from openhands.sdk.llm.llm_profile_store import LLMProfileStore from openhands.sdk.mcp.config import dump_mcp_config -from openhands.sdk.secret import SecretSource, StaticSecret +from openhands.sdk.secret import StaticSecret from openhands.sdk.security.confirmation_policy import NeverConfirm +from openhands.sdk.security.llm_analyzer import LLMSecurityAnalyzer from openhands.sdk.security.risk import SecurityRisk +from openhands.sdk.security.toolshield_llm_analyzer import ( + ToolShieldLLMSecurityAnalyzer, +) from openhands.sdk.settings import ACPAgentSettings from openhands.sdk.subagent.registry import ( _reset_registry_for_tests, @@ -124,7 +129,6 @@ def sample_stored_conversation(): return StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -145,7 +149,6 @@ def test_conversation_info_takes_tags_from_the_metadata(sample_stored_conversati agent=_sample_agent(), workspace=stored.workspace, execution_status=ConversationExecutionStatus.RUNNING, - confirmation_policy=stored.confirmation_policy, tags={"project": "p1"}, ) @@ -168,7 +171,6 @@ async def test_meta_json_has_no_agent_and_reload_uses_base_state(tmp_path): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService(conversations_dir=conversations_dir) as service: info, _ = await service.start_conversation(request) @@ -467,7 +469,6 @@ async def persisted_conversation(tmp_path) -> tuple[Path, UUID]: request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService(conversations_dir=conversations_dir) as service: conversation_info, _ = await service.start_conversation(request) @@ -491,7 +492,6 @@ async def test_start_conversation_registers_and_injects_client_tools( request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), client_tools=[ ClientToolSpec( name="srv_show_dialog", @@ -518,7 +518,6 @@ async def fake_start_event_service(stored: StoredConversation, **kwargs): agent=agent, workspace=stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored.confirmation_policy, ) return service @@ -574,7 +573,6 @@ async def test_start_conversation_decrypts_encrypted_agent_settings_mcp_env( }, }, workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), secrets_encrypted=True, ) assert request.agent is None @@ -592,7 +590,6 @@ async def fake_start_event_service(stored: StoredConversation, **kwargs): agent=agent, workspace=stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored.confirmation_policy, ) return service @@ -624,7 +621,6 @@ async def test_second_service_does_not_resume_active_running_conversation(tmp_pa request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService(conversations_dir=conversations_dir) as primary: @@ -675,7 +671,6 @@ async def test_stale_owner_cannot_append_after_lease_takeover(tmp_path): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService(conversations_dir=conversations_dir) as primary: @@ -779,7 +774,6 @@ async def test_event_services_use_centralized_lease_renewal(tmp_path): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService(conversations_dir=conversations_dir) as svc: @@ -809,7 +803,6 @@ async def test_centralized_lease_renewal_invokes_renew(tmp_path): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) with patch( @@ -858,7 +851,6 @@ async def test_event_services_share_dedicated_run_executor(tmp_path): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService( @@ -955,7 +947,6 @@ def request() -> StartConversationRequest: return StartConversationRequest( agent=_sample_agent(), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService(conversations_dir=conversations_dir) as service: @@ -1144,7 +1135,6 @@ async def test_restart_resumes_conversations_after_non_graceful_shutdown(tmp_pat request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService(conversations_dir=conversations_dir) as primary: @@ -1184,7 +1174,6 @@ async def test_startup_and_search_do_not_hydrate_idle_conversation(tmp_path): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService(conversations_dir=conversations_dir) as primary: conversation_info, _ = await primary.start_conversation(request) @@ -1214,7 +1203,6 @@ async def test_concurrent_access_hydrates_conversation_once(tmp_path): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService(conversations_dir=conversations_dir) as primary: conversation_info, _ = await primary.start_conversation(request) @@ -1387,7 +1375,6 @@ async def test_waiting_hydration_cannot_restore_deleted_conversation( agent=_sample_agent(), workspace=record.stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=record.stored.confirmation_policy, ) deleting_runtime = AsyncMock(spec=EventService) deleting_runtime.is_open.return_value = True @@ -1453,7 +1440,6 @@ async def test_shutdown_closes_runtime_from_in_flight_hydration( agent=_sample_agent(), workspace=record.stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=record.stored.confirmation_policy, ) startup_entered = asyncio.Event() finish_startup = asyncio.Event() @@ -1506,7 +1492,6 @@ async def test_fork_rejects_id_of_unloaded_persisted_conversation(tmp_path): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService(conversations_dir=conversations_dir) as primary: source, _ = await primary.start_conversation(request) @@ -1555,7 +1540,6 @@ async def test_search_conversations_basic( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -1586,7 +1570,6 @@ async def test_search_conversations_with_critic_redacts_api_key( stored_conv = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -1600,7 +1583,6 @@ async def test_search_conversations_with_critic_redacts_api_key( agent=agent, workspace=stored_conv.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored_conv.confirmation_policy, ) conversation_service._event_services[stored_conv.id] = mock_service @@ -1632,7 +1614,6 @@ async def test_search_conversations_status_filter(self, conversation_service): stored_conv = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, i, 0, tzinfo=UTC), @@ -1646,7 +1627,6 @@ async def test_search_conversations_status_filter(self, conversation_service): agent=_sample_agent(), workspace=stored_conv.workspace, execution_status=status, - confirmation_policy=stored_conv.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -1683,7 +1663,6 @@ async def test_search_conversations_sorting(self, conversation_service): stored_conv = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime( @@ -1699,7 +1678,6 @@ async def test_search_conversations_sorting(self, conversation_service): agent=_sample_agent(), workspace=stored_conv.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored_conv.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -1759,7 +1737,6 @@ async def test_search_conversations_pagination(self, conversation_service): stored_conv = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, i, 0, tzinfo=UTC), @@ -1773,7 +1750,6 @@ async def test_search_conversations_pagination(self, conversation_service): agent=_sample_agent(), workspace=stored_conv.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored_conv.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -1828,7 +1804,6 @@ async def test_search_conversations_combined_filter_and_sort( stored_conv = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=created_at, @@ -1842,7 +1817,6 @@ async def test_search_conversations_combined_filter_and_sort( agent=_sample_agent(), workspace=stored_conv.workspace, execution_status=status, - confirmation_policy=stored_conv.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -1870,7 +1844,6 @@ async def test_search_conversations_invalid_page_id( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -1919,7 +1892,6 @@ async def test_count_conversations_basic( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -1944,7 +1916,6 @@ async def test_count_conversations_status_filter(self, conversation_service): stored_conv = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, i, 0, tzinfo=UTC), @@ -1958,7 +1929,6 @@ async def test_count_conversations_status_filter(self, conversation_service): agent=_sample_agent(), workspace=stored_conv.workspace, execution_status=status, - confirmation_policy=stored_conv.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -1993,7 +1963,6 @@ async def test_count_conversations_includes_regular_and_acp( legacy_conversation = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -2002,7 +1971,6 @@ async def test_count_conversations_includes_regular_and_acp( acp_conversation = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 13, 0, 0, tzinfo=UTC), @@ -2017,7 +1985,6 @@ async def test_count_conversations_includes_regular_and_acp( agent=_sample_agent(), workspace=stored_conv.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored_conv.confirmation_policy, ) conversation_service._event_services[stored_conv.id] = mock_service @@ -2027,129 +1994,6 @@ async def test_count_conversations_includes_regular_and_acp( class TestConversationServiceStartConversation: """Test cases for ConversationService.start_conversation method.""" - @pytest.mark.asyncio - async def test_start_conversation_with_secrets(self, conversation_service): - """Test that secrets are passed to new conversations when starting.""" - # Create test secrets - test_secrets: dict[str, SecretSource] = { - "api_key": StaticSecret(value=SecretStr("secret-api-key-123")), - "database_url": StaticSecret( - value=SecretStr("postgresql://user:pass@host:5432/db") - ), - } - - # Create a start conversation request with secrets - with tempfile.TemporaryDirectory() as temp_dir: - request = StartConversationRequest( - agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), - workspace=LocalWorkspace(working_dir=temp_dir), - confirmation_policy=NeverConfirm(), - secrets=test_secrets, - ) - - # Mock the EventService constructor and start method - with patch( - "openhands.agent_server.conversation_service.EventService" - ) as mock_event_service_class: - mock_event_service = AsyncMock(spec=EventService) - mock_event_service_class.return_value = mock_event_service - - # Mock the state that would be returned - mock_state = ConversationState( - id=uuid4(), - agent=request.agent, - workspace=request.workspace, - execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=request.confirmation_policy, - ) - mock_event_service.get_state.return_value = mock_state - mock_event_service.stored = StoredConversation( - id=mock_state.id, - **request.model_dump(mode="json", context={"expose_secrets": True}), - created_at=datetime.now(UTC), - updated_at=datetime.now(UTC), - ) - - # Start the conversation - result, _ = await conversation_service.start_conversation(request) - - # Verify EventService was created with the correct parameters - mock_event_service_class.assert_called_once() - call_args = mock_event_service_class.call_args - stored_conversation = call_args.kwargs["stored"] - - # Verify that secrets were passed to the stored conversation - assert stored_conversation.secrets == test_secrets - assert "api_key" in stored_conversation.secrets - assert "database_url" in stored_conversation.secrets - assert ( - stored_conversation.secrets["api_key"].get_value() - == "secret-api-key-123" - ) - assert ( - stored_conversation.secrets["database_url"].get_value() - == "postgresql://user:pass@host:5432/db" - ) - - # Verify the conversation was started - mock_event_service.start.assert_called_once() - - # Verify the result - assert result.id == mock_state.id - assert result.execution_status == ConversationExecutionStatus.IDLE - - @pytest.mark.asyncio - async def test_start_conversation_without_secrets(self, conversation_service): - """Test that conversations can be started without secrets.""" - # Create a start conversation request without secrets - with tempfile.TemporaryDirectory() as temp_dir: - request = StartConversationRequest( - agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), - workspace=LocalWorkspace(working_dir=temp_dir), - confirmation_policy=NeverConfirm(), - ) - - # Mock the EventService constructor and start method - with patch( - "openhands.agent_server.conversation_service.EventService" - ) as mock_event_service_class: - mock_event_service = AsyncMock(spec=EventService) - mock_event_service_class.return_value = mock_event_service - - # Mock the state that would be returned - mock_state = ConversationState( - id=uuid4(), - agent=request.agent, - workspace=request.workspace, - execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=request.confirmation_policy, - ) - mock_event_service.get_state.return_value = mock_state - mock_event_service.stored = StoredConversation( - id=mock_state.id, - **request.model_dump(mode="json", context={"expose_secrets": True}), - created_at=datetime.now(UTC), - updated_at=datetime.now(UTC), - ) - - # Start the conversation - result, _ = await conversation_service.start_conversation(request) - - # Verify EventService was created with the correct parameters - mock_event_service_class.assert_called_once() - call_args = mock_event_service_class.call_args - stored_conversation = call_args.kwargs["stored"] - - # Verify that secrets is an empty dict (default) - assert stored_conversation.secrets == {} - - # Verify the conversation was started - mock_event_service.start.assert_called_once() - - # Verify the result - assert result.id == mock_state.id - assert result.execution_status == ConversationExecutionStatus.IDLE - @pytest.mark.asyncio async def test_start_conversation_with_worktree_uses_git_worktree( self, conversation_service, tmp_path @@ -2162,7 +2006,6 @@ async def test_start_conversation_with_worktree_uses_git_worktree( conversation_id=conversation_id, agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=repo_dir), - confirmation_policy=NeverConfirm(), worktree=True, ) @@ -2180,7 +2023,6 @@ def _event_service_factory(**kwargs): agent=agent or _sample_agent(), workspace=stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored.confirmation_policy, ) return mock_event_service @@ -2246,7 +2088,6 @@ def _event_service_factory(**kwargs): agent=cast(AgentBase, kwargs.get("agent")) or _sample_agent(), workspace=stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored.confirmation_policy, ) return mock_event_service @@ -2281,7 +2122,6 @@ async def test_start_conversation_with_worktree_preserves_relative_workspace( conversation_id=conversation_id, agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=workspace_dir), - confirmation_policy=NeverConfirm(), worktree=True, ) @@ -2299,7 +2139,6 @@ def _event_service_factory(**kwargs): agent=agent or _sample_agent(), workspace=stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored.confirmation_policy, ) return mock_event_service @@ -2332,7 +2171,6 @@ async def test_start_conversation_with_worktree_ignores_non_git_workspace( conversation_id=conversation_id, agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=workspace_dir), - confirmation_policy=NeverConfirm(), worktree=True, ) @@ -2350,7 +2188,6 @@ def _event_service_factory(**kwargs): agent=agent or _sample_agent(), workspace=stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored.confirmation_policy, ) return mock_event_service @@ -2579,7 +2416,6 @@ async def test_start_conversation_with_custom_id(self, conversation_service): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=temp_dir), - confirmation_policy=NeverConfirm(), conversation_id=custom_id, ) @@ -2597,7 +2433,6 @@ async def test_start_conversation_with_duplicate_id(self, conversation_service): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=temp_dir), - confirmation_policy=NeverConfirm(), conversation_id=custom_id, ) @@ -2608,7 +2443,6 @@ async def test_start_conversation_with_duplicate_id(self, conversation_service): duplicate_request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=temp_dir), - confirmation_policy=NeverConfirm(), conversation_id=custom_id, ) @@ -2725,7 +2559,6 @@ async def test_start_conversation_reuse_checks_is_open(self, conversation_servic mock_event_service.stored = StoredConversation( id=custom_id, workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -2737,7 +2570,6 @@ async def test_start_conversation_reuse_checks_is_open(self, conversation_servic request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=temp_dir), - confirmation_policy=NeverConfirm(), conversation_id=custom_id, ) @@ -2749,7 +2581,6 @@ async def test_start_conversation_reuse_checks_is_open(self, conversation_servic mock_new_service.stored = StoredConversation( id=custom_id, workspace=request.workspace, - confirmation_policy=request.confirmation_policy, initial_message=request.initial_message, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -2760,7 +2591,6 @@ async def test_start_conversation_reuse_checks_is_open(self, conversation_servic agent=request.agent, workspace=request.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=request.confirmation_policy, ) mock_new_service.get_state.return_value = mock_state mock_start.return_value = mock_new_service @@ -2783,7 +2613,6 @@ async def test_start_conversation_reuse_when_open(self, conversation_service): mock_event_service.stored = StoredConversation( id=custom_id, workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -2794,7 +2623,6 @@ async def test_start_conversation_reuse_when_open(self, conversation_service): agent=_sample_agent(), workspace=mock_event_service.stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=mock_event_service.stored.confirmation_policy, ) mock_event_service.get_state.return_value = mock_state conversation_service._event_services[custom_id] = mock_event_service @@ -2803,7 +2631,6 @@ async def test_start_conversation_reuse_when_open(self, conversation_service): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=temp_dir), - confirmation_policy=NeverConfirm(), conversation_id=custom_id, ) @@ -2827,7 +2654,6 @@ async def test_start_conversation_returns_existing_acp_conversation( stored = StoredConversation( id=custom_id, workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -2844,7 +2670,6 @@ async def test_start_conversation_returns_existing_acp_conversation( agent=acp_agent, workspace=stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored.confirmation_policy, ) conversation_service._event_services[custom_id] = mock_event_service @@ -2852,7 +2677,6 @@ async def test_start_conversation_returns_existing_acp_conversation( request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=temp_dir), - confirmation_policy=NeverConfirm(), conversation_id=custom_id, ) @@ -2879,7 +2703,6 @@ async def test_start_event_service_failure_cleanup(self, conversation_service): stored = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir=temp_dir), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -2914,7 +2737,6 @@ async def test_start_event_service_success_stores_service( stored = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir=temp_dir), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -2964,7 +2786,6 @@ async def test_update_conversation_success( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -2996,7 +2817,6 @@ async def test_update_conversation_strips_whitespace( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -3029,7 +2849,6 @@ async def test_update_conversation_tags_uses_state_lock( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) acquire_spy = MagicMock(wraps=mock_state._lock.acquire) release_spy = MagicMock(wraps=mock_state._lock.release) @@ -3065,7 +2884,6 @@ async def test_update_conversation_tags_wait_does_not_block_event_loop( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = state @@ -3140,7 +2958,6 @@ async def test_update_conversation_tags_wait_leaves_default_executor_free( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.RUNNING, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = state conversation_id = sample_stored_conversation.id @@ -3211,7 +3028,6 @@ async def test_update_conversation_notifies_webhooks( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -3245,7 +3061,6 @@ async def test_update_acp_conversation_notifies_webhooks_with_acp_shape( stored_conversation = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -3260,7 +3075,6 @@ async def test_update_acp_conversation_notifies_webhooks_with_acp_shape( agent=acp_agent, workspace=stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -3293,7 +3107,6 @@ async def test_update_conversation_persists_changes( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -3326,7 +3139,6 @@ async def test_update_conversation_multiple_times( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -3378,7 +3190,6 @@ async def test_update_conversation_sets_updated_at( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -3437,7 +3248,6 @@ async def test_delete_conversation_success(self, conversation_service): mock_service.stored = StoredConversation( id=conversation_id, workspace=LocalWorkspace(working_dir="/tmp/test_workspace"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -3448,7 +3258,6 @@ async def test_delete_conversation_success(self, conversation_service): agent=_sample_agent(), workspace=mock_service.stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=mock_service.stored.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -3494,7 +3303,6 @@ async def test_delete_conversation_notifies_webhooks_with_deleting_status( agent=_sample_agent(), workspace=sample_stored_conversation.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=sample_stored_conversation.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -3546,7 +3354,6 @@ async def test_delete_conversation_webhook_failure(self, conversation_service): mock_service.stored = StoredConversation( id=conversation_id, workspace=LocalWorkspace(working_dir="/tmp/test_workspace"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -3588,7 +3395,6 @@ async def test_delete_conversation_close_failure(self, conversation_service): mock_service.stored = StoredConversation( id=conversation_id, workspace=LocalWorkspace(working_dir="/tmp/test_workspace"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -3599,7 +3405,6 @@ async def test_delete_conversation_close_failure(self, conversation_service): agent=_sample_agent(), workspace=mock_service.stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=mock_service.stored.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -3636,14 +3441,12 @@ async def test_delete_conversation_retains_retryable_credential_close( mock_service.stored = StoredConversation( id=conversation_id, workspace=LocalWorkspace(working_dir=tmp_path / "workspace"), - confirmation_policy=NeverConfirm(), ) mock_service.get_state.return_value = ConversationState( id=conversation_id, agent=_sample_agent(), workspace=mock_service.stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=mock_service.stored.confirmation_policy, ) mock_service.close.side_effect = CredentialSyncError("broker unavailable") binding = MagicMock() @@ -3682,7 +3485,6 @@ async def test_delete_conversation_directory_removal_failure( mock_service.stored = StoredConversation( id=conversation_id, workspace=LocalWorkspace(working_dir="/tmp/test_workspace"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -3693,7 +3495,6 @@ async def test_delete_conversation_directory_removal_failure( agent=_sample_agent(), workspace=mock_service.stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=mock_service.stored.confirmation_policy, ) mock_service.get_state.return_value = mock_state @@ -3814,7 +3615,6 @@ def _make_service( stored = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, title=title, @@ -4655,7 +4455,6 @@ def _seed(conversations_dir: Path, count: int, workspace_dir: Path) -> list[UUID stored = StoredConversation( id=conversation_id, workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) stored.created_at = datetime(2026, 1, 1, tzinfo=UTC).replace(microsecond=i) stored.updated_at = stored.created_at @@ -4805,7 +4604,6 @@ async def test_live_conversation_status_is_read_from_memory(self, tmp_path): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="live-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService(conversations_dir=conversations_dir) as svc: live, _ = await svc.start_conversation(request) @@ -4959,7 +4757,6 @@ async def test_search_live_conversation_does_not_wait_for_state_lock(tmp_path): request = StartConversationRequest( agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), workspace=LocalWorkspace(working_dir=str(workspace_dir)), - confirmation_policy=NeverConfirm(), ) async with ConversationService(conversations_dir=conversations_dir) as service: @@ -5126,7 +4923,6 @@ async def fake_start_event_service(stored: StoredConversation, **kwargs): agent=agent, workspace=stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored.confirmation_policy, ) return service @@ -5173,7 +4969,6 @@ async def fake_start_event_service(stored: StoredConversation, **kwargs): agent=agent, workspace=stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored.confirmation_policy, ) return service @@ -5213,7 +5008,6 @@ async def fake_start_event_service(stored: StoredConversation, **kwargs): agent=agent, workspace=stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored.confirmation_policy, ) return service @@ -5235,3 +5029,135 @@ async def fake_start_event_service(stored: StoredConversation, **kwargs): create_agent.assert_called_once() assert "browser_tool_set" in [t.name for t in captured["agent"].tools] + + +@pytest.mark.asyncio +async def test_start_conversation_decrypts_encrypted_security_analyzer( + conversation_service, tmp_path +): + """With secrets_encrypted=True the analyzer's nested LLM api_key must be + decrypted via the cipher before reaching the live EventService, exactly as + the agent and the secrets dict are. + """ + cipher = Cipher("analyzer-cipher-key") + conversation_service.cipher = cipher + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + + encrypted_guard_key = cipher.encrypt(SecretStr("sk-GUARD-plaintext")) + assert encrypted_guard_key is not None + request = StartConversationRequest( + agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), + workspace=LocalWorkspace(working_dir=str(workspace_dir)), + security_analyzer=ToolShieldLLMSecurityAnalyzer( + llm=LLM( + model="gpt-4o-mini", + usage_id="guard", + api_key=SecretStr(encrypted_guard_key), + ) + ), + secrets_encrypted=True, + ) + # The request object parsed from the HTTP body still carries ciphertext. + assert isinstance(request.security_analyzer, ToolShieldLLMSecurityAnalyzer) + assert isinstance(request.security_analyzer.llm.api_key, SecretStr) + assert request.security_analyzer.llm.api_key.get_secret_value() == ( + encrypted_guard_key + ) + + captured: dict[str, Any] = {} + + async def fake_start_event_service(stored: StoredConversation, **kwargs): + agent = kwargs["launched"].agent + captured["security_analyzer"] = kwargs.get("security_analyzer") + service = AsyncMock(spec=EventService) + service.stored = stored + service.get_state.return_value = ConversationState( + id=stored.id, + agent=agent, + workspace=stored.workspace, + execution_status=ConversationExecutionStatus.IDLE, + ) + return service + + with patch.object( + conversation_service, + "_start_event_service", + side_effect=fake_start_event_service, + ): + await conversation_service.start_conversation(request) + + analyzer = captured["security_analyzer"] + assert isinstance(analyzer, ToolShieldLLMSecurityAnalyzer) + assert isinstance(analyzer.llm.api_key, SecretStr) + assert analyzer.llm.api_key.get_secret_value() == "sk-GUARD-plaintext" + + +@pytest.mark.asyncio +async def test_fork_conversation_preserves_secrets(tmp_path): + """A fork made through the server has the source's secrets.""" + conversations_dir = tmp_path / "conversations" + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + cipher = Cipher("fork-secret-key") + request = StartConversationRequest( + agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), + workspace=LocalWorkspace(working_dir=str(workspace_dir)), + secrets={"MY_TOKEN": StaticSecret(value=SecretStr("s3kr3t-value"))}, + ) + async with ConversationService( + conversations_dir=conversations_dir, cipher=cipher + ) as service: + info, _ = await service.start_conversation(request) + + fork_info = await service.fork_conversation(info.id) + assert fork_info is not None + assert fork_info.id != info.id + + fork_event_service = await service.get_event_service(fork_info.id) + assert fork_event_service is not None + sources = (await fork_event_service.get_state()).secret_registry.secret_sources + assert sources["MY_TOKEN"].get_value() == "s3kr3t-value" + + +@pytest.mark.asyncio +async def test_start_conversation_decrypts_encrypted_conversation_secrets(tmp_path): + """With secrets_encrypted=True (the Agent Canvas create path) a conversation + secret arrives as cipher text and the conversation gets its plain value.""" + cipher = Cipher("request-secrets-key") + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + encrypted = cipher.encrypt(SecretStr("plain-token-value")) + assert encrypted is not None + request = StartConversationRequest( + agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), + workspace=LocalWorkspace(working_dir=str(workspace_dir)), + secrets={"TOKEN": StaticSecret(value=SecretStr(encrypted))}, + secrets_encrypted=True, + ) + + async with ConversationService( + conversations_dir=tmp_path / "conversations", cipher=cipher + ) as service: + info, _ = await service.start_conversation(request) + event_service = await service.get_event_service(info.id) + assert event_service is not None + state = await event_service.get_state() + token = state.secret_registry.secret_sources["TOKEN"] + assert token.get_value() == "plain-token-value" + + +def test_request_security_analyzer_refuses_encrypted_secrets_without_a_cipher( + tmp_path, +): + """secrets_encrypted=True on a server with no cipher is an error, not an + analyzer whose key is the cipher text.""" + request = StartConversationRequest( + agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), + workspace=LocalWorkspace(working_dir=str(tmp_path)), + security_analyzer=LLMSecurityAnalyzer(), + secrets_encrypted=True, + ) + + with pytest.raises(ValueError, match="cipher not configured"): + request_security_analyzer(request, None) diff --git a/tests/agent_server/test_conversation_tags.py b/tests/agent_server/test_conversation_tags.py index 42ffe96b9e2..97c881879af 100644 --- a/tests/agent_server/test_conversation_tags.py +++ b/tests/agent_server/test_conversation_tags.py @@ -22,7 +22,6 @@ from openhands.agent_server.utils import utc_now from openhands.sdk import LLM, Agent, Tool from openhands.sdk.conversation.state import ConversationExecutionStatus -from openhands.sdk.security.confirmation_policy import NeverConfirm from openhands.sdk.workspace import LocalWorkspace from tests.agent_server.conftest import install_test_context @@ -241,7 +240,6 @@ async def test_event_service_start_forwards_tags_to_local_conversation(tmp_path) stored = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir=str(tmp_path)), - confirmation_policy=NeverConfirm(), tags=tags, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), updated_at=datetime(2025, 1, 1, 12, 30, 0, tzinfo=UTC), @@ -282,7 +280,6 @@ async def test_event_service_start_forwards_observability_span_name(tmp_path): stored = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir=str(tmp_path)), - confirmation_policy=NeverConfirm(), observability_span_name="pr_review_evaluation", created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), updated_at=datetime(2025, 1, 1, 12, 30, 0, tzinfo=UTC), diff --git a/tests/agent_server/test_credential_binding.py b/tests/agent_server/test_credential_binding.py index d7bc87de659..a7f89836769 100644 --- a/tests/agent_server/test_credential_binding.py +++ b/tests/agent_server/test_credential_binding.py @@ -1,4 +1,5 @@ import json +import shutil import threading from collections.abc import Generator from dataclasses import dataclass, field @@ -410,7 +411,8 @@ async def test_direct_start_strips_reserved_conversation_secret(tmp_path) -> Non info, _ = await service.start_conversation(request) event_service = await service.get_event_service(info.id) assert event_service is not None - assert "CODEX_AUTH_JSON" not in event_service.stored.secrets + state = await event_service.get_state() + assert "CODEX_AUTH_JSON" not in state.secret_registry.secret_sources assert "CODEX_AUTH_JSON" in event_service.credential_bindings assert not event_service.stored.required_runtime_credential_bindings @@ -444,7 +446,8 @@ def binding() -> HttpVersionedCredentialBinding: event_service = await service.get_event_service(conversation_id) assert event_service is not None assert event_service.credential_bindings == {} - assert "CODEX_AUTH_JSON" in event_service.stored.secrets + state = await event_service.get_state() + assert "CODEX_AUTH_JSON" in state.secret_registry.secret_sources # Pending binding consumed when the unloaded conversation loads again. await service.prepare_for_sandbox_pause() @@ -458,7 +461,8 @@ def binding() -> HttpVersionedCredentialBinding: meta = json.loads(meta_file.read_text()) assert meta["required_runtime_credential_bindings"] == [] - assert "CODEX_AUTH_JSON" in meta["secrets"] + base_state = json.loads((meta_file.parent / "base_state.json").read_text()) + assert "CODEX_AUTH_JSON" in base_state["secret_registry"]["secret_sources"] assert await service.get_event_service(conversation_id) is not None @@ -592,10 +596,8 @@ async def test_one_off_credential_survives_cold_restart_without_binding( event_service = await restarted.get_event_service(info.id) assert event_service is not None assert not event_service.credential_bindings - assert ( - event_service.stored.secrets["CODEX_AUTH_JSON"].get_value() - == "one-off-credential" - ) + sources = (await event_service.get_state()).secret_registry.secret_sources + assert sources["CODEX_AUTH_JSON"].get_value() == "one-off-credential" @pytest.mark.asyncio @@ -631,8 +633,6 @@ async def test_managed_start_scrubs_all_durable_credential_copies(tmp_path) -> N assert event_service is not None state = await event_service.get_state() - assert "CODEX_AUTH_JSON" not in event_service.stored.secrets - assert "KEEP" in event_service.stored.secrets # The agent is no longer stored on meta.json; it lives on the live # conversation / base_state.json. Assert the scrub there. assert event_service._conversation is not None @@ -648,7 +648,6 @@ async def test_managed_start_scrubs_all_durable_credential_copies(tmp_path) -> N conversation_dir = tmp_path / "conversations" / info.id.hex meta = json.loads((conversation_dir / "meta.json").read_text()) base_state = json.loads((conversation_dir / "base_state.json").read_text()) - assert "CODEX_AUTH_JSON" not in meta["secrets"] # meta.json no longer carries the agent at all. assert "agent" not in meta assert "CODEX_AUTH_JSON" not in base_state["agent"]["agent_context"]["secrets"] @@ -730,7 +729,6 @@ def fail_first_write(*args, **kwargs): state = await event_service.get_state() assert event_service.credential_bindings["CODEX_AUTH_JSON"] is binding - assert "CODEX_AUTH_JSON" not in event_service.stored.secrets assert "CODEX_AUTH_JSON" not in state.secret_registry.secret_sources # The agent is on the live conversation / base_state.json, not meta.json. assert event_service._conversation is not None @@ -741,7 +739,6 @@ def fail_first_write(*args, **kwargs): conversation_dir = tmp_path / "conversations" / info.id.hex meta = json.loads((conversation_dir / "meta.json").read_text()) base_state = json.loads((conversation_dir / "base_state.json").read_text()) - assert "CODEX_AUTH_JSON" not in meta["secrets"] assert "agent" not in meta assert "CODEX_AUTH_JSON" not in base_state["agent"]["agent_context"]["secrets"] assert "CODEX_AUTH_JSON" not in base_state["secret_registry"]["secret_sources"] @@ -984,54 +981,96 @@ async def test_resume_uses_plaintext_fallback_without_binding(tmp_path) -> None: assert resumed is not None state = await resumed.get_state() assert "CODEX_AUTH_JSON" not in resumed.credential_bindings - assert "CODEX_AUTH_JSON" in resumed.stored.secrets assert "CODEX_AUTH_JSON" in state.secret_registry.secret_sources @pytest.mark.asyncio -async def test_failed_resume_does_not_retain_plaintext_fallback( - tmp_path, - monkeypatch, -) -> None: +@pytest.mark.parametrize("loaded", [True, False], ids=["live", "cold"]) +async def test_resume_decrypts_an_encrypted_fallback(tmp_path, loaded) -> None: + """A request that marks its secrets as encrypted has them decrypted on a + resume, as on a create.""" + cipher = Cipher("resume-fallback-key") request = StartConversationRequest( agent=ACPAgent(acp_command=["codex-acp"], acp_server="codex"), workspace=LocalWorkspace(working_dir=tmp_path / "workspace"), ) async with ConversationService( - conversations_dir=tmp_path / "conversations" + conversations_dir=tmp_path / "conversations", cipher=cipher ) as service: info, _ = await service.start_conversation(request) - assert service._event_services is not None - event_service = service._event_services.pop(info.id) - await event_service.close() + if not loaded: + assert service._event_services is not None + await service._event_services.pop(info.id).close() + + resume = request.model_copy( + update={ + "conversation_id": info.id, + "secrets": { + "CODEX_AUTH_JSON": StaticSecret(value=SecretStr("fallback-copy")) + }, + } + ) + await service.start_conversation( + StartConversationRequest.model_validate( + resume.model_dump(mode="json", context={"cipher": cipher}) + | {"secrets_encrypted": True} + ) + ) + + resumed = await service.get_event_service(info.id) + assert resumed is not None + sources = (await resumed.get_state()).secret_registry.secret_sources + assert sources["CODEX_AUTH_JSON"].get_value() == "fallback-copy" + - async def fail_load(conversation_id, **kwargs): - assert conversation_id == info.id - raise RuntimeError("resume failed") +@pytest.mark.asyncio +async def test_secrets_from_an_older_meta_json_survive_a_failed_first_load( + tmp_path, +) -> None: + """An older server could leave a start-time secret only in meta.json. The + credential scrub rewrites that file during a load, so the secret is moved + to base_state.json first and a load that fails afterwards keeps it.""" + stores = _server_stores(tmp_path / "settings") + stores.secrets.set_secret("CODEX_AUTH_JSON", "canonical") + cipher = Cipher("legacy-meta-key") + conversations_dir = tmp_path / "conversations" + workspace_dir = tmp_path / "workspace" + request = StartConversationRequest( + agent=ACPAgent(acp_command=["codex-acp"], acp_server="codex"), + workspace=LocalWorkspace(working_dir=workspace_dir), + ) - monkeypatch.setattr( - service, - "_get_or_load_event_service_locked", - fail_load, + async with ConversationService( + conversations_dir=conversations_dir, stores=stores, cipher=cipher + ) as service: + info, _ = await service.start_conversation(request) + + meta_file = conversations_dir / info.id.hex / "meta.json" + meta = json.loads(meta_file.read_text()) + meta["secrets"] = { + "ONLY_IN_META": StaticSecret(value=SecretStr("legacy-value")).model_dump( + mode="json", context={"cipher": cipher} ) - with pytest.raises(RuntimeError, match="resume failed"): - await service.start_conversation( - request.model_copy( - update={ - "conversation_id": info.id, - "secrets": { - "CODEX_AUTH_JSON": StaticSecret( - value=SecretStr("temporary-fallback") - ) - }, - } - ) - ) + } + meta_file.write_text(json.dumps(meta)) + shutil.rmtree(workspace_dir) + workspace_dir.write_text("not a directory") + + async with ConversationService( + conversations_dir=conversations_dir, stores=stores, cipher=cipher + ) as service: + with pytest.raises(OSError): + await service.get_event_service(info.id) + assert "secrets" not in json.loads(meta_file.read_text()) + base_state = json.loads((meta_file.parent / "base_state.json").read_text()) + assert "ONLY_IN_META" in base_state["secret_registry"]["secret_sources"] + workspace_dir.unlink() - record = service._conversation_records[info.id] - assert "CODEX_AUTH_JSON" not in record.stored.secrets - assert "temporary-fallback" not in record.stored.model_dump_json() + restored = await service.get_event_service(info.id) + assert restored is not None + secrets = (await restored.get_state()).secret_registry.secret_sources + assert secrets["ONLY_IN_META"].get_value() == "legacy-value" @pytest.mark.asyncio @@ -1154,11 +1193,8 @@ async def test_resume_removes_legacy_persisted_credential(tmp_path) -> None: ) assert not started - record = service._conversation_records[info.id] - assert "CODEX_AUTH_JSON" not in record.stored.secrets meta = json.loads((conversation_dir / "meta.json").read_text()) base_state = json.loads((conversation_dir / "base_state.json").read_text()) - assert "CODEX_AUTH_JSON" not in meta["secrets"] # meta.json no longer carries the agent; the agent (and its scrubbed # agent_context) lives solely in base_state.json. assert "agent" not in meta @@ -1199,3 +1235,84 @@ async def test_failed_canonical_preflight_preserves_legacy_auth_file(tmp_path) - ) assert legacy_auth_file.read_text(encoding="utf-8") == "only-legacy-copy" + + +@pytest.mark.asyncio +async def test_start_keeps_a_bound_credential_out_of_the_registry(tmp_path) -> None: + """A start secret named like a bound credential never reaches the + conversation: the binding is its only source.""" + store = FileSecretsStore(tmp_path / "settings") + store.set_secret("CODEX_AUTH_JSON", "canonical") + service = EventService( + stored=StoredConversation( + id=uuid4(), + workspace=LocalWorkspace(working_dir=tmp_path / "workspace"), + ), + conversations_dir=tmp_path / "conversations", + agent=ACPAgent(acp_command=["codex-acp"], acp_server="codex"), + secrets={ + "CODEX_AUTH_JSON": StaticSecret(value=SecretStr("request-copy")), + "KEEP": StaticSecret(value=SecretStr("keep-value")), + }, + credential_bindings={ + "CODEX_AUTH_JSON": LocalVersionedCredentialBinding(store, "CODEX_AUTH_JSON") + }, + ) + await service.start() + try: + sources = (await service.get_state()).secret_registry.secret_sources + assert set(sources) == {"KEEP"} + finally: + await service.close() + + +@pytest.mark.asyncio +async def test_resume_takes_legacy_meta_secrets_but_not_a_bound_credential( + tmp_path, +) -> None: + """An older server's meta.json can hold the Codex credential next to other + start-time secrets. With a managed credential bound, the first resume takes + over the others and keeps that one out of base_state.json.""" + stores = _server_stores(tmp_path / "settings") + cipher = Cipher("legacy-binding-key") + conversations_dir = tmp_path / "conversations" + request = StartConversationRequest( + agent=ACPAgent(acp_command=["codex-acp"], acp_server="codex"), + workspace=LocalWorkspace(working_dir=tmp_path / "workspace"), + ) + async with ConversationService( + conversations_dir=conversations_dir, stores=stores, cipher=cipher + ) as service: + info, _ = await service.start_conversation(request) + + conversation_dir = conversations_dir / info.id.hex + meta_file = conversation_dir / "meta.json" + meta = json.loads(meta_file.read_text()) + meta["secrets"] = { + name: StaticSecret(value=SecretStr(value)).model_dump( + mode="json", context={"cipher": cipher} + ) + for name, value in { + "CODEX_AUTH_JSON": "legacy-copy", + "KEEP": "keep-value", + }.items() + } + meta_file.write_text(json.dumps(meta)) + stores.secrets.set_secret( + "CODEX_AUTH_JSON", + json.dumps({"auth_mode": "chatgpt", "tokens": {"refresh_token": "r"}}), + ) + + async with ConversationService( + conversations_dir=conversations_dir, stores=stores, cipher=cipher + ) as service: + resumed = await service.get_event_service(info.id) + assert resumed is not None + assert "CODEX_AUTH_JSON" in resumed.credential_bindings + sources = (await resumed.get_state()).secret_registry.secret_sources + assert sources["KEEP"].get_value() == "keep-value" + assert "CODEX_AUTH_JSON" not in sources + + base_state = json.loads((conversation_dir / "base_state.json").read_text()) + assert "CODEX_AUTH_JSON" not in base_state["secret_registry"]["secret_sources"] + assert "secrets" not in json.loads(meta_file.read_text()) diff --git a/tests/agent_server/test_event_service.py b/tests/agent_server/test_event_service.py index 06a2b496f19..e605a1bfb01 100644 --- a/tests/agent_server/test_event_service.py +++ b/tests/agent_server/test_event_service.py @@ -71,11 +71,11 @@ from openhands.sdk.llm.llm_registry import LLMRegistry from openhands.sdk.mcp.config import coerce_mcp_config from openhands.sdk.secret import StaticSecret -from openhands.sdk.security.confirmation_policy import NeverConfirm +from openhands.sdk.security.confirmation_policy import AlwaysConfirm, NeverConfirm +from openhands.sdk.security.llm_analyzer import LLMSecurityAnalyzer from openhands.sdk.skills import Skill from openhands.sdk.subagent.schema import AgentDefinition -from openhands.sdk.utils.cipher import FERNET_TOKEN_PREFIX, Cipher -from openhands.sdk.utils.pydantic_secrets import REDACTED_SECRET_VALUE +from openhands.sdk.utils.cipher import Cipher from openhands.sdk.workspace import LocalWorkspace from openhands.tools.terminal import TerminalAction, TerminalObservation from tests.agent_server.stress.scripts import ( @@ -114,7 +114,6 @@ def sample_stored_conversation(): return StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -2345,23 +2344,14 @@ async def test_save_meta_never_exposes_a_partial_meta_json( async def test_trajectory_zip_built_during_save_meta_has_no_partial_meta_json( self, sample_stored_conversation, tmp_path, monkeypatch ): - """A trajectory zip built mid-save holds a complete, redacted meta.json. - - Partial JSON does not parse, so the zip cannot redact it, and the - encrypted secrets in it would be archived as they are. - """ - sample_stored_conversation.secrets = { - "GITHUB_TOKEN": StaticSecret(value=SecretStr("ghp_zip_during_save")) - } + """A trajectory zip built mid-save holds a complete meta.json, never a + half-written one.""" service = EventService( stored=sample_stored_conversation, conversations_dir=tmp_path, - cipher=Cipher("trajectory-zip-during-save-meta"), ) service.conversation_dir.mkdir() await service.save_meta() - meta_file = service.conversation_dir / "meta.json" - assert FERNET_TOKEN_PREFIX in meta_file.read_text() archive_path = tmp_path / "trajectory.zip" _pause_text_writes_halfway( monkeypatch, @@ -2372,14 +2362,8 @@ async def test_trajectory_zip_built_during_save_meta_has_no_partial_meta_json( await service.save_meta() with zipfile.ZipFile(archive_path) as archive: - members = {name: archive.read(name) for name in archive.namelist()} - assert not [ - name - for name, data in members.items() - if FERNET_TOKEN_PREFIX.encode() in data - ] - meta = json.loads(members[f"{service.stored.id.hex}/meta.json"]) - assert meta["secrets"]["GITHUB_TOKEN"]["value"] == REDACTED_SECRET_VALUE + meta = json.loads(archive.read(f"{service.stored.id.hex}/meta.json")) + assert StoredConversation.model_validate(meta).id == service.stored.id @pytest.mark.asyncio async def test_switch_acp_model_persists_via_conversation(self, tmp_path): @@ -2395,7 +2379,6 @@ async def test_switch_acp_model_persists_via_conversation(self, tmp_path): stored = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir=str(tmp_path)), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, ) @@ -2441,7 +2424,6 @@ async def test_switch_acp_model_inactive_service_raises_service_inactive( stored = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir=str(tmp_path)), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, ) @@ -3698,7 +3680,6 @@ def _make_service_with_callback(self): stored = StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir="workspace/project"), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), @@ -4093,7 +4074,6 @@ def _make_stored(tmp_path: Path) -> StoredConversation: return StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir=str(tmp_path)), - confirmation_policy=NeverConfirm(), initial_message=None, metrics=None, ) @@ -4151,6 +4131,73 @@ async def test_event_service_creates_lease_with_custom_ttl(tmp_path: Path) -> No assert (tmp_path / stored.id.hex / LEASE_FILE_NAME).exists() +@pytest.mark.asyncio +async def test_new_conversation_keeps_its_start_settings_in_base_state_only(tmp_path): + """The start policy, analyzer and secrets seed the conversation state; + meta.json gets none of them (#4810).""" + stored = _make_stored(tmp_path / "workspace") + service = EventService( + stored=stored, + conversations_dir=tmp_path / "conversations", + agent=_sample_agent(), + initial_confirmation_policy=AlwaysConfirm(), + initial_security_analyzer=LLMSecurityAnalyzer(), + secrets={"TOKEN": StaticSecret(value=SecretStr("start-value"))}, + cipher=Cipher("event-service-key"), + ) + await service.start() + try: + state = await service.get_state() + assert isinstance(state.confirmation_policy, AlwaysConfirm) + assert isinstance(state.security_analyzer, LLMSecurityAnalyzer) + token = state.secret_registry.secret_sources["TOKEN"] + assert token.get_value() == "start-value" + await service.save_meta() + finally: + await service.close() + + conversation_dir = tmp_path / "conversations" / stored.id.hex + meta = json.loads((conversation_dir / "meta.json").read_text()) + assert not {"confirmation_policy", "security_analyzer", "secrets"} & meta.keys() + base_state = json.loads((conversation_dir / "base_state.json").read_text()) + assert base_state["confirmation_policy"]["kind"] == "AlwaysConfirm" + assert base_state["security_analyzer"]["kind"] == "LLMSecurityAnalyzer" + assert "TOKEN" in base_state["secret_registry"]["secret_sources"] + + +@pytest.mark.asyncio +async def test_resume_keeps_the_saved_policy_and_analyzer_over_the_start_fields( + tmp_path, +): + """A resume takes the policy and the analyzer from base_state.json, whatever + the EventService was constructed with.""" + stored = _make_stored(tmp_path / "workspace") + conversations_dir = tmp_path / "conversations" + first = EventService( + stored=stored, + conversations_dir=conversations_dir, + agent=_sample_agent(), + initial_confirmation_policy=AlwaysConfirm(), + ) + await first.start() + await first.set_security_analyzer(LLMSecurityAnalyzer()) + await first.close() + + resumed = EventService( + stored=stored, + conversations_dir=conversations_dir, + initial_confirmation_policy=NeverConfirm(), + initial_security_analyzer=None, + ) + await resumed.start() + try: + state = await resumed.get_state() + assert isinstance(state.confirmation_policy, AlwaysConfirm) + assert isinstance(state.security_analyzer, LLMSecurityAnalyzer) + finally: + await resumed.close() + + @pytest.mark.asyncio async def test_event_service_checks_the_workspace_repo_off_the_event_loop( tmp_path: Path, diff --git a/tests/agent_server/test_managed_llm_key.py b/tests/agent_server/test_managed_llm_key.py index 3b667f447c8..8887cbdc346 100644 --- a/tests/agent_server/test_managed_llm_key.py +++ b/tests/agent_server/test_managed_llm_key.py @@ -33,7 +33,6 @@ register_local_secret_resolver, unregister_local_secret_resolver, ) -from openhands.sdk.security.confirmation_policy import NeverConfirm from openhands.sdk.workspace import LocalWorkspace @@ -284,7 +283,6 @@ async def test_switched_llm_keeps_refresh_hook(monkeypatch, tmp_path): stored=StoredConversation( id=uuid4(), workspace=LocalWorkspace(working_dir=str(workspace)), - confirmation_policy=NeverConfirm(), ), agent=_agent(_llm("loaded", MANAGED_BASE_URL)), conversations_dir=tmp_path / "conversations", diff --git a/tests/agent_server/test_models.py b/tests/agent_server/test_models.py index d8084df4e59..96abc699b9b 100644 --- a/tests/agent_server/test_models.py +++ b/tests/agent_server/test_models.py @@ -5,7 +5,11 @@ import pytest from pydantic import SecretStr, ValidationError -from openhands.agent_server.models import UpdateSecretsRequest +from openhands.agent_server.models import ( + ConversationConfig, + StoredConversation, + UpdateSecretsRequest, +) from openhands.sdk.secret import LookupSecret, StaticSecret @@ -116,3 +120,25 @@ def test_update_secrets_request_invalid_input(): # Non-dict input should be preserved (will fail validation later) with pytest.raises(ValidationError): UpdateSecretsRequest(secrets="not-a-dict") # type: ignore[arg-type] + + +_START_ONLY_FIELDS = { + "confirmation_policy", + "security_analyzer", + "secrets", + "secrets_encrypted", +} + + +def test_stored_conversation_has_no_field_for_what_the_conversation_state_owns(): + """``meta.json`` cannot hold a second copy of what ``base_state.json`` owns.""" + owned_by_state = _START_ONLY_FIELDS | {"agent", "max_budget_per_run"} + + assert owned_by_state.isdisjoint(StoredConversation.model_fields) + + +def test_conversation_config_keeps_the_fields_released_servers_read(): + """Agent-servers up to 1.62.0 build their record on ``ConversationConfig`` + and read these fields from it, and they do not pin the SDK version they run + on.""" + assert _START_ONLY_FIELDS <= set(ConversationConfig.model_fields) diff --git a/tests/agent_server/test_parent_child_conversations.py b/tests/agent_server/test_parent_child_conversations.py index d4452f7f2e8..018ab98df95 100644 --- a/tests/agent_server/test_parent_child_conversations.py +++ b/tests/agent_server/test_parent_child_conversations.py @@ -1,5 +1,6 @@ """Tests for parent/child conversation relationships.""" +import asyncio from dataclasses import dataclass from pathlib import Path from types import SimpleNamespace @@ -10,6 +11,7 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient +from pydantic import SecretStr from openhands.agent_server.api import add_domain_error_handler from openhands.agent_server.conversation_router import conversation_router @@ -30,6 +32,7 @@ ) from openhands.sdk import LLM, Agent, Message, TextContent from openhands.sdk.agent.base import AgentBase +from openhands.sdk.conversation.secret_registry import SecretRegistry from openhands.sdk.conversation.state import ( ConversationExecutionStatus, ConversationState, @@ -37,6 +40,9 @@ from openhands.sdk.event import MessageEvent from openhands.sdk.git.exceptions import GitCommandError from openhands.sdk.git.utils import run_git_command +from openhands.sdk.secret import StaticSecret +from openhands.sdk.security.confirmation_policy import AlwaysConfirm +from openhands.sdk.security.llm_analyzer import LLMSecurityAnalyzer from openhands.sdk.workspace import LocalWorkspace @@ -294,7 +300,9 @@ def factory(**kwargs): agent=agent, workspace=stored.workspace, execution_status=ConversationExecutionStatus.IDLE, - confirmation_policy=stored.confirmation_policy, + confirmation_policy=kwargs["initial_confirmation_policy"], + security_analyzer=kwargs["initial_security_analyzer"], + secret_registry=SecretRegistry(secret_sources=kwargs["secrets"]), ) services[stored.id] = event_service return event_service @@ -313,13 +321,16 @@ class _ChildLaunch: async def _launch_child( - tmp_path: Path, workspace_dir: Path, request: StartChildConversationRequest + tmp_path: Path, + workspace_dir: Path, + request: StartChildConversationRequest, + **parent_overrides: Any, ) -> _ChildLaunch: """Start a parent in *workspace_dir*, then launch a child from it.""" services: dict[UUID, Any] = {} worktree_root = tmp_path / "conversation-worktrees" parent_request = _start_request(workspace_dir).model_copy( - update={"tags": {"source": "test"}} + update={"tags": {"source": "test"}, **parent_overrides} ) with patch( "openhands.agent_server.conversation_service.EventService", @@ -399,6 +410,102 @@ async def test_start_child_conversation_inherits_parent_configuration( assert launch.result.title == "Docs" +@pytest.mark.asyncio +async def test_start_child_conversation_inherits_parent_state_config( + tmp_path, workspace_dir +): + """Policy, analyzer and secrets live in base_state.json, not meta.json.""" + request = StartChildConversationRequest(task="Write the docs") + + launch = await _launch_child( + tmp_path, + workspace_dir, + request, + confirmation_policy=AlwaysConfirm(), + security_analyzer=LLMSecurityAnalyzer(), + secrets={"TOKEN": StaticSecret(value=SecretStr("shh"))}, + ) + + assert launch.result is not None + parent_state = await launch.services[launch.parent_id].get_state() + child_state = await launch.services[launch.result.conversation_id].get_state() + assert isinstance(child_state.confirmation_policy, AlwaysConfirm) + assert isinstance(child_state.security_analyzer, LLMSecurityAnalyzer) + assert child_state.security_analyzer is not parent_state.security_analyzer + token = child_state.secret_registry.secret_sources["TOKEN"] + assert token.get_value() == "shh" + assert token is not parent_state.secret_registry.secret_sources["TOKEN"] + + +def _launch_child_like_the_tool( + service: ConversationService, + parent_id: UUID, + loop: asyncio.AbstractEventLoop, +) -> StartChildConversationResponse | None: + """Block a thread on a launch the event loop has to answer. + + The ``start_child_conversation`` tool does this with a synchronous HTTP + request to this server's ``/children`` route. + """ + future = asyncio.run_coroutine_threadsafe( + service.start_child_conversation( + parent_id, StartChildConversationRequest(task="Write the docs") + ), + loop, + ) + try: + return future.result(timeout=5) + except BaseException: + future.cancel() + raise + + +@pytest.mark.asyncio +@pytest.mark.parametrize("step_runs_on", ["event_loop", "worker_thread"]) +async def test_start_child_conversation_does_not_wait_for_parent_state_lock( + tmp_path, workspace_dir, step_runs_on +): + """A parent starts a child from inside its own step. + + The step holds the parent's state lock until the tool call returns, and + the tool call returns only when the server has answered the launch. + ``arun()`` holds the lock on the event-loop thread while it awaits the + tool; ``run()`` holds it on a worker thread. + """ + services: dict[UUID, Any] = {} + with patch( + "openhands.agent_server.conversation_service.EventService", + side_effect=_event_service_factory(services), + ): + async with ConversationService( + conversations_dir=tmp_path / "conversations", + conversation_worktree_root=tmp_path / "conversation-worktrees", + ) as service: + parent_info, _ = await service.start_conversation( + _start_request(workspace_dir) + ) + parent_state = await services[parent_info.id].get_state() + loop = asyncio.get_running_loop() + + if step_runs_on == "event_loop": + with parent_state: + result = await loop.run_in_executor( + None, _launch_child_like_the_tool, service, parent_info.id, loop + ) + else: + + def step() -> StartChildConversationResponse | None: + with parent_state: + return _launch_child_like_the_tool( + service, parent_info.id, loop + ) + + result = await loop.run_in_executor(None, step) + + assert result is not None + assert result.parent_conversation_id == parent_info.id + + @pytest.mark.asyncio async def test_start_child_conversation_shares_workspace_outside_git_repo( tmp_path, workspace_dir diff --git a/tests/cross/test_agent_server_restore_mutations.py b/tests/cross/test_agent_server_restore_mutations.py new file mode 100644 index 00000000000..719068d13a2 --- /dev/null +++ b/tests/cross/test_agent_server_restore_mutations.py @@ -0,0 +1,481 @@ +"""Agent-server restore semantics for state owned by ``base_state.json``. + +``confirmation_policy``, ``security_analyzer`` and ``secrets`` can change after a +conversation starts. Their only persisted home is the conversation's +``base_state.json``; ``meta.json`` does not carry them. These tests drive a real +``ConversationService`` through a restart (close, then a fresh service on the +same directory) and check that the changes survive, also for a fork made after +the restart (#4810). +""" + +from __future__ import annotations + +import json +import sys +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +import pytest +from pydantic import SecretStr + +from openhands.agent_server.conversation_service import ConversationService +from openhands.agent_server.event_service import EventService +from openhands.sdk import LLM, Agent +from openhands.sdk.conversation.impl.local_conversation import LocalConversation +from openhands.sdk.conversation.request import ( + StartChildConversationRequest, + StartConversationRequest, +) +from openhands.sdk.secret import LookupSecret, StaticSecret +from openhands.sdk.security.confirmation_policy import ( + AlwaysConfirm, + ConfirmRisky, + NeverConfirm, +) +from openhands.sdk.security.llm_analyzer import LLMSecurityAnalyzer +from openhands.sdk.utils.cipher import Cipher +from openhands.sdk.workspace import LocalWorkspace + + +pytestmark = pytest.mark.skipif( + sys.platform == "win32", + reason="The agent-server conversation service targets Unix hosts.", +) + + +def _request(workspace_dir) -> StartConversationRequest: + return StartConversationRequest( + agent=Agent(llm=LLM(model="gpt-4o", usage_id="test-llm"), tools=[]), + workspace=LocalWorkspace(working_dir=str(workspace_dir)), + confirmation_policy=NeverConfirm(), + secrets={"START_TOKEN": StaticSecret(value=SecretStr("start-value"))}, + ) + + +@pytest.mark.asyncio +async def test_mutations_survive_a_server_restart_and_a_later_fork(tmp_path): + conversations_dir = tmp_path / "conversations" + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + cipher = Cipher("restore-mutations-key") + + async with ConversationService( + conversations_dir=conversations_dir, cipher=cipher + ) as service: + info, _ = await service.start_conversation(_request(workspace_dir)) + event_service = await service.get_event_service(info.id) + assert event_service is not None + await event_service.set_confirmation_policy(AlwaysConfirm()) + await event_service.set_security_analyzer(LLMSecurityAnalyzer()) + await event_service.update_secrets({"ADDED_TOKEN": "added-value"}) + + meta = json.loads((conversations_dir / info.id.hex / "meta.json").read_text()) + for field in ("confirmation_policy", "security_analyzer", "secrets"): + assert field not in meta + + async with ConversationService( + conversations_dir=conversations_dir, cipher=cipher + ) as service: + restored = await service.get_event_service(info.id) + assert restored is not None + state = await restored.get_state() + assert isinstance(state.confirmation_policy, AlwaysConfirm) + assert isinstance(state.security_analyzer, LLMSecurityAnalyzer) + secrets = state.secret_registry.secret_sources + assert secrets["START_TOKEN"].get_value() == "start-value" + assert secrets["ADDED_TOKEN"].get_value() == "added-value" + + fork_info = await service.fork_conversation(info.id) + assert fork_info is not None + fork_service = await service.get_event_service(fork_info.id) + assert fork_service is not None + fork_state = await fork_service.get_state() + assert isinstance(fork_state.confirmation_policy, AlwaysConfirm) + assert isinstance(fork_state.security_analyzer, LLMSecurityAnalyzer) + fork_secrets = fork_state.secret_registry.secret_sources + assert fork_secrets["START_TOKEN"].get_value() == "start-value" + assert fork_secrets["ADDED_TOKEN"].get_value() == "added-value" + + +@pytest.mark.asyncio +async def test_secrets_an_older_server_left_in_meta_json_are_kept(tmp_path): + """Older servers kept start-time secrets in meta.json and could leave them + out of base_state.json. The first resume takes over the ones base_state.json + lacks; a value it already has wins over the older copy.""" + conversations_dir = tmp_path / "conversations" + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + cipher = Cipher("restore-mutations-key") + + async with ConversationService( + conversations_dir=conversations_dir, cipher=cipher + ) as service: + info, _ = await service.start_conversation(_request(workspace_dir)) + event_service = await service.get_event_service(info.id) + assert event_service is not None + await event_service.update_secrets({"START_TOKEN": "changed-value"}) + + meta_file = conversations_dir / info.id.hex / "meta.json" + meta = json.loads(meta_file.read_text()) + meta["secrets"] = { + name: StaticSecret(value=SecretStr(value)).model_dump( + mode="json", context={"cipher": cipher} + ) + for name, value in { + "START_TOKEN": "start-value", + "ONLY_IN_META": "legacy-value", + }.items() + } + meta_file.write_text(json.dumps(meta)) + + async with ConversationService( + conversations_dir=conversations_dir, cipher=cipher + ) as service: + restored = await service.get_event_service(info.id) + assert restored is not None + secrets = (await restored.get_state()).secret_registry.secret_sources + assert secrets["ONLY_IN_META"].get_value() == "legacy-value" + assert secrets["START_TOKEN"].get_value() == "changed-value" + + assert "secrets" not in json.loads(meta_file.read_text()) + base_state = json.loads((meta_file.parent / "base_state.json").read_text()) + assert "ONLY_IN_META" in base_state["secret_registry"]["secret_sources"] + + +@pytest.mark.asyncio +async def test_fork_keeps_secret_values_on_a_server_without_a_cipher(tmp_path): + """Without a cipher the fork's base_state.json has no secret values, so the + server hands the fork the ones its source holds in memory.""" + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + + async with ConversationService( + conversations_dir=tmp_path / "conversations" + ) as service: + info, _ = await service.start_conversation(_request(workspace_dir)) + event_service = await service.get_event_service(info.id) + assert event_service is not None + await event_service.update_secrets({"ADDED_TOKEN": "added-value"}) + + fork_info = await service.fork_conversation(info.id) + assert fork_info is not None + fork_service = await service.get_event_service(fork_info.id) + assert fork_service is not None + secrets = (await fork_service.get_state()).secret_registry.secret_sources + assert secrets["START_TOKEN"].get_value() == "start-value" + assert secrets["ADDED_TOKEN"].get_value() == "added-value" + + +@pytest.mark.asyncio +async def test_conversations_started_from_one_request_have_their_own_analyzer( + tmp_path, +): + """An analyzer can keep per-conversation history, so two conversations + never share one, also when the same request object starts both.""" + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + request = _request(workspace_dir).model_copy( + update={"security_analyzer": LLMSecurityAnalyzer()} + ) + + async with ConversationService( + conversations_dir=tmp_path / "conversations" + ) as service: + analyzers = [] + for _ in range(2): + info, _ = await service.start_conversation(request) + event_service = await service.get_event_service(info.id) + assert event_service is not None + analyzers.append((await event_service.get_state()).security_analyzer) + + assert all(isinstance(analyzer, LLMSecurityAnalyzer) for analyzer in analyzers) + assert analyzers[0] is not analyzers[1] + assert all(analyzer is not request.security_analyzer for analyzer in analyzers) + + +@pytest.mark.asyncio +async def test_retry_of_a_create_that_failed_halfway_gets_the_requested_settings( + tmp_path, +): + """A create can fail after base_state.json is written. The retry with the + same id is still a create: it applies the request's policy and analyzer.""" + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + request = _request(workspace_dir).model_copy( + update={ + "conversation_id": uuid4(), + "confirmation_policy": AlwaysConfirm(), + "security_analyzer": LLMSecurityAnalyzer(), + } + ) + update_secrets = LocalConversation.update_secrets + calls = 0 + + def fail_the_first_save(self, secrets): + nonlocal calls + calls += 1 + if calls == 1: + raise OSError(28, "No space left on device") + return update_secrets(self, secrets) + + async with ConversationService( + conversations_dir=tmp_path / "conversations" + ) as service: + with patch.object(LocalConversation, "update_secrets", fail_the_first_save): + with pytest.raises(OSError): + await service.start_conversation(request) + info, _ = await service.start_conversation(request) + + event_service = await service.get_event_service(info.id) + assert event_service is not None + state = await event_service.get_state() + assert isinstance(state.confirmation_policy, AlwaysConfirm) + assert isinstance(state.security_analyzer, LLMSecurityAnalyzer) + + +@pytest.mark.asyncio +async def test_child_inherits_what_its_parent_has_now(tmp_path): + """A child started on a real service gets the parent's current policy, + analyzer and secrets, changes made after the parent's start included, in + its own base_state.json and not in its meta.json.""" + conversations_dir = tmp_path / "conversations" + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + + async with ConversationService( + conversations_dir=conversations_dir, + conversation_worktree_root=tmp_path / "worktrees", + cipher=Cipher("restore-mutations-key"), + ) as service: + parent, _ = await service.start_conversation(_request(workspace_dir)) + parent_service = await service.get_event_service(parent.id) + assert parent_service is not None + await parent_service.set_confirmation_policy(AlwaysConfirm()) + await parent_service.set_security_analyzer(LLMSecurityAnalyzer()) + await parent_service.update_secrets({"ADDED_TOKEN": "added-value"}) + + with patch.object(EventService, "run", new=AsyncMock()): + child = await service.start_child_conversation( + parent.id, StartChildConversationRequest(task="Reply with ok.") + ) + assert child is not None + child_service = await service.get_event_service(child.conversation_id) + assert child_service is not None + parent_state = await parent_service.get_state() + child_state = await child_service.get_state() + + assert isinstance(child_state.confirmation_policy, AlwaysConfirm) + assert isinstance(child_state.security_analyzer, LLMSecurityAnalyzer) + assert child_state.security_analyzer is not parent_state.security_analyzer + secrets = child_state.secret_registry.secret_sources + assert secrets["START_TOKEN"].get_value() == "start-value" + assert secrets["ADDED_TOKEN"].get_value() == "added-value" + + child_dir = conversations_dir / child.conversation_id.hex + base_state = json.loads((child_dir / "base_state.json").read_text()) + assert base_state["confirmation_policy"]["kind"] == "AlwaysConfirm" + assert base_state["security_analyzer"]["kind"] == "LLMSecurityAnalyzer" + assert set(base_state["secret_registry"]["secret_sources"]) == { + "START_TOKEN", + "ADDED_TOKEN", + } + meta = json.loads((child_dir / "meta.json").read_text()) + assert not {"confirmation_policy", "security_analyzer", "secrets"} & set(meta) + + +@pytest.mark.asyncio +async def test_unreadable_legacy_meta_secret_costs_only_itself(tmp_path): + """One legacy secret this server cannot parse (a secret kind it does not + know) is left behind, and so is one an older server without a key stored + without its value. The conversation loads, and the other legacy secrets in + the same file are still taken over.""" + conversations_dir = tmp_path / "conversations" + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + cipher = Cipher("restore-mutations-key") + + async with ConversationService( + conversations_dir=conversations_dir, cipher=cipher + ) as service: + info, _ = await service.start_conversation(_request(workspace_dir)) + + meta_file = conversations_dir / info.id.hex / "meta.json" + meta = json.loads(meta_file.read_text()) + meta["secrets"] = { + "GONE": {"kind": "SecretKindFromARemovedPlugin"}, + "REDACTED": StaticSecret(value=SecretStr("lost-value")).model_dump(mode="json"), + "ONLY_IN_META": StaticSecret(value=SecretStr("legacy-value")).model_dump( + mode="json", context={"cipher": cipher} + ), + } + meta_file.write_text(json.dumps(meta)) + + async with ConversationService( + conversations_dir=conversations_dir, cipher=cipher + ) as service: + restored = await service.get_event_service(info.id) + assert restored is not None + secrets = (await restored.get_state()).secret_registry.secret_sources + assert set(secrets) == {"START_TOKEN", "ONLY_IN_META"} + assert secrets["START_TOKEN"].get_value() == "start-value" + assert secrets["ONLY_IN_META"].get_value() == "legacy-value" + + +@pytest.mark.asyncio +async def test_start_request_for_an_existing_id_does_not_reseed_its_state(tmp_path): + """A second start request with the id of an existing conversation returns + that conversation: its policy, analyzer and secrets stay as saved, whether + it is loaded or not, whatever the request carries.""" + conversations_dir = tmp_path / "conversations" + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + cipher = Cipher("restore-mutations-key") + conversation_id = uuid4() + first = _request(workspace_dir).model_copy( + update={ + "conversation_id": conversation_id, + "confirmation_policy": AlwaysConfirm(), + "security_analyzer": LLMSecurityAnalyzer(), + } + ) + again = first.model_copy( + update={ + "confirmation_policy": ConfirmRisky(), + "security_analyzer": None, + "secrets": { + "START_TOKEN": StaticSecret(value=SecretStr("second-value")), + "EXTRA": StaticSecret(value=SecretStr("extra-value")), + }, + } + ) + + async def assert_unchanged(service: ConversationService) -> None: + event_service = await service.get_event_service(conversation_id) + assert event_service is not None + state = await event_service.get_state() + assert isinstance(state.confirmation_policy, AlwaysConfirm) + assert isinstance(state.security_analyzer, LLMSecurityAnalyzer) + secrets = state.secret_registry.secret_sources + assert set(secrets) == {"START_TOKEN"} + assert secrets["START_TOKEN"].get_value() == "start-value" + + async with ConversationService( + conversations_dir=conversations_dir, cipher=cipher + ) as service: + await service.start_conversation(first) + _, started = await service.start_conversation(again) + assert not started + await assert_unchanged(service) + + async with ConversationService( + conversations_dir=conversations_dir, cipher=cipher + ) as service: + _, started = await service.start_conversation(again) + assert not started + await assert_unchanged(service) + + +@pytest.mark.parametrize( + ("legacy_secret", "load_cipher"), + [ + (StaticSecret(value=SecretStr("legacy-value")), Cipher("another-key")), + (LookupSecret(url="https://example.test/legacy"), Cipher("another-key")), + (StaticSecret(value=SecretStr("legacy-value")), None), + ], + ids=[ + "static-secret-another-key", + "lookup-secret-another-key", + "static-secret-no-key", + ], +) +@pytest.mark.asyncio +async def test_legacy_secrets_this_key_cannot_decrypt_do_not_cost_the_agent_its_key( + tmp_path, legacy_secret, load_cipher +): + """A load that cannot decrypt the files (another key, or none) takes + nothing over from an older meta.json, whatever kind of secret it holds, so + it does not save the state: with the right key back, the agent's key and + the saved secrets are too.""" + conversations_dir = tmp_path / "conversations" + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + cipher = Cipher("restore-mutations-key") + request = _request(workspace_dir).model_copy( + update={ + "agent": Agent( + llm=LLM( + model="gpt-4o", api_key=SecretStr("agent-key"), usage_id="test-llm" + ), + tools=[], + ) + } + ) + + async with ConversationService( + conversations_dir=conversations_dir, cipher=cipher + ) as service: + info, _ = await service.start_conversation(request) + + meta_file = conversations_dir / info.id.hex / "meta.json" + meta = json.loads(meta_file.read_text()) + meta["secrets"] = { + "ONLY_IN_META": legacy_secret.model_dump( + mode="json", context={"cipher": cipher} + ) + } + meta_file.write_text(json.dumps(meta)) + base_state_file = meta_file.parent / "base_state.json" + saved_state = base_state_file.read_text() + + async with ConversationService( + conversations_dir=conversations_dir, cipher=load_cipher + ) as service: + assert await service.get_event_service(info.id) is not None + assert base_state_file.read_text() == saved_state + + async with ConversationService( + conversations_dir=conversations_dir, cipher=cipher + ) as service: + restored = await service.get_event_service(info.id) + assert restored is not None + state = await restored.get_state() + api_key = state.agent.llm.api_key + assert isinstance(api_key, SecretStr) + assert api_key.get_secret_value() == "agent-key" + secrets = state.secret_registry.secret_sources + assert secrets["START_TOKEN"].get_value() == "start-value" + + +@pytest.mark.asyncio +async def test_server_without_a_key_does_not_take_legacy_cipher_text_for_a_value( + tmp_path, +): + """Without a key, a value an older server encrypted in meta.json is still + cipher text. It is not taken over as the secret's value, also when + base_state.json holds nothing encrypted.""" + conversations_dir = tmp_path / "conversations" + workspace_dir = tmp_path / "workspace" + workspace_dir.mkdir() + cipher = Cipher("restore-mutations-key") + + async with ConversationService( + conversations_dir=conversations_dir, cipher=cipher + ) as service: + info, _ = await service.start_conversation( + _request(workspace_dir).model_copy(update={"secrets": {}}) + ) + + meta_file = conversations_dir / info.id.hex / "meta.json" + meta = json.loads(meta_file.read_text()) + meta["secrets"] = { + "ONLY_IN_META": StaticSecret(value=SecretStr("legacy-value")).model_dump( + mode="json", context={"cipher": cipher} + ) + } + meta_file.write_text(json.dumps(meta)) + + async with ConversationService(conversations_dir=conversations_dir) as service: + restored = await service.get_event_service(info.id) + assert restored is not None + secrets = (await restored.get_state()).secret_registry.secret_sources + assert "ONLY_IN_META" not in secrets