diff --git a/.agents/skills/verify-agent-server/references/feature-map/F10-fork-navigate-condense-ask.md b/.agents/skills/verify-agent-server/references/feature-map/F10-fork-navigate-condense-ask.md index 5f3e1e8de4c..e2593f52e3b 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F10-fork-navigate-condense-ask.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F10-fork-navigate-condense-ask.md @@ -599,7 +599,9 @@ SIDE_AGENT="{\"llm\": {\"model\": \"openai/qa-stub\", \"api_key\": \"qa-stub-key both the response and persisted metadata. The title is stored separately and adds no tag (#4814). Other stored configuration (secrets, plugins, `parent_conversation_id`, client tools) is copied from the source. The - fork's `last_user_message_id` starts `null`. + fork's `last_user_message_id` is the last user message among the events + it copied, and what a hook decided about a copied action or message + (`blocked_actions`, `blocked_messages`) comes with them. - The fork's `base_state.json` is written with the source's cipher (the server's), so the fork's agent keeps its model keys (`F10.fork-keeps-credentials`) and its conversation secrets; the fork is diff --git a/.agents/skills/verify-sdk/programs/S04/fork_state.py b/.agents/skills/verify-sdk/programs/S04/fork_state.py new file mode 100644 index 00000000000..ffd496ce3d7 --- /dev/null +++ b/.agents/skills/verify-sdk/programs/S04/fork_state.py @@ -0,0 +1,126 @@ +"""Fork a conversation that was built with a hook and carries a confirmation +policy, a security analyzer with its own model key, and secrets, all set after +construction, and report what the fork has: in memory, in its +`base_state.json`, and after it is loaded again from disk. No model is called. + +Usage: fork_state.py +""" + +import json +from pathlib import Path + +from pydantic import SecretStr +from sdk_verify import emit, make_llm, run_paths + +from openhands.sdk import LLM, Agent, LocalConversation +from openhands.sdk.hooks import HookConfig, HookDefinition, HookMatcher +from openhands.sdk.security.confirmation_policy import ConfirmRisky, NeverConfirm +from openhands.sdk.security.risk import SecurityRisk +from openhands.sdk.security.toolshield_llm_analyzer import ( + ToolShieldLLMSecurityAnalyzer, +) +from openhands.sdk.utils.cipher import Cipher + + +paths = run_paths() +BASE = paths["run"] / "fork-state" +WORKSPACE = str(paths["workspace"]) +VALUES = ("start-secret-value-1", "guard-key-value-2", "runtime-secret-value-3") +HOOK_REASON = "blocked by the source hook" +cipher = Cipher("sdk-verify-fork-state-key") + + +def agent() -> Agent: + return Agent(llm=make_llm([{"finish": "unused"}]), tools=[]) + + +def secret_values(conversation) -> dict[str, str | None]: + registry = conversation.state.secret_registry + return {name: registry.get_secret_value(name) for name in registry.secret_sources} + + +def guard_key(conversation) -> str | None: + analyzer = conversation.state.security_analyzer + if not isinstance(analyzer, ToolShieldLLMSecurityAnalyzer): + return None + key = analyzer.llm.api_key + return key.get_secret_value() if isinstance(key, SecretStr) else None + + +source = LocalConversation( + agent=agent(), + workspace=WORKSPACE, + persistence_dir=str(BASE), + cipher=cipher, + secrets={"START_TOKEN": VALUES[0]}, + hook_config=HookConfig( + user_prompt_submit=[ + HookMatcher( + hooks=[HookDefinition(command=f"echo '{HOOK_REASON}' >&2; exit 2")] + ) + ] + ), + visualizer=None, +) +source.set_confirmation_policy( + ConfirmRisky(threshold=SecurityRisk.MEDIUM, confirm_unknown=False) +) +source.set_security_analyzer( + ToolShieldLLMSecurityAnalyzer( + llm=LLM(model="openai/guard", api_key=SecretStr(VALUES[1]), usage_id="guard") + ) +) +source.update_secrets({"RUNTIME_TOKEN": VALUES[2]}) + +fork = source.fork() +fork_dir = Path(fork.state.persistence_dir or "") +saved = json.loads((fork_dir / "base_state.json").read_text()) +saved_text = "".join(p.read_text() for p in fork_dir.rglob("*.json")) +out: dict = { + "ok": True, + "fork_policy": fork.state.confirmation_policy.model_dump(mode="json"), + "fork_analyzer": type(fork.state.security_analyzer).__name__, + "fork_analyzer_is_own_object": fork.state.security_analyzer + is not source.state.security_analyzer, + "fork_guard_key_kept": guard_key(fork) == VALUES[1], + "fork_secrets": secret_values(fork), + "saved_policy": saved.get("confirmation_policy"), + "saved_analyzer": (saved.get("security_analyzer") or {}).get("kind"), + "saved_secret_names": sorted(saved["secret_registry"]["secret_sources"]), + "saved_guard_key_is_placeholder": set( + ((saved.get("security_analyzer") or {}).get("llm") or {}).get("api_key") or "*" + ) + == {"*"}, + "plaintext_on_disk": any(value in saved_text for value in VALUES), +} + +# A prompt sent to the fork goes through the hook the source was built with. +fork.send_message("hello") +out["fork_prompt_block"] = next(iter(fork.state.blocked_messages.values()), None) + +# Changing the fork does not reach back into the source. +fork.set_confirmation_policy(NeverConfirm()) +fork.update_secrets({"FORK_ONLY": "fork-only-value-4"}) +out["source_policy_after_fork_change"] = type(source.state.confirmation_policy).__name__ +out["source_has_fork_only_secret"] = ( + "FORK_ONLY" in source.state.secret_registry.secret_sources +) + +fork_id = fork.id +fork.close() +source.close() +reloaded = LocalConversation( + agent=agent(), + workspace=WORKSPACE, + persistence_dir=str(BASE), + conversation_id=fork_id, + cipher=cipher, + visualizer=None, +) +out |= { + "reloaded_policy": type(reloaded.state.confirmation_policy).__name__, + "reloaded_guard_key_kept": guard_key(reloaded) == VALUES[1], + "reloaded_secrets": secret_values(reloaded), +} +reloaded.close() +emit(out) diff --git a/.agents/skills/verify-sdk/references/feature-map/README.md b/.agents/skills/verify-sdk/references/feature-map/README.md index 0dd0b1a8c75..332328e108f 100644 --- a/.agents/skills/verify-sdk/references/feature-map/README.md +++ b/.agents/skills/verify-sdk/references/feature-map/README.md @@ -12,7 +12,7 @@ that map's `Fnn`. Maintenance baseline: main@141423921fdaeb08cd5f53464a69e860da70b66f (2026-11-19). The next maintenance pass starts from this commit, a pass proposes the next baseline in its PR, and merging that PR accepts it. -The map has **504 sub-features**, **36 of them untested** (no unit or cross +The map has **505 sub-features**, **36 of them untested** (no unit or cross test exercises the behavior; most of those have an example script). ## What each sub-feature links @@ -78,7 +78,7 @@ an interactive login are labeled **blocked**. | S01 | [Agent step loop](S01-agent-loop.md) | step handling (tool calls, text, empty replies), iteration and budget limits, stuck detector, parallel tools, messages sent mid-run, image input, `ask_agent`, structured output, recovery from bad tool calls and context overflow | `Conversation(max_iteration_per_run=, stuck_detection=, max_budget_per_run=)`, `Agent(tool_concurrency_limit=)`, `ask_agent()`, `response_schema` | llm, tmux, network | 24 | 1 | | S02 | [LLM core](S02-llm-core.md) | completion, responses and `generate`; streaming; reasoning and thinking blocks; native and prompted tool calling; metrics, stats and pricing; retries and error mapping; caching, token counts, image inlining, completion logs | `LLM(...)`, `llm.completion`/`responses`, `Conversation(token_callbacks=)`, `llm.metrics` | llm, network | 28 | 0 | | S03 | [LLM registry, routing, fallback, profiles and model switching](S03-llm-routing-profiles.md) | `LLMRegistry`, multimodal and random routers, `FallbackStrategy`, `LLMProfileStore`, `switch_profile`/`switch_llm`, the switch-LLM tool, `route_task_to_model`, subscription login | `LLMRegistry`, `MultimodalRouter`, `FallbackStrategy`, `LLMProfileStore`, `LLM.subscription_login()` | llm, network | 30 | 4 | -| S04 | [Local conversation lifecycle](S04-local-conversation.md) | factory and ids, persistence and resume, callbacks, close, pause, `arun` and interrupt, secrets, visualizers, fork and branches, goal loop (and its stop on a pending `ask_user` question, `GoalAsked`), final response, stats, title and tags | `Conversation(persistence_dir=, callbacks=, visualizer=, secrets=)`, `run`/`pause`/`arun`/`interrupt`/`fork`/`run_goal` | llm, tmux, network, uvx | 34 | 4 | +| S04 | [Local conversation lifecycle](S04-local-conversation.md) | factory and ids, persistence and resume, callbacks, close, pause, `arun` and interrupt, secrets, visualizers, fork and branches, goal loop (and its stop on a pending `ask_user` question, `GoalAsked`), final response, stats, title and tags | `Conversation(persistence_dir=, callbacks=, visualizer=, secrets=)`, `run`/`pause`/`arun`/`interrupt`/`fork`/`run_goal` | llm, tmux, network, uvx | 35 | 4 | | S05 | [Events and the event log](S05-events.md) | event model, JSON round trip, conversion to LLM messages and OpenAI formats, unmatched actions, the `EventLog`, error classification, completion and token events | `Event.model_validate_json`, `events_to_messages`, `conversation.state.events` | llm, tmux | 18 | 1 | | S06 | [Agent context, system prompt, condensers and memory](S06-context-condensers.md) | `AgentContext`, system prompt sections and vendor files, persona, persistent memory, the condenser `View`, `LLMSummarizingCondenser`, `condense()`, overflow condensation, no-op and pipeline condensers | `Agent(agent_context=, system_prompt=, condenser=)`, `AgentContext(load_memory=)`, `conversation.condense()` | llm, tmux | 24 | 1 | | S07 | [Skills](S07-skills.md) | skill sources (programmatic, Markdown, AGENTS.md, SKILL.md, user, project, public, installed), keyword, task and path triggers, ``, `invoke_skill`, inline commands, disabling, install and marketplace, the `OH_PUBLIC_SKILLS_ENABLED` switch | `AgentContext(skills=, load_*_skills=, disabled_skills=)`, `.agents/skills/`, `install_skill`, `OH_PUBLIC_SKILLS_ENABLED` | llm, git, network | 25 | 0 | diff --git a/.agents/skills/verify-sdk/references/feature-map/S04-local-conversation.md b/.agents/skills/verify-sdk/references/feature-map/S04-local-conversation.md index 6aac84668bc..2ff200c4dca 100644 --- a/.agents/skills/verify-sdk/references/feature-map/S04-local-conversation.md +++ b/.agents/skills/verify-sdk/references/feature-map/S04-local-conversation.md @@ -34,6 +34,7 @@ Examples: `examples/01_standalone_sdk/06_interactive_terminal_w_reasoning.py`, ` - `S04.fork`: `fork()` deep-copies the conversation under a new id (or `conversation_id=`), starts IDLE, leaves the source untouched, resets metrics unless `reset_metrics=False`, accepts `agent=` and `tags=` (`title=` is deprecated and ignored locally; only `RemoteConversation` stores it), shares the workspace, and persists so the fork survives a reload. Unit: `tests/sdk/conversation/local/test_fork.py`. Example: `examples/01_standalone_sdk/48_conversation_fork.py`. Real-LLM: none. - `S04.fork-keeps-secrets`: `fork()` of a `LocalConversation(..., cipher=...)` with a `persistence_dir` creates the fork with the source's cipher and copies the source's conversation secrets (`secrets=`, `update_secrets`), so the fork's `base_state.json` holds the LLM api key and every secret encrypted (not redacted); `LocalConversation(agent=None, conversation_id=fork_id, cipher=...)` reopens the fork with the real key and secrets, and it runs. Without a cipher the fork's key stays redacted on disk. Since [#5765](https://github.com/OpenHands/software-agent-sdk/pull/5765) ([#5465](https://github.com/OpenHands/software-agent-sdk/issues/5465)). Unit: `tests/sdk/conversation/local/test_fork.py::test_fork_keeps_source_cipher_for_persisted_secrets`, `::test_fork_without_cipher_still_redacts_secrets`, `tests/agent_server/test_conversation_service.py::TestConversationTreeForkAndNavigate::test_cipher_backed_fork_resumes_with_credentials`. Example: none. Real-LLM: none. - `S04.fork-branch`: events carry parent ids forming a tree; `fork(from_event_id=...)` copies only that branch, and `navigate_to(event_id)` moves HEAD so the next event starts a sibling branch (`None` empties the active branch). Unit: `tests/sdk/conversation/local/test_conversation_tree.py`. Example: `examples/01_standalone_sdk/48_conversation_fork.py`. Real-LLM: none. +- `S04.fork-keeps-state`: a fork carries the source's confirmation policy, its own copy of the security analyzer (credentials kept) and the conversation secrets, and is written with the source's cipher, so its `base_state.json` alone restores them; it also runs the hooks in the `hook_config` the source was built with. Unit: `tests/sdk/conversation/local/test_fork.py`, `tests/sdk/conversation/local/test_fork_hook_blocks.py`. Example: none. Real-LLM: none. - `S04.goal-loop`: `run_goal(conversation, objective, judge_llm, max_iterations=N)` sends the objective, runs, asks the judge LLM for a verdict, re-prompts with the missing items, and returns a `GoalOutcome` with status `"complete"` or `"capped"`, the iteration count and the verdict. Unit: `tests/sdk/conversation/goal/`. Example: `examples/01_standalone_sdk/54_goal_completion_loop.py`. Real-LLM: none. - `S04.goal-asked`: when a round ends on the agent's `ask_user` question, `run_goal` returns `GoalAsked(ask_id, question)` for it without calling the judge, and the objective it sent is a user message with `sender="goal"`, which does not answer the question; called again while the question is pending it returns the same `GoalAsked` at once, sending nothing and calling neither model; once the user has answered (`conversation.answer_ask(...)` plus the returned message) it runs and the judge can say complete. Unit: `tests/tools/ask_user/test_ask_user.py::test_a_goal_stops_on_a_question_without_judging`. Example: none. Real-LLM: none. - `S04.goal-asked-branch`: only a question on the active branch stops a goal: after `navigate_to(...)` back to the user message before an unanswered question, `run_goal` with a new objective sends it (`sender="goal"`), runs the agent and asks the judge. Unit: **none (untested)**. Example: none. Real-LLM: none. @@ -277,6 +278,22 @@ Scripted bullets use the SDK's `TestLLM` (`--script`, or `make_llm(script=...)` --check old_branch_in_active eq false --check active_ends_with eq "answer on a new branch" \ --check unknown_event_error contains "Unknown event_id" --check active_after_none eq 0 --check log_after_none eq 7 ``` +- **What a fork keeps (`S04.fork-keeps-state`).** A ciphered conversation built with a `UserPromptSubmit` hook that blocks every prompt gets `ConfirmRisky(MEDIUM)`, a `ToolShieldLLMSecurityAnalyzer` with its own model key and a runtime secret after construction, then is forked; the fork is sent a message, changed and loaded again from disk. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S04/fork_state \ + --check fork_policy.kind eq ConfirmRisky --check fork_policy.threshold eq MEDIUM --check fork_policy.confirm_unknown eq false \ + --check fork_analyzer eq ToolShieldLLMSecurityAnalyzer --check fork_analyzer_is_own_object eq true --check fork_guard_key_kept eq true \ + --check fork_secrets.START_TOKEN eq start-secret-value-1 --check fork_secrets.RUNTIME_TOKEN eq runtime-secret-value-3 \ + --check saved_policy.kind eq ConfirmRisky --check saved_analyzer eq ToolShieldLLMSecurityAnalyzer \ + --check saved_secret_names.0 eq RUNTIME_TOKEN --check saved_secret_names.1 eq START_TOKEN \ + --check saved_guard_key_is_placeholder eq false --check plaintext_on_disk eq false \ + --check fork_prompt_block eq "blocked by the source hook" \ + --check source_policy_after_fork_change eq ConfirmRisky --check source_has_fork_only_secret eq false \ + --check reloaded_policy eq NeverConfirm --check reloaded_guard_key_kept eq true \ + --check reloaded_secrets.START_TOKEN eq start-secret-value-1 --check reloaded_secrets.FORK_ONLY eq fork-only-value-4 + ``` + The fork has the policy with its threshold, an analyzer object of its own whose key still works, and both secrets; its `base_state.json` holds them encrypted (neither the plaintext nor the `**********` placeholder). The message sent to the fork is blocked with the source hook's reason. Setting `NeverConfirm` and a new secret on the fork leaves the source as it was, and the fork loaded again from disk has the fork's values. - **The goal loop (`S04.goal-loop`).** `run_goal` with a scripted judge: incomplete then complete (the follow-up names what was missing), and a capped run whose last verdict cannot be parsed. ```sh export SDK_VERIFY_RUN=$(control-sdk new --field run) diff --git a/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py b/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py index 7d79561d611..ada4121c91b 100644 --- a/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py +++ b/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py @@ -118,7 +118,7 @@ load_available_plugins, ) from openhands.sdk.plugin.strict_fetch import expand_plugin_source -from openhands.sdk.secret import StaticSecret +from openhands.sdk.secret import SecretSource, StaticSecret from openhands.sdk.security.analyzer import SecurityAnalyzerBase from openhands.sdk.security.confirmation_policy import ( ConfirmationPolicyBase, @@ -143,7 +143,7 @@ from openhands.sdk.tool.client_tool import ClientToolSpec from openhands.sdk.tool.schema import Action, Observation from openhands.sdk.utils.cipher import FERNET_TOKEN_PREFIX, Cipher -from openhands.sdk.utils.pydantic_secrets import client_context +from openhands.sdk.utils.pydantic_secrets import client_context, copy_keeping_secrets from openhands.sdk.utils.redact import ( redact_text_secrets, redact_url_credentials, @@ -220,6 +220,19 @@ def _copy_event_for_fork(event: Event, source: EventLog) -> Event: return copied.model_copy(update={"parent_id": bridged}) +def _copy_secret_source_for_fork(source: SecretSource) -> SecretSource: + """A copy of a secret source that the fork owns. + + A deep copy keeps what a custom source holds outside its fields, such as a + value in a private attribute. A source with a member that cannot be copied + (a lock, an open client) is rebuilt from its fields instead. + """ + try: + return source.model_copy(deep=True) + except Exception: + return copy_keeping_secrets(source) + + def _hooks_not_in(hooks: HookConfig, existing: HookConfig) -> HookConfig: """Return the matchers of ``hooks`` that ``existing`` does not already hold.""" @@ -284,6 +297,9 @@ class LocalConversation(BaseConversation): _resolved_plugins: list[ResolvedPluginSource] | None _plugins_loaded: bool _pending_hook_config: HookConfig | None # Hook config to combine with plugin hooks + # The caller's hook_config, never merged with plugin hooks, so a fork can + # be given it without receiving the plugin hooks twice. + _explicit_hook_config: HookConfig | None _mcp_tool_provider: MCPToolProvider _mcp_startup_budget: float | None _late_mcp_lock: threading.Lock @@ -422,6 +438,7 @@ def __init__( self._resolved_plugins = None self._plugins_loaded = False self._pending_hook_config = hook_config # Will be combined with plugin hooks + self._explicit_hook_config = hook_config self._agent_ready = False # Agent initialized lazily after plugins loaded self._mcp_tool_provider = mcp_tool_provider or DefaultMCPToolProvider() self._mcp_startup_budget = mcp_startup_budget @@ -992,6 +1009,9 @@ def fork( Events are copied so the source remains immutable. The fork starts in ``execution_status='idle'``; calling ``run()`` resumes from the copied state — meaning the agent has full event memory of the source. + It also keeps the source's confirmation policy, the ``hook_config`` the + source was created with, its own copy of the security analyzer and of + the secrets, and the source's cipher. Args: conversation_id: ID for the forked conversation (auto-generated @@ -1018,17 +1038,12 @@ def fork( """ del title # no local title storage; see docstring fork_id = conversation_id or uuid.uuid4() - # Always deep-copy the agent (supplied or source) so the fork owns + # Always copy the agent (supplied or source) so the fork owns # its own object graph. Required because __init__ binds # per-conversation call context on the LLM (#2917, #3443): a # shared/aliased agent would clobber the source conversation's - # context. Round-trip via JSON avoids thread-lock pickling issues - # with model_copy(deep=True). - source_agent = agent if agent is not None else self.agent - agent_cls = type(source_agent) - fork_agent = agent_cls.model_validate( - source_agent.model_dump(context=client_context("plaintext")), - ) + # context. + fork_agent = copy_keeping_secrets(agent if agent is not None else self.agent) # Hold the state lock while reading mutable state from the source # conversation to avoid torn reads if run() is executing concurrently. @@ -1037,6 +1052,17 @@ def fork( # does not leave an orphaned persistence directory behind. if from_event_id is not None and from_event_id not in self._state.events: raise ValueError(f"Unknown from_event_id: {from_event_id}") + # Copied before the fork exists too, for the same reason. + source_analyzer = self._state.security_analyzer + fork_analyzer = ( + copy_keeping_secrets(source_analyzer) + if source_analyzer is not None + else None + ) + fork_secrets = { + name: _copy_secret_source_for_fork(source) + for name, source in self._state.secret_registry.secret_sources.items() + } # Determine persistence_dir for the fork. # Pass the *base* directory only — __init__ calls @@ -1053,6 +1079,7 @@ def fork( agent=fork_agent, workspace=self.workspace, plugins=self._plugin_specs, + hook_config=self._explicit_hook_config, persistence_dir=fork_persistence, conversation_id=fork_id, max_iteration_per_run=self.max_iteration_per_run, @@ -1110,17 +1137,40 @@ def fork( self._state.activated_path_rules ) fork_conv._state.agent_state = copy.deepcopy(self._state.agent_state) + fork_conv._state.invoked_skills = list(self._state.invoked_skills) + # A hook's verdict on a copied action or message still holds in the + # fork: without it the fork would run an action a hook denied. + copied_ids = {event.id for event in source_events} + fork_conv._state.blocked_actions = { + event_id: reason + for event_id, reason in self._state.blocked_actions.items() + if event_id in copied_ids + } + fork_conv._state.blocked_messages = { + event_id: reason + for event_id, reason in self._state.blocked_messages.items() + if event_id in copied_ids + } + last_user_message_id = self._state.last_user_message_id + if last_user_message_id not in copied_ids: + last_user_message_id = next( + ( + event.id + for event in reversed(source_events) + if isinstance(event, MessageEvent) and event.source == "user" + ), + None, + ) + fork_conv._state.last_user_message_id = last_user_message_id + fork_conv._state.confirmation_policy = self._state.confirmation_policy + fork_conv._state.security_analyzer = fork_analyzer # Conversation secrets (``secrets=`` / update_secrets) live in the # registry, not the agent. Assigning the field (rather than # mutating the registry) autosaves them, encrypted with the cipher. - source_secrets = self._state.secret_registry.secret_sources fork_conv._state.secret_registry = SecretRegistry( secret_sources={ **fork_conv._state.secret_registry.secret_sources, - **{ - name: source.model_copy(deep=True) - for name, source in source_secrets.items() - }, + **fork_secrets, } ) diff --git a/openhands-sdk/openhands/sdk/security/grayswan/analyzer.py b/openhands-sdk/openhands/sdk/security/grayswan/analyzer.py index 13d39536744..9bd606c3bb1 100644 --- a/openhands-sdk/openhands/sdk/security/grayswan/analyzer.py +++ b/openhands-sdk/openhands/sdk/security/grayswan/analyzer.py @@ -10,7 +10,7 @@ import json import os from collections.abc import Sequence -from typing import Any +from typing import Annotated, Any import httpx from pydantic import Field, PrivateAttr, SecretStr, model_validator @@ -20,6 +20,7 @@ from openhands.sdk.security.analyzer import SecurityAnalyzerBase from openhands.sdk.security.grayswan.utils import convert_events_to_openai_messages from openhands.sdk.security.risk import SecurityRisk +from openhands.sdk.utils.pydantic_secrets import SecretField logger = get_logger(__name__) @@ -66,7 +67,7 @@ class GraySwanAnalyzer(SecurityAnalyzerBase): default="https://api.grayswan.ai/cygnal/monitor", description="GraySwan Cygnal API endpoint", ) - api_key: SecretStr | None = Field( + api_key: Annotated[SecretStr | None, SecretField()] = Field( default=None, description="GraySwan API key (via GRAYSWAN_API_KEY env var)", ) diff --git a/openhands-sdk/openhands/sdk/utils/pydantic_secrets.py b/openhands-sdk/openhands/sdk/utils/pydantic_secrets.py index 65d48852991..9c1209e3677 100644 --- a/openhands-sdk/openhands/sdk/utils/pydantic_secrets.py +++ b/openhands-sdk/openhands/sdk/utils/pydantic_secrets.py @@ -466,3 +466,15 @@ def decrypt_str_with_cipher_or_keep( ) return value return decrypted + + +def copy_keeping_secrets[ModelT: BaseModel](model: ModelT) -> ModelT: + """A copy that shares nothing with ``model`` and keeps its credentials. + + A dump/validate round trip rather than ``model_copy(deep=True)``: a deep + copy carries private runtime state along (an analyzer's history) and fails + on members that cannot be copied (a lock, an open client). + """ + return type(model).model_validate( + model.model_dump(context=client_context("plaintext")) + ) diff --git a/tests/sdk/conversation/local/test_fork.py b/tests/sdk/conversation/local/test_fork.py index ac233de692a..d74e346de2a 100644 --- a/tests/sdk/conversation/local/test_fork.py +++ b/tests/sdk/conversation/local/test_fork.py @@ -9,9 +9,10 @@ from typing import Self import pytest -from pydantic import SecretStr +from pydantic import Field, PrivateAttr, SecretStr from openhands.sdk.agent import Agent +from openhands.sdk.context import AgentContext from openhands.sdk.conversation import Conversation, LocalConversation from openhands.sdk.conversation.state import ( ConversationExecutionStatus, @@ -19,6 +20,12 @@ ) from openhands.sdk.event.llm_convertible import MessageEvent, SystemPromptEvent from openhands.sdk.llm import LLM, Message, TextContent +from openhands.sdk.secret import SecretSource +from openhands.sdk.security.confirmation_policy import ConfirmRisky +from openhands.sdk.security.risk import SecurityRisk +from openhands.sdk.security.toolshield_llm_analyzer import ( + ToolShieldLLMSecurityAnalyzer, +) from openhands.sdk.tool import Action, Observation, ToolDefinition, ToolExecutor from openhands.sdk.utils.cipher import Cipher @@ -38,6 +45,34 @@ def _msg(event_id: str, text: str = "hi") -> MessageEvent: ) +class _LockedSecretSource(SecretSource): + """A secret source that holds something a deep copy cannot copy.""" + + _lock: threading.Lock = PrivateAttr(default_factory=threading.Lock) + + def get_value(self) -> str: + with self._lock: + return "locked-source-value" + + +class _PrivateValueSecretSource(SecretSource): + """A secret source that keeps its value outside its fields.""" + + _value: str | None = PrivateAttr(default=None) + + def get_value(self) -> str | None: + return self._value + + +class _ExcludedFieldSecretSource(SecretSource): + """A secret source that keeps its value in a field a dump leaves out.""" + + token: str = Field(default="", exclude=True) + + def get_value(self) -> str | None: + return self.token or None + + class ForkCopyMockAction(Action): command: str = "test" @@ -422,6 +457,247 @@ def test_fork_without_cipher_still_redacts_secrets(tmp_path): assert set(_persisted_base_state(fork)["agent"]["llm"]["api_key"]) == {"*"} +def test_fork_keeps_confirmation_policy(tmp_path): + """A fork asks for confirmation under the same policy as its source. + + The fork's base_state.json must carry it too: the agent-server resumes a + fork from that file. + """ + policy = ConfirmRisky(threshold=SecurityRisk.MEDIUM, confirm_unknown=False) + src = LocalConversation( + agent=_agent(), + workspace=str(tmp_path / "workspace"), + persistence_dir=str(tmp_path / "conversations"), + ) + src.set_confirmation_policy(policy) + + fork = src.fork() + + assert fork.state.confirmation_policy == policy + assert _persisted_base_state(fork)["confirmation_policy"] == { + "kind": "ConfirmRisky", + "threshold": "MEDIUM", + "confirm_unknown": False, + } + + +def test_fork_copies_secret_registry(): + """Registry-direct secrets must survive a fork. + + base_state.json is the fork's only source of truth for secrets, so fork() + must copy the source's secret_registry into the fork's state; otherwise a + token injected via ``Conversation(secrets=...)`` is empty on the fork. + """ + with tempfile.TemporaryDirectory() as tmpdir: + src = Conversation( + agent=_agent(), + persistence_dir=tmpdir, + workspace=tmpdir, + secrets={"MY_TOKEN": "s3kr3t-value"}, + ) + + fork = src.fork() + + assert fork.state.secret_registry.get_secret_value("MY_TOKEN") == ( + "s3kr3t-value" + ) + + +def test_fork_secret_change_does_not_reach_the_source_masking(): + """The fork resolves and masks with a registry of its own: replacing a + secret on the fork leaves the source masking its own value.""" + with tempfile.TemporaryDirectory() as tmpdir: + src = Conversation( + agent=_agent(), + persistence_dir=tmpdir, + workspace=tmpdir, + secrets={"TOKEN": "source-token-value"}, + ) + src_registry = src.state.secret_registry + assert src_registry.get_secrets_as_env_vars("echo $TOKEN") == { + "TOKEN": "source-token-value" + } + + fork = src.fork() + fork.update_secrets({"TOKEN": "fork-token-value"}) + assert fork.state.secret_registry.get_secrets_as_env_vars("echo $TOKEN") == { + "TOKEN": "fork-token-value" + } + + assert ( + src.state.secret_registry.mask_secrets_in_output("out: source-token-value") + == "out: " + ) + + +def test_fork_keeps_a_replacement_agents_own_secrets(): + """`fork(agent=...)` with an agent that brings secrets in its context has + them next to the ones copied from the source.""" + with tempfile.TemporaryDirectory() as tmpdir: + src = Conversation( + agent=_agent(), + persistence_dir=tmpdir, + workspace=tmpdir, + secrets={"SOURCE_TOKEN": "source-value"}, + ) + replacement = Agent( + llm=LLM(model="gpt-4o-mini", api_key=SecretStr("k"), usage_id="other"), + tools=[], + agent_context=AgentContext(secrets={"AGENT_TOKEN": "agent-value"}), + ) + + fork = src.fork(agent=replacement) + + registry = fork.state.secret_registry + assert registry.get_secret_value("AGENT_TOKEN") == "agent-value" + assert registry.get_secret_value("SOURCE_TOKEN") == "source-value" + + +def test_fork_gives_independent_security_analyzer(): + """The fork's analyzer is a copy of its own, down to the guardrail LLM: a + shared analyzer, or a shallow copy of one, would mix the two conversations + in a stateful analyzer such as ToolShieldLLMSecurityAnalyzer.""" + with tempfile.TemporaryDirectory() as tmpdir: + src = Conversation(agent=_agent(), persistence_dir=tmpdir, workspace=tmpdir) + src.set_security_analyzer( + ToolShieldLLMSecurityAnalyzer( + llm=LLM(model="gpt-4o-mini", usage_id="guard"), history_window=7 + ) + ) + + fork = src.fork() + + source_analyzer = src.state.security_analyzer + fork_analyzer = fork.state.security_analyzer + assert isinstance(source_analyzer, ToolShieldLLMSecurityAnalyzer) + assert isinstance(fork_analyzer, ToolShieldLLMSecurityAnalyzer) + assert fork_analyzer is not source_analyzer + assert fork_analyzer.llm is not source_analyzer.llm + assert fork_analyzer.history_window == 7 + assert fork_analyzer.llm.model == "gpt-4o-mini" + + +def test_fork_copies_a_secret_source_that_cannot_be_deep_copied(): + """A custom secret source may hold a lock or a client; the fork still gets + a working copy of it.""" + with tempfile.TemporaryDirectory() as tmpdir: + src = Conversation( + agent=_agent(), + persistence_dir=tmpdir, + workspace=tmpdir, + secrets={"LOCKED": _LockedSecretSource()}, + ) + + fork = src.fork() + + assert fork.state.secret_registry.get_secret_value("LOCKED") == ( + "locked-source-value" + ) + fork_source = fork.state.secret_registry.secret_sources["LOCKED"] + assert fork_source is not src.state.secret_registry.secret_sources["LOCKED"] + + +def _private_value_source() -> SecretSource: + source = _PrivateValueSecretSource() + source._value = "held-outside-the-fields" + return source + + +@pytest.mark.parametrize( + "make_source", + [ + _private_value_source, + lambda: _ExcludedFieldSecretSource(token="held-outside-the-fields"), + ], + ids=["private-attribute", "excluded-field"], +) +def test_fork_keeps_what_a_secret_source_holds_outside_its_fields(make_source): + """A custom secret source may keep its value in a private attribute or in + a field a dump leaves out; the fork's copy still resolves.""" + with tempfile.TemporaryDirectory() as tmpdir: + src = Conversation( + agent=_agent(), workspace=tmpdir, secrets={"HELD": make_source()} + ) + + fork = src.fork() + + assert fork.state.secret_registry.get_secret_value("HELD") == ( + "held-outside-the-fields" + ) + fork_source = fork.state.secret_registry.secret_sources["HELD"] + assert fork_source is not src.state.secret_registry.secret_sources["HELD"] + + +def test_fork_keeps_such_a_secret_source_from_the_agents_context(): + """The agent goes through a dump, which drops what such a source holds; + the fork's registry still has the working copy.""" + with tempfile.TemporaryDirectory() as tmpdir: + agent = _agent().model_copy( + update={ + "agent_context": AgentContext(secrets={"CTX": _private_value_source()}) + } + ) + src = Conversation(agent=agent, workspace=tmpdir) + + fork = src.fork() + + assert fork.state.secret_registry.get_secret_value("CTX") == ( + "held-outside-the-fields" + ) + + +def test_fork_copies_invoked_skills(): + with tempfile.TemporaryDirectory() as tmpdir: + src = Conversation(agent=_agent(), persistence_dir=tmpdir, workspace=tmpdir) + src._state.invoked_skills = ["release-notes"] + + fork = src.fork() + + assert fork.state.invoked_skills == ["release-notes"] + + +def test_fork_preserves_ciphered_security_analyzer_credentials(): + """A fork persisted with the parent's cipher must keep the analyzer's + nested guardrail-LLM api_key instead of redacting it to base_state.json. + """ + with tempfile.TemporaryDirectory() as tmpdir: + cipher = Cipher("stable-conversation-key") + src = LocalConversation( + agent=_agent(), + persistence_dir=tmpdir, + workspace=tmpdir, + cipher=cipher, + ) + src.set_security_analyzer( + ToolShieldLLMSecurityAnalyzer( + llm=LLM( + model="gpt-4o-mini", + api_key=SecretStr("sk-GUARD-xyz"), + usage_id="guard", + ) + ) + ) + + fork = src.fork() + + # Resume the fork from its base_state.json with the cipher, the same way + # the agent-server fork path reloads it. + assert fork.state.persistence_dir is not None + base = str(Path(fork.state.persistence_dir).parent) + resumed = LocalConversation( + agent=_agent(), + workspace=tmpdir, + persistence_dir=base, + conversation_id=fork.id, + cipher=cipher, + ) + + resumed_analyzer = resumed.state.security_analyzer + assert isinstance(resumed_analyzer, ToolShieldLLMSecurityAnalyzer) + assert isinstance(resumed_analyzer.llm.api_key, SecretStr) + assert resumed_analyzer.llm.api_key.get_secret_value() == "sk-GUARD-xyz" + + def test_fork_preserves_max_budget_per_run(): with tempfile.TemporaryDirectory() as tmpdir: src = Conversation( diff --git a/tests/sdk/conversation/local/test_fork_hook_blocks.py b/tests/sdk/conversation/local/test_fork_hook_blocks.py new file mode 100644 index 00000000000..760cc464e0f --- /dev/null +++ b/tests/sdk/conversation/local/test_fork_hook_blocks.py @@ -0,0 +1,286 @@ +"""A fork keeps the source's hooks and what they decided about the events it copies.""" + +import json +from collections.abc import Sequence +from pathlib import Path +from typing import ClassVar, Self +from unittest.mock import patch + +from litellm import ChatCompletionMessageToolCall +from litellm.types.utils import ( + Choices, + Function, + Message as LiteLLMMessage, + ModelResponse, +) +from pydantic import SecretStr + +from openhands.sdk.agent import Agent +from openhands.sdk.context import AgentContext +from openhands.sdk.conversation import LocalConversation +from openhands.sdk.event.llm_convertible import UserRejectObservation +from openhands.sdk.hooks.config import HookConfig +from openhands.sdk.llm import LLM +from openhands.sdk.marketplace import MarketplaceRegistration +from openhands.sdk.plugin import PluginSource +from openhands.sdk.security.confirmation_policy import AlwaysConfirm +from openhands.sdk.tool import ( + Action, + Observation, + Tool, + ToolDefinition, + ToolExecutor, + register_tool, +) + + +class _ForkHookAction(Action): + command: str + + +class _ForkHookObservation(Observation): + pass + + +class _RecordingExecutor(ToolExecutor[_ForkHookAction, _ForkHookObservation]): + executed: ClassVar[list[str]] = [] + + def __call__( + self, + action: _ForkHookAction, + conversation: LocalConversation | None = None, + ) -> _ForkHookObservation: + self.executed.append(action.command) + return _ForkHookObservation.from_text(action.command) + + +class _ForkHookTool(ToolDefinition[_ForkHookAction, _ForkHookObservation]): + name: ClassVar[str] = "fork_hook_tool" + + @classmethod + def create(cls, *args, **kwargs) -> Sequence[Self]: + return [ + cls( + description="records the commands it is given", + action_type=_ForkHookAction, + observation_type=_ForkHookObservation, + executor=_RecordingExecutor(), + ) + ] + + +register_tool(_ForkHookTool.name, _ForkHookTool) + + +def _deny(hook_event: str, reason: str) -> HookConfig: + return HookConfig.from_dict( + { + "hooks": { + hook_event: [ + { + "matcher": "*", + "hooks": [ + { + "type": "command", + "command": f"echo '{reason}' >&2; exit 2", + } + ], + } + ] + } + } + ) + + +def _response(*, tool_command: str | None = None) -> ModelResponse: + tool_calls = None + if tool_command is not None: + tool_calls = [ + ChatCompletionMessageToolCall( + id="call_1", + type="function", + function=Function( + name=_ForkHookTool.name, + arguments=f'{{"command": "{tool_command}"}}', + ), + ) + ] + return ModelResponse( + id="response", + choices=[ + Choices( + message=LiteLLMMessage( + role="assistant", content="ok", tool_calls=tool_calls + ) + ) + ], + created=0, + model="test-model", + object="chat.completion", + ) + + +def _plugin_with_hook(plugin_dir: Path, matcher: str) -> PluginSource: + (plugin_dir / ".plugin").mkdir(parents=True) + (plugin_dir / ".plugin" / "plugin.json").write_text( + json.dumps({"name": plugin_dir.name}) + ) + (plugin_dir / "hooks").mkdir() + hooks = {"PreToolUse": [{"matcher": matcher, "hooks": [{"command": "exit 0"}]}]} + (plugin_dir / "hooks" / "hooks.json").write_text(json.dumps({"hooks": hooks})) + return PluginSource(source=str(plugin_dir)) + + +def _marketplace_with_hook_plugin(marketplace_dir: Path, matcher: str) -> AgentContext: + """An agent context whose marketplace offers one plugin, named "runtime".""" + _plugin_with_hook(marketplace_dir / "plugins" / "runtime", matcher) + (marketplace_dir / ".plugin").mkdir() + manifest = { + "name": "market", + "owner": {"name": "test"}, + "plugins": [{"name": "runtime", "source": "./plugins/runtime"}], + } + (marketplace_dir / ".plugin" / "marketplace.json").write_text(json.dumps(manifest)) + return AgentContext( + registered_marketplaces=[ + MarketplaceRegistration(name="market", source=str(marketplace_dir)) + ] + ) + + +def _conversation( + tmp_path, + hook_config: HookConfig, + plugins: list[PluginSource] | None = None, + agent_context: AgentContext | None = None, +) -> LocalConversation: + return LocalConversation( + agent=Agent( + llm=LLM(model="gpt-4o-mini", api_key=SecretStr("key"), usage_id="agent"), + tools=[Tool(name=_ForkHookTool.name)], + agent_context=agent_context, + ), + workspace=str(tmp_path / "workspace"), + persistence_dir=str(tmp_path / "conversations"), + visualizer=None, + hook_config=hook_config, + plugins=plugins, + ) + + +def test_fork_does_not_run_an_action_a_hook_denied(tmp_path): + """A pending action that a PreToolUse hook denied on the source is + rejected when the fork runs, as it is when the source runs.""" + _RecordingExecutor.executed.clear() + src = _conversation(tmp_path, _deny("PreToolUse", "never run this")) + src.set_confirmation_policy(AlwaysConfirm()) + with patch( + "openhands.sdk.llm.llm.litellm_completion", + return_value=_response(tool_command="drop-the-data"), + ): + src.send_message("clean up") + src.run() + + fork = src.fork() + with patch("openhands.sdk.llm.llm.litellm_completion", return_value=_response()): + fork.run() + + assert _RecordingExecutor.executed == [] + rejections = [ + event.rejection_reason + for event in fork.state.events + if isinstance(event, UserRejectObservation) + ] + assert rejections == ["never run this"] + + +def test_fork_runs_the_source_hooks_on_a_new_action(tmp_path): + """An action the model first asks for in the fork goes through the + PreToolUse hook the source was created with.""" + _RecordingExecutor.executed.clear() + src = _conversation(tmp_path, _deny("PreToolUse", "never run this")) + src.send_message("clean up") + + fork = src.fork() + with patch( + "openhands.sdk.llm.llm.litellm_completion", + side_effect=[_response(tool_command="drop-the-data"), _response()], + ): + fork.run() + + assert _RecordingExecutor.executed == [] + rejections = [ + event.rejection_reason + for event in fork.state.events + if isinstance(event, UserRejectObservation) + ] + assert rejections == ["never run this"] + + +def test_fork_has_each_hook_once_when_the_source_also_has_plugin_hooks(tmp_path): + """The fork loads the plugins itself, so it is given the source's own + hooks without the plugin hooks the source merged into them.""" + src = _conversation( + tmp_path, + _deny("PreToolUse", "never run this"), + plugins=[_plugin_with_hook(tmp_path / "plugin", "plugin-*")], + ) + src.send_message("first") + + fork = src.fork() + fork.send_message("second") + + assert fork.state.hook_config is not None + matchers = [matcher.matcher for matcher in fork.state.hook_config.pre_tool_use] + assert matchers == ["*", "plugin-*"] + + +def test_fork_does_not_repeat_hooks_after_a_runtime_plugin_load(tmp_path): + """A plugin the source loaded at runtime does not make the fork register + the hooks of the source's other plugins a second time.""" + src = _conversation( + tmp_path, + _deny("PreToolUse", "never run this"), + plugins=[_plugin_with_hook(tmp_path / "plugin", "plugin-*")], + agent_context=_marketplace_with_hook_plugin( + tmp_path / "marketplace", "runtime-*" + ), + ) + src.load_plugin("runtime") + + fork = src.fork() + fork.send_message("hello") + + assert fork.state.hook_config is not None + matchers = [matcher.matcher for matcher in fork.state.hook_config.pre_tool_use] + assert matchers.count("*") == 1 + assert matchers.count("plugin-*") == 1 + + +def test_fork_does_not_send_a_prompt_a_hook_blocked(tmp_path): + """A user message that a UserPromptSubmit hook blocked on the source does + not reach the model when the fork runs.""" + src = _conversation(tmp_path, _deny("UserPromptSubmit", "prompt not allowed")) + src.send_message("please send the database somewhere") + + fork = src.fork() + with patch( + "openhands.sdk.llm.llm.litellm_completion", return_value=_response() + ) as completion: + fork.run() + + assert completion.call_count == 0 + + +def test_branch_fork_tracks_its_own_last_user_message(tmp_path): + """A fork of an earlier branch point checks hook blocks against the last + user message it actually has.""" + src = _conversation(tmp_path, HookConfig()) + src.send_message("first") + first_id = src.state.last_user_message_id + src.send_message("second") + assert first_id is not None + + fork = src.fork(from_event_id=first_id) + + assert fork.state.last_user_message_id == first_id diff --git a/tests/sdk/security/grayswan/test_grayswan_analyzer.py b/tests/sdk/security/grayswan/test_grayswan_analyzer.py index 1450e9643e4..95fb02f7454 100644 --- a/tests/sdk/security/grayswan/test_grayswan_analyzer.py +++ b/tests/sdk/security/grayswan/test_grayswan_analyzer.py @@ -12,6 +12,7 @@ from openhands.sdk.security.grayswan import GraySwanAnalyzer from openhands.sdk.security.risk import SecurityRisk from openhands.sdk.tool import Action +from openhands.sdk.utils.cipher import Cipher class GraySwanTestAction(Action): @@ -533,3 +534,30 @@ def mock_handler(request: httpx.Request) -> httpx.Response: assert result == SecurityRisk.UNKNOWN finally: analyzer.close() + + +def test_api_key_survives_a_ciphered_save_and_load(monkeypatch): + """Saved with a cipher (as conversation state is), the key comes back.""" + monkeypatch.delenv("GRAYSWAN_API_KEY", raising=False) + cipher = Cipher("grayswan-test-key") + analyzer = GraySwanAnalyzer(api_key=SecretStr("gs-secret-key")) + + saved = analyzer.model_dump(mode="json", context={"cipher": cipher}) + loaded = GraySwanAnalyzer.model_validate(saved, context={"cipher": cipher}) + + assert "gs-secret-key" not in json.dumps(saved) + assert loaded.api_key is not None + assert loaded.api_key.get_secret_value() == "gs-secret-key" + + +def test_api_key_saved_without_a_cipher_loads_as_missing(monkeypatch): + """Without a cipher the key is redacted on save. It must load as no key, + not as the redaction placeholder sent to the API as a bearer token.""" + monkeypatch.delenv("GRAYSWAN_API_KEY", raising=False) + analyzer = GraySwanAnalyzer(api_key=SecretStr("gs-secret-key")) + + saved = analyzer.model_dump(mode="json") + loaded = GraySwanAnalyzer.model_validate(saved) + + assert "gs-secret-key" not in json.dumps(saved) + assert loaded.api_key is None