diff --git a/.agents/skills/verify-agent-server/references/feature-map/F04-conversation-lifecycle.md b/.agents/skills/verify-agent-server/references/feature-map/F04-conversation-lifecycle.md index 5d3dcba592e..0ef3a7d84d0 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F04-conversation-lifecycle.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F04-conversation-lifecycle.md @@ -30,6 +30,7 @@ Routes: `GET /api/conversations/search`, `GET /api/conversations/count`, - `F04.create-validation-redaction`: a 422 caused by a wrongly typed field next to the LLM key (in an inline `agent.llm` and in `agent_settings.llm`) never contains the posted key; where the error echoes the enclosing object, the key reads ``. - `F04.create-idempotent`: re-posting an existing client `conversation_id` returns 200 with the same conversation, creates nothing and does not resend the initial message. - `F04.create-initial-message-run-false`: an `initial_message` with `"run": false` is recorded without starting a run (the same body with `"run": true` runs). +- `F04.create-initial-sender`: an `initial_message` with `sender_conversation_id` naming another conversation saves the first user message with `sender: "conversation:"` and a first text part naming the sender (`[Sent by [](conversation:<hex>), another conversation, not by the user. ...]`); an unknown sender or the new conversation's own id is 422 and creates nothing. - `F04.create-parent`: `parent_conversation_id` links a child that the parent lists in `sub_conversation_ids`; an unknown parent, a different workspace and a self-parent are 422. - `F04.create-worktree`: `worktree: true` inside a git repository moves the conversation into its own worktree on branch `openhands/<id>`. - `F04.create-observability`: the `X-OpenHands-Observability-*` headers are validated (422 for non-object metadata) and stored in `meta.json`. @@ -49,6 +50,7 @@ Routes: `GET /api/conversations/search`, `GET /api/conversations/count`, - `F04.search-at-scale`: with 2,000 conversations, search pages through every one exactly once in both orders, count is 2,000, the last page answers within 4× the first page's p95 (first page under 2 s), 50 listings grow the server's RSS by under 300 MB, and all 2,000 are listed again after a restart. - `F04.patch-title`: PATCH `title` is stripped, stored in `meta.json`, bumps `updated_at` and moves the conversation to the top of `UPDATED_AT_DESC`. - `F04.patch-tags`: PATCH `tags` replaces the whole map in `meta.json` and `base_state.json` and pushes a `tags` state update on the events socket. +- `F04.patch-tags-while-running`: while a run holds the conversation's state for a tool step, a tags PATCH waits for it, but GET, the batch get and search already show the new tags (from `meta.json`) although `base_state.json` still has the old ones; once the step ends the PATCH answers 200 and both files agree. - `F04.patch-errors`: an empty or 201-character title, a bad tag key and a malformed id are 422; no key is 401; the stored values are untouched. - `F04.patch-whitespace-title`: a whitespace-only title is rejected instead of erasing the stored title. - `F04.patch-unknown-id`: PATCH on an unknown id answers the 404 the OpenAPI document declares, not 200 `{"success": false}`. @@ -61,6 +63,7 @@ Routes: `GET /api/conversations/search`, `GET /api/conversations/count`, - `F04.child-unborn`: the default child launch succeeds in a fresh workspace whose automatically initialized git repository has no commit. - `F04.child-errors`: missing or invalid child-launch fields are 422, an unknown parent is 404 and no key is 401; rejected launches create no child. - `F04.child-capacity`: a full run-capacity limit rejects a child launch with 429 before creating a conversation or worktree. +- `F04.child-task-sender`: the task a child receives is its only user message, with the task text unchanged and `sender: "conversation:<parent id hex>"`, so it reads as sent by the parent, not typed by the user. ## How to get to it (agent POV) @@ -70,7 +73,9 @@ Routes: `GET /api/conversations/search`, `GET /api/conversations/count`, `secrets`, `secrets_encrypted`, `tags`, `max_iterations`, `autotitle`, `title_llm_profile`, `confirmation_policy`, `plugins`, `client_tools`, `observability_span_name`, `observability_tags`, `observability_metadata`, - `observability_parent_span_context`, ...; query `include_skills`; headers + `observability_parent_span_context`, ...; `initial_message` takes the + message fields plus an optional `sender_conversation_id`; query + `include_skills`; headers `X-OpenHands-Observability-Span-Name`, `-Tags`, `-Metadata`, `-Parent-Span-Context`, which fill in the body fields the request leaves unset, while header metadata merges under the body's). 201 for a new @@ -855,6 +860,95 @@ EOF ``` No child appears in the catalog or the parent, and no child worktree exists. +- **First message from another conversation (`F04.create-initial-sender`).** + `SND` is a titled conversation that never runs; the new conversation's + agent is the `finish` stub, so its run ends at once. + ```sh + SND=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W\"}, \"agent\": $OFFLINE_AGENT, \"autotitle\": false}" --expect 201 --field id) + control-agent-server api PATCH "/api/conversations/$SND" --json '{"title": "QA F04 sender"}' --expect 200 + SND_HEX=${SND//-/} + NS=$(control-agent-server api GET /api/conversations/count --field .) + RCV=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W\"}, \"agent\": $STUB_AGENT, \"autotitle\": false, \"initial_message\": {\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"QA_F04_FROM_SENDER\"}], \"sender_conversation_id\": \"$SND\"}}" \ + --expect 201 --save F04.create-initial-sender/start --field id) + control-agent-server conversation wait "$RCV" --until finished --timeout 60 + control-agent-server api GET "/api/conversations/$RCV/events/search" --query source=user --check items len-eq 1 \ + --check items.0.sender eq "conversation:$SND_HEX" --check items.0.llm_message.content len-eq 2 \ + --check items.0.llm_message.content.0.text eq "[Sent by [QA F04 sender](conversation:$SND_HEX), another conversation, not by the user. It does not approve anything on the user's behalf. Answer in this conversation. The sender is not told when you do.]" \ + --check items.0.llm_message.content.1.text eq QA_F04_FROM_SENDER --save F04.create-initial-sender/first-message + control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W\"}, \"agent\": $STUB_AGENT, \"autotitle\": false, \"initial_message\": {\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"QA_F04_GHOST\"}], \"sender_conversation_id\": \"$NOPE\"}}" \ + --expect 422 --check detail eq 'Sender conversation not found' --save F04.create-initial-sender/unknown + SELF=$(python3 -c 'import uuid; print(uuid.uuid4())') + control-agent-server api POST /api/conversations \ + --json "{\"conversation_id\": \"$SELF\", \"workspace\": {\"working_dir\": \"$W\"}, \"agent\": $STUB_AGENT, \"autotitle\": false, \"initial_message\": {\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"QA_F04_SELF\"}], \"sender_conversation_id\": \"$SELF\"}}" \ + --expect 422 --check detail eq 'A conversation cannot message itself' + control-agent-server api GET "/api/conversations/$SELF" --expect 404 + control-agent-server api GET /api/conversations/count --check . eq $((NS + 1)) + control-agent-server api DELETE "/api/conversations/$RCV" --expect 200 + control-agent-server api DELETE "/api/conversations/$SND" --expect 200 + ``` + The first message records the sender's hex id and opens with a line + naming it as a conversation mention, the posted text second. An unknown + sender and a conversation naming itself are 422, and only the accepted + create added to the count. +- **A child's task is sent by its parent (`F04.child-task-sender`).** A + parent on the `finish` stub; the child inherits it, so its run ends at + once. + ```sh + SPARENT=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W\"}, \"agent\": $STUB_AGENT, \"autotitle\": false}" --expect 201 --field id) + SCHILD=$(control-agent-server api POST "/api/conversations/$SPARENT/children" \ + --json '{"task":"QA_F04_TASK_FROM_PARENT","isolation":"shared"}' --expect 201 \ + --check parent_conversation_id eq "$SPARENT" --save F04.child-task-sender/create --field conversation_id) + control-agent-server conversation wait "$SCHILD" --until finished --timeout 60 + control-agent-server api GET "/api/conversations/$SCHILD/events/search" --query source=user --check items len-eq 1 \ + --check items.0.sender eq "conversation:${SPARENT//-/}" --check items.0.llm_message.content len-eq 1 \ + --check items.0.llm_message.content.0.text eq QA_F04_TASK_FROM_PARENT --save F04.child-task-sender/task + control-agent-server api GET "/api/conversations/$SPARENT/events/count" --query source=user --check . eq 0 + control-agent-server api DELETE "/api/conversations/$SCHILD" --expect 200 + control-agent-server api DELETE "/api/conversations/$SPARENT" --expect 200 + ``` + The child's one user message is the task, word for word, with the + parent's hex id as `sender`; the parent has no user message of its own. +- **New tags while a tool step holds the state (`F04.patch-tags-while-running`).** + A stub that asks for `terminal` `sleep 25` and then `finish`: the run + holds the conversation's state for the whole tool step, so the PATCH + (in the background) waits for it. + + Needs: `tmux` + ```sh + SLEEPY=$(control-agent-server fixture llm-stub --name qa-f04-sleepy \ + --step 'tool:terminal:{"command":"sleep 25","summary":"QA F04 wait"}' --step 'tool:finish:{"message":"QA_F04_TAGS_DONE"}' --print-path) + SLEEPY_AGENT="{\"llm\": {\"model\": \"openai/qa-stub\", \"api_key\": \"qa-stub-key\", \"base_url\": \"$SLEEPY/v1\", \"num_retries\": 0}, \"tools\": [{\"name\": \"terminal\"}]}" + mkdir -p "$W/qa-tags" + TG=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$W/qa-tags\"}, \"agent\": $SLEEPY_AGENT, \"autotitle\": false, \"tags\": {\"state\": \"open\"}, \"initial_message\": {\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"Wait.\"}]}}" \ + --expect 201 --field id) + control-agent-server api GET "/api/conversations/$TG/events/count" --query kind=ActionEvent --check . eq 1 --until-ok 30 + control-agent-server api GET "/api/conversations/$TG/events/count" --query kind=ObservationEvent --check . eq 0 + control-agent-server api PATCH "/api/conversations/$TG" --json '{"tags": {"state": "resolved"}}' --timeout 90 \ + --expect 200 --check success eq true --save F04.patch-tags-while-running/patch > /dev/null & + TAG_PATCH=$! + control-agent-server api GET "/api/conversations/$TG" --check tags.state eq resolved --check execution_status eq running \ + --until-ok 10 --save F04.patch-tags-while-running/get-during + control-agent-server state cat "server/workspace/conversations/${TG//-/}/base_state.json" --max-chars 200 --check tags.state eq open + control-agent-server api GET "/api/conversations/$TG/events/count" --query kind=ObservationEvent --check . eq 0 + control-agent-server api GET /api/conversations --query ids="$TG" --check 0.tags.state eq resolved --save F04.patch-tags-while-running/batch-during + control-agent-server api GET /api/conversations/search --query limit=100 --field items \ + | jq -e --arg id "$TG" '[.[] | select(.id == $id) | .tags.state] == ["resolved"]' + wait "$TAG_PATCH" + control-agent-server conversation wait "$TG" --until finished --timeout 60 + control-agent-server state cat "server/workspace/conversations/${TG//-/}/base_state.json" --max-chars 200 --check tags.state eq resolved + control-agent-server state cat "server/workspace/conversations/${TG//-/}/meta.json" --max-chars 200 --check tags.state eq resolved + control-agent-server api GET "/api/conversations/$TG" --check tags.state eq resolved --check execution_status eq finished + control-agent-server api DELETE "/api/conversations/$TG" --expect 200 + ``` + While `sleep 25` runs (no observation yet) and `base_state.json` still + says `open`, GET, the batch get and search already say `resolved`. The + PATCH answers 200 once the step ends, and then both files say + `resolved`. - **Search at scale (`F04.search-at-scale`).** Its own run, so the 2,000 conversations stay out of the main catalog. A helper seeds them over `POST /api/conversations` (eight at a time, offline agent, no first diff --git a/.agents/skills/verify-agent-server/references/feature-map/F06-conversation-events.md b/.agents/skills/verify-agent-server/references/feature-map/F06-conversation-events.md index 548fd7a0dfa..0fb9f45665b 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F06-conversation-events.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F06-conversation-events.md @@ -12,7 +12,7 @@ count, get-one and batch-get (a GET with a JSON body, or its browser-safe `since`), then every new event as raw JSON; a message frame sent on the socket is appended and runs the agent. -Source: `openhands-agent-server/openhands/agent_server/event_router.py`, `openhands-agent-server/openhands/agent_server/event_service.py`, `openhands-agent-server/openhands/agent_server/sockets.py`, `openhands-agent-server/openhands/agent_server/pub_sub.py`, `openhands-agent-server/openhands/agent_server/dependencies.py`, `openhands-sdk/openhands/sdk/conversation/request.py`, `openhands-sdk/openhands/sdk/conversation/event_store.py`, `openhands-sdk/openhands/sdk/event/` +Source: `openhands-agent-server/openhands/agent_server/event_router.py`, `openhands-agent-server/openhands/agent_server/event_service.py`, `openhands-agent-server/openhands/agent_server/sockets.py`, `openhands-agent-server/openhands/agent_server/pub_sub.py`, `openhands-agent-server/openhands/agent_server/dependencies.py`, `openhands-sdk/openhands/sdk/conversation/request.py`, `openhands-sdk/openhands/sdk/conversation/message_request.py`, `openhands-agent-server/openhands/agent_server/message_sender.py`, `openhands-sdk/openhands/sdk/conversation/event_store.py`, `openhands-sdk/openhands/sdk/event/` Needs: `llm` @@ -59,6 +59,9 @@ Routes: `GET /api/conversations/{conversation_id}/events/search`, - `F06.send-run-limit`: with every run slot taken, `POST .../events` with `run: true` returns 429 `Message saved...` and the message is persisted; a socket send gets a `ConversationRunLimitExceeded` `ServerErrorEvent`. - `F06.send-while-running`: `POST .../events` with `run: true` on a conversation that is already running returns 200 (not the 409 of `POST /run`), persists the message, keeps the conversation `running` and starts no second run. +- `F06.send-sender-conversation`: `POST .../events` with `sender_conversation_id` naming another existing conversation saves the user `MessageEvent` with `sender: "conversation:<sender id hex>"` and a first text part `[Sent by [<sender title>](conversation:<hex>), another conversation, not by the user. ...]` before the posted parts, the same in search, get-one and the socket replay; a message without the field has `sender: null` and no such line. +- `F06.send-sender-errors`: a `sender_conversation_id` naming an unknown conversation (422 `Sender conversation not found`), the receiving conversation itself (422 `A conversation cannot message itself`) or a non-UUID (422 at `sender_conversation_id`) is refused and nothing is appended. + - `F06.send-during-tool-call`: `POST .../events` to a conversation whose agent is inside a long tool call saves the message and answers at once, before the tool call ends. ## How to get to it (agent POV) @@ -67,7 +70,10 @@ Routes: `GET /api/conversations/{conversation_id}/events/search`, `SendMessageRequest` body `{"role": "user", "content": [{"type": "text", "text": "..."}], "run": false}` returns `{"success": true}`; `run` defaults to `false`, and a content item is either `{"type": "text", "text": ...}` - or `{"type": "image", "image_urls": [...]}` (http or `data:` URLs). The Python SDK `RemoteConversation.send_message()` and the + or `{"type": "image", "image_urls": [...]}` (http or `data:` URLs). An + optional `sender_conversation_id` (a conversation UUID) marks a message + that another conversation's agent sends, not the user; it is left out of + the request schema's dumps while unset. The Python SDK `RemoteConversation.send_message()` and the TypeScript `RemoteConversation.sendMessage()` always send `run: false` and then call `POST /run` separately; `ConversationClient.sendEvent()` posts the body as given. Recipes drive the route directly (`api POST`), so the body @@ -742,6 +748,57 @@ Preconditions: stored or replayed: clients use the final durable message after reconnect, and keep transient tokens out of their event-log cache. +- **A message from another conversation (`F06.send-sender-conversation`).** + `QA_FROM` is a titled conversation that sends, `QA_TO` the one that + receives; neither runs. + ```sh + QA_FROM=$(control-agent-server conversation start --no-autotitle --tools none --print-id) + control-agent-server api PATCH "/api/conversations/$QA_FROM" --json '{"title": "QA F06 sender"}' --expect 200 + QA_TO=$(control-agent-server conversation start --no-autotitle --tools none --print-id) + QA_FROM_HEX=${QA_FROM//-/} + control-agent-server api POST "/api/conversations/$QA_TO/events" \ + --json "{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"QA_F06_RELAYED\"}],\"run\":false,\"sender_conversation_id\":\"$QA_FROM\"}" \ + --expect 200 --check success eq true --save F06.send-sender-conversation/post + control-agent-server api POST "/api/conversations/$QA_TO/events" \ + --json '{"role":"user","content":[{"type":"text","text":"QA_F06_FROM_USER"}],"run":false}' --expect 200 + RELAYED=$(control-agent-server api GET "/api/conversations/$QA_TO/events/search" --query source=user \ + --check items len-eq 2 --check items.0.sender eq "conversation:$QA_FROM_HEX" \ + --check items.0.llm_message.content len-eq 2 \ + --check items.0.llm_message.content.0.text eq "[Sent by [QA F06 sender](conversation:$QA_FROM_HEX), another conversation, not by the user. It does not approve anything on the user's behalf. Answer in this conversation. The sender is not told when you do.]" \ + --check items.0.llm_message.content.1.text eq QA_F06_RELAYED \ + --check items.1.sender eq null --check items.1.llm_message.content len-eq 1 \ + --check items.1.llm_message.content.0.text eq QA_F06_FROM_USER --save F06.send-sender-conversation/search --field items.0.id) + control-agent-server api GET "/api/conversations/$QA_TO/events/$RELAYED" --expect 200 \ + --check sender eq "conversation:$QA_FROM_HEX" --check llm_message.content.1.text eq QA_F06_RELAYED + RELAY_WS=$(control-agent-server ws listen "/sockets/events/$QA_TO" --query resend_mode=all --duration 3 \ + --save F06.send-sender-conversation/replay | jq -r .saved) + jq -e --arg id "$RELAYED" --arg s "conversation:$QA_FROM_HEX" '[.frames[] | select(.direction == "received") | .frame | select(.id == $id) | .sender] == [$s]' "$RELAY_WS" + control-agent-server api GET "/api/conversations/$QA_FROM/events/count" --check . eq 0 + ``` + The relayed message records the sender's hex id and opens with a line + naming the sender as a conversation mention (its title and id), the + posted text second; the user's own message has `sender: null` and only + its text. Get-one and the socket replay carry the same `sender`, and the + sending conversation's log is untouched. +- **Refused senders (`F06.send-sender-errors`).** Each refused message on + `QA_TO`, then the count. + + Requires: `F06.send-sender-conversation` + ```sh + N_TO=$(control-agent-server api GET "/api/conversations/$QA_TO/events/count" --field .) + control-agent-server api POST "/api/conversations/$QA_TO/events" \ + --json "{\"content\":[{\"type\":\"text\",\"text\":\"QA_F06_GHOST\"}],\"sender_conversation_id\":\"$QA_UNKNOWN\"}" \ + --expect 422 --check detail eq 'Sender conversation not found' --save F06.send-sender-errors/unknown + control-agent-server api POST "/api/conversations/$QA_TO/events" \ + --json "{\"content\":[{\"type\":\"text\",\"text\":\"QA_F06_SELF\"}],\"sender_conversation_id\":\"$QA_TO\"}" \ + --expect 422 --check detail eq 'A conversation cannot message itself' --save F06.send-sender-errors/self + control-agent-server api POST "/api/conversations/$QA_TO/events" \ + --json '{"content":[{"type":"text","text":"QA_F06_BADID"}],"sender_conversation_id":"qa-not-a-uuid"}' \ + --expect 422 --check detail.0.loc contains sender_conversation_id --check detail.0.type eq uuid_parsing + control-agent-server api GET "/api/conversations/$QA_TO/events/count" --check . eq "$N_TO" --save F06.send-sender-errors/count + control-agent-server api GET "/api/conversations/$QA_TO/events/count" --query body=QA_F06_GHOST --check . eq 0 + ``` + All three are 422 and the receiving log keeps its count. - **A message during a tool call (`F06.send-during-tool-call`), known bug #6729.** Last bullet. A scripted model asks for `terminal` `sleep 20` on every call (a real model cannot be held inside a tool call on demand); once its @@ -776,6 +833,14 @@ Preconditions: ## Gotchas +- `RemoteConversation.send_message(..., + sender=...)` posts a plain `sender` field, which this route ignores (the + message is saved with `sender: null`); only `sender_conversation_id` + marks a sender over REST (`F06.send-sender-conversation`). Known bug + [#7004](https://github.com/OpenHands/software-agent-sdk/issues/7004): a + local conversation records the same `sender` (`S04.conversation-sender`), + so the goal runner's, `TaskManager`'s and the delegate tool's senders are + lost on a remote conversation. - `kind` filters take either the bare class name events carry in `kind` (`MessageEvent`) or the fully-qualified class path (`openhands.sdk.event.llm_convertible.message.MessageEvent`, diff --git a/.agents/skills/verify-agent-server/references/feature-map/F09-model-switching-plugins.md b/.agents/skills/verify-agent-server/references/feature-map/F09-model-switching-plugins.md index 583071b78b6..2284244f25e 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F09-model-switching-plugins.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F09-model-switching-plugins.md @@ -5,14 +5,20 @@ recreating it: swap the agent's LLM for a caller-supplied config (inline key, a cipher token from an `encrypted` read, or a saved provider connection), switch it to a saved LLM profile, change the model and the reasoning effort an ACP conversation's subprocess runs, and load a plugin from the conversation's registered -marketplaces into the live agent (its skills, commands and hooks). Every +marketplaces into the live agent (its skills, commands and hooks). A +conversation whose LLM is a model router (`RouterLLM`) also exposes the +router: `GET .../model_router` reports its routes and selection, `POST +.../model_router/routes` adds a saved profile as a route (wrapping a plain +LLM in a router first), and `PATCH .../model_router` pins a route or returns +to automatic routing, all without replacing the router; `switch_llm` and +`switch_profile` still replace the whole LLM, router included. Every switch answers `{"success": true}`, is written to the conversation's `base_state.json`, shows up in `GET /api/conversations/{id}` and as a `ConversationStateUpdateEvent` on the events socket, and survives a restart; the next turn runs on the new model and its usage is recorded under the new LLM's `usage_id`. -Source: `openhands-agent-server/openhands/agent_server/conversation_router.py`, `openhands-agent-server/openhands/agent_server/event_service.py`, `openhands-agent-server/openhands/agent_server/_secrets_exposure.py`, `openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py`, `openhands-sdk/openhands/sdk/llm/llm_registry.py`, `openhands-sdk/openhands/sdk/llm/llm_profile_store.py`, `openhands-sdk/openhands/sdk/marketplace/`, `openhands-sdk/openhands/sdk/plugin/`, `openhands-sdk/openhands/sdk/agent/acp_agent.py` +Source: `openhands-agent-server/openhands/agent_server/conversation_router.py`, `openhands-agent-server/openhands/agent_server/event_service.py`, `openhands-agent-server/openhands/agent_server/_secrets_exposure.py`, `openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py`, `openhands-sdk/openhands/sdk/llm/llm_registry.py`, `openhands-sdk/openhands/sdk/llm/llm_profile_store.py`, `openhands-sdk/openhands/sdk/llm/router/`, `openhands-sdk/openhands/sdk/marketplace/`, `openhands-sdk/openhands/sdk/plugin/`, `openhands-sdk/openhands/sdk/agent/acp_agent.py` Needs: `llm`, `tmux`, `node`, `network` @@ -20,7 +26,10 @@ Routes: `POST /api/conversations/{conversation_id}/switch_profile`, `POST /api/conversations/{conversation_id}/switch_llm`, `POST /api/conversations/{conversation_id}/load_plugin`, `POST /api/conversations/{conversation_id}/switch_acp_model`, -`POST /api/conversations/{conversation_id}/switch_acp_effort` +`POST /api/conversations/{conversation_id}/switch_acp_effort`, +`GET /api/conversations/{conversation_id}/model_router`, +`POST /api/conversations/{conversation_id}/model_router/routes`, +`PATCH /api/conversations/{conversation_id}/model_router` ## Sub-features @@ -61,6 +70,18 @@ Routes: `POST /api/conversations/{conversation_id}/switch_profile`, - `F09.load-plugin-twice`: loading the same plugin twice leaves one copy of its hooks, so they still run once per message. - `F09.load-plugin-sdk`: the Python SDK's `RemoteConversation.load_plugin(ref)`, on a conversation reattached with `RemoteConversation.attach`, raises `HTTPStatusError` 404 for an unknown plugin and then loads the plugin, whose skill and hook `GET` shows. - `F09.restart-persist`: after a restart the switched LLM, the switched profile (which the next turn runs on), the deferred ACP model and the loaded plugin's skills and hooks are still in place. +- `F09.model-router-status`: `GET .../model_router` on a plain-LLM conversation is 200 `{"enabled": false, "selection": null, "last_route_name": null, "routes": []}`; on a conversation created with a `MultimodalRouter` LLM it reports `enabled: true`, `selection.mode` `auto`, no `last_route_name` and each route's `name`, `model` and `label`; an unknown conversation is 404 and no key 401. +- `F09.model-router-add-route`: `POST .../model_router/routes` with a saved profile wraps a plain LLM in a `base_router` (the old LLM becomes route `default`, or `base_route_name`), adds the profile as a route named `route_name` (default: the profile name) with `usage_id` `profile:<name>`, extends an existing router keeping its type, stays in `auto` mode and answers the new status; `GET`, `base_state.json` (keys encrypted) and the socket's `agent` update show the routes, and no plaintext key is stored or pushed. +- `F09.model-router-add-route-errors`: an unknown profile is 404 `Profile '<name>' not found`, a route name already in use 400 `Model router route already exists: <name>`, an invalid profile name 400, a missing `profile_name` 422, an unknown conversation 404 and no key 401; the routes are unchanged. +- `F09.model-router-select`: `PATCH .../model_router` with `{"selection": {"mode": "manual", "manual_route_name": ...}}` pins that route and `{"mode": "auto"}` unpins it, keeping the router and its routes; the answer, `GET .../model_router`, `GET` (`agent.llm.selection`), `base_state.json` and the socket's `agent` update all show the selection. +- `F09.model-router-select-errors`: an unknown route is 400 naming the available routes, `manual` without a route, `auto` with one, an unknown `mode` and a missing `selection` are 422, a conversation whose LLM is not a router is 400 `Current agent LLM is not a RouterLLM.`, an unknown conversation 404 and no key 401; the selection is unchanged. +- `F09.model-router-switch-replaces`: `switch_profile` and `switch_llm` on a router conversation install the plain LLM in place of the router (routes and pin gone, `model_router` reports `enabled: false`, `PATCH` is then 400), and a later route add wraps the switched LLM in a new router. +- `F09.model-router-acp-refused`: on an ACP conversation `GET .../model_router` reports `enabled: false`, and adding a route or changing the selection is 400 pointing at `switch_acp_model`, leaving the `acp-managed` placeholder LLM alone. +- `F09.model-router-sdk`: the Python SDK's `RemoteConversation.get_model_router()`, `add_model_router_route(profile, route_name=, base_route_name=)`, `update_model_router_selection(selection)`, `switch_routed_llm(route)` and `resume_auto_llm_routing()` reach the routes and return `ModelRouterStatus`; a refused route or profile raises `httpx.HTTPStatusError` 400 or 404. +- `F09.ts-client-model-router`: the TypeScript client's `getModelRouter`, `addModelRouterRoute`, `updateModelRouter`, `switchModelRouter` and `resumeAutoModelRouting` on `ConversationClient` and `ConversationManager` reach the routes and resolve with the status; an unknown route rejects with 400, an unknown profile 404, a duplicate route 400 and a keyless client 401. +- `F09.model-router-run`: after a first turn, a route added from the `deepseek-flash` profile and pinned serves the next turn (its tokens under `profile:deepseek-flash`, `last_route_name` the route), and after a return to `auto` the next turn runs on the first route (`default`) again. +- `F09.model-router-before-first-run`: a route added and pinned before the conversation's first message serves the first run, and that run's tokens are recorded under the route's `usage_id` `profile:deepseek-flash`. +- `F09.model-router-restart`: after a restart the routes and the pinned selection are still in place and the selection can be changed again; conversations whose router a switch replaced stay plain. ## How to get to it (agent POV) @@ -71,6 +92,14 @@ Routes: `POST /api/conversations/{conversation_id}/switch_profile`, `POST /api/conversations/{id}/switch_acp_effort` `{"effort": "..."}`, `POST /api/conversations/{id}/load_plugin` `{"plugin_ref": "<plugin>[@<marketplace>]"}`. Each answers `{"success": true}`; every recipe below drives them. + Model router, owned here: `GET /api/conversations/{id}/model_router`, + `POST /api/conversations/{id}/model_router/routes` + `{"profile_name": "...", "route_name": "..."?, "base_route_name": "default"?}`, + `PATCH /api/conversations/{id}/model_router` + `{"selection": {"mode": "auto"|"manual", "manual_route_name": "..."?}}`. + Each answers a `ModelRouterStatus` + (`{"enabled", "selection", "last_route_name", "routes": [{"name", "model", "label"}]}`); + `/server_info` lists the feature flag `conversation_model_router_routes_v1`. - Second views: `GET /api/conversations/{id}` (`agent.llm`, `agent.condenser.llm`, `stats.usage_to_metrics.<usage_id>`, `current_model_id`, `agent.acp_model`, `current_effort`, `available_efforts`, `agent.acp_effort`, @@ -91,12 +120,27 @@ Routes: `POST /api/conversations/{conversation_id}/switch_profile`, (`F09.load-plugin-sdk` drives it). `RemoteConversation` has no `switch_llm`, `switch_profile`, `switch_acp_model` or `switch_acp_effort` method; Python consumers post to the routes directly. + For the model router, `LocalConversation.add_model_router_route`, + `update_model_router_selection`, `switch_routed_llm` and + `resume_auto_llm_routing` are what the routes call (SDK side: + `S03.model-router-*` in the SDK map), and `RemoteConversation` has + `get_model_router()`, `add_model_router_route(...)`, + `update_model_router_selection(selection)`, `switch_routed_llm(route)` and + `resume_auto_llm_routing()`, each returning a `ModelRouterStatus` + (`F09.model-router-sdk` drives them). - TypeScript client: `ConversationClient.switchProfile(id, name)`, `switchLLM(id, llm)`, `switchAcpModel(id, model)`, `switchAcpEffort(id, effort)`; `RemoteConversation.switchProfile`, `switchLlm`, `switchAcpModel`, `switchAcpEffort`; `ConversationManager.switchProfile`; + model router: `getModelRouter`, `addModelRouterRoute(id, profileName, + routeName?, baseRouteName = 'default')`, `updateModelRouter(id, selection)`, + `switchModelRouter(id, routeName)` and `resumeAutoModelRouting(id)` on + `ConversationClient` and `ConversationManager`, and the same methods + without the id on `RemoteConversation` (types `ModelRouterSelection`, + `ModelRouteInfo`, `ModelRouterStatus`; `F09.ts-client-model-router` drives + the `ConversationClient` and `ConversationManager` ones); `ConversationClient.loadPlugin(id, ref)` and `PluginsClient.loadPluginForConversation(id, ref)` post `{"plugin_ref": ref}` to `load_plugin` (driven by `F34.load-plugin`). @@ -938,6 +982,399 @@ Preconditions: All four conversations come back as switched, and the first turn after the restart runs on `deepseek-pro` again (its token count grows past `PRO1`). +- **Model-router status (`F09.model-router-status`).** Save two dummy + profiles, `qa-f09-route-a` and `qa-f09-route-b` (unreachable endpoints: + nothing in the router bullets before `F09.model-router-run` calls a model), + and start `RQ`, an idle conversation whose agent carries a plain + placeholder LLM, and `RM`, one whose agent LLM is a `MultimodalRouter` + with routes `primary` and `secondary`. + ```sh + control-agent-server api POST /api/profiles/qa-f09-route-a --json '{"llm": {"model": "openai/qa-f09-route-a", "api_key": "sk-qa-f09-route-a", "base_url": "http://127.0.0.1:9/v1"}}' --expect 201 + control-agent-server api POST /api/profiles/qa-f09-route-b --json '{"llm": {"model": "openai/qa-f09-route-b", "api_key": "sk-qa-f09-route-b", "base_url": "http://127.0.0.1:9/v1"}}' --expect 201 + RQ=$(control-agent-server conversation start --no-autotitle --body-json '{"agent": {"kind": "Agent", "tools": [], "llm": {"model": "openai/qa-f09-route-base", "api_key": "sk-qa-f09-route-base", "base_url": "http://127.0.0.1:9/v1"}}}' --print-id) + RM=$(control-agent-server conversation start --no-autotitle --body-json '{"agent": {"kind": "Agent", "tools": [], "llm": {"router_name": "multimodal_router", "usage_id": "qa-f09-mm", "llms_for_routing": {"primary": {"model": "openai/qa-f09-primary", "api_key": "sk-qa-f09-route-mm", "base_url": "http://127.0.0.1:9/v1", "usage_id": "qa-f09-primary"}, "secondary": {"model": "openai/qa-f09-secondary", "api_key": "sk-qa-f09-route-mm", "base_url": "http://127.0.0.1:9/v1", "usage_id": "qa-f09-secondary"}}}}}' --print-id) + control-agent-server api GET "/api/conversations/$RQ/model_router" --expect 200 --check enabled eq false --check selection eq null \ + --check last_route_name eq null --check routes len-eq 0 --save F09.model-router-status/plain + control-agent-server api GET "/api/conversations/$RM/model_router" --expect 200 --check enabled eq true --check selection.mode eq auto \ + --check selection.manual_route_name eq null --check last_route_name eq null --check routes len-eq 2 \ + --check routes.0.name eq primary --check routes.0.model eq openai/qa-f09-primary --check routes.0.label eq primary \ + --check routes.1.name eq secondary --check routes.1.model eq openai/qa-f09-secondary --save F09.model-router-status/router + control-agent-server api GET "/api/conversations/$RM" --quiet --check agent.llm.router_name eq multimodal_router \ + --check agent.llm.model eq multimodal_router --check agent.llm.usage_id eq qa-f09-mm \ + --check agent.llm.llms_for_routing.primary.api_key missing --check agent.llm.selection.mode eq auto + control-agent-server api GET /api/conversations/00000000-0000-4000-8000-000000000000/model_router --expect 404 + control-agent-server api GET "/api/conversations/$RM/model_router" --auth none --expect 401 --check detail eq Unauthorized + ``` + The plain conversation reports `enabled: false` with no selection and no + routes. The router conversation reports both routes in order, each with + its model and its name as `label`, `selection.mode` `auto` and no + `last_route_name` (only a model call sets it). `GET` shows the router + itself: its `model` is the placeholder `multimodal_router`, its routes' + keys are masked. An unknown id is 404, no key 401. +- **Add a route from a saved profile (`F09.model-router-add-route`).** Wrap + `RQ`'s plain LLM, add a second route under its own name, extend `RM`'s + router, and wrap `RQ2`'s LLM under a chosen base route name. + + Requires: `F09.model-router-status` + ```sh + control-agent-server ws start "/sockets/events/$RQ" --name qa-f09-router --duration 120 + control-agent-server api POST "/api/conversations/$RQ/model_router/routes" --json '{"profile_name": "qa-f09-route-a"}' \ + --expect 200 --check enabled eq true --check selection.mode eq auto --check routes len-eq 2 \ + --check routes.0.name eq default --check routes.0.model eq openai/qa-f09-route-base \ + --check routes.1.name eq qa-f09-route-a --check routes.1.model eq openai/qa-f09-route-a --save F09.model-router-add-route/wrap + control-agent-server api POST "/api/conversations/$RQ/model_router/routes" --json '{"profile_name": "qa-f09-route-b", "route_name": "reviewer"}' \ + --expect 200 --check routes len-eq 3 --check routes.2.name eq reviewer --check routes.2.model eq openai/qa-f09-route-b \ + --check selection.mode eq auto --save F09.model-router-add-route/named + control-agent-server api GET "/api/conversations/$RQ" --quiet --check agent.llm.router_name eq base_router \ + --check agent.llm.llms_for_routing.default.usage_id eq default \ + --check agent.llm.llms_for_routing.qa-f09-route-a.usage_id eq profile:qa-f09-route-a \ + --check agent.llm.llms_for_routing.reviewer.usage_id eq profile:qa-f09-route-b \ + --check agent.llm.llms_for_routing.reviewer.api_key missing --save F09.model-router-add-route/get + control-agent-server state cat "server/workspace/conversations/${RQ//-/}/base_state.json" --max-chars 200 \ + --check agent.llm.router_name eq base_router --check agent.llm.llms_for_routing.reviewer.model eq openai/qa-f09-route-b \ + --check agent.llm.llms_for_routing.reviewer.api_key matches '^gAAAAA' --not-contains sk-qa-f09-route + control-agent-server ws read qa-f09-router --kinds ConversationStateUpdateEvent --contains 'profile:qa-f09-route-b' --expect-min 1 --wait 20 \ + --save F09.model-router-add-route/frames + control-agent-server ws stop qa-f09-router --contains sk-qa-f09-route --expect-none + control-agent-server api POST "/api/conversations/$RM/model_router/routes" --json '{"profile_name": "qa-f09-route-b", "route_name": "reviewer", "base_route_name": "unused"}' \ + --expect 200 --check routes len-eq 3 --check routes.0.name eq primary --check routes.2.name eq reviewer --save F09.model-router-add-route/extend + control-agent-server api GET "/api/conversations/$RM" --quiet --check agent.llm.router_name eq multimodal_router \ + --check agent.llm.llms_for_routing.reviewer.usage_id eq profile:qa-f09-route-b + RQ2=$(control-agent-server conversation start --no-autotitle --body-json '{"agent": {"kind": "Agent", "tools": [], "llm": {"model": "openai/qa-f09-route-base2", "api_key": "sk-qa-f09-route-base2", "base_url": "http://127.0.0.1:9/v1"}}}' --print-id) + control-agent-server api POST "/api/conversations/$RQ2/model_router/routes" --json '{"profile_name": "qa-f09-route-a", "base_route_name": "main"}' \ + --expect 200 --check routes.0.name eq main --check routes.0.model eq openai/qa-f09-route-base2 --check routes.1.name eq qa-f09-route-a + control-agent-server state grep sk-qa-f09-route --glob 'server/**/*' --expect-none + ``` + The first add turns `RQ`'s LLM into a `base_router` whose route `default` + is the old LLM (its `usage_id` stays `default`) and whose second route is + named after the profile, with `usage_id` `profile:qa-f09-route-a`; the + second add uses the given `route_name`. The router stays in `auto` mode. + `GET` and `base_state.json` show the routes (keys masked in `GET`, Fernet + tokens on disk), the socket pushes the new `agent`, and no plaintext key + is stored or pushed. On `RM` the route joins the existing + `MultimodalRouter`, which keeps its type, so `base_route_name` is unused; + on the plain `RQ2` it names the wrapped LLM's route. +- **Route add errors (`F09.model-router-add-route-errors`).** Unknown, + invalid and missing profiles, route names already taken, an unknown + conversation and no key, on `RQ`. + + Requires: `F09.model-router-add-route` + ```sh + control-agent-server api POST "/api/conversations/$RQ/model_router/routes" --json '{"profile_name": "qa-f09-nope"}' \ + --expect 404 --check detail eq "Profile 'qa-f09-nope' not found" --save F09.model-router-add-route-errors/unknown-profile + control-agent-server api POST "/api/conversations/$RQ/model_router/routes" --json '{"profile_name": "qa-f09-route-b", "route_name": "default"}' \ + --expect 400 --check detail eq 'Model router route already exists: default' --save F09.model-router-add-route-errors/duplicate + control-agent-server api POST "/api/conversations/$RQ/model_router/routes" --json '{"profile_name": "qa-f09-route-a"}' \ + --expect 400 --check detail eq 'Model router route already exists: qa-f09-route-a' + control-agent-server api POST "/api/conversations/$RQ/model_router/routes" --json '{"profile_name": "../qa"}' --expect 400 --check detail contains 'Invalid profile name' + control-agent-server api POST "/api/conversations/$RQ/model_router/routes" --json '{"route_name": "qa-f09-x"}' --expect 422 --check detail.0.loc.1 eq profile_name + control-agent-server api POST /api/conversations/00000000-0000-4000-8000-000000000000/model_router/routes --json '{"profile_name": "qa-f09-route-a"}' --expect 404 + control-agent-server api POST "/api/conversations/$RQ/model_router/routes" --auth none --json '{"profile_name": "qa-f09-route-b", "route_name": "qa-f09-noauth"}' --expect 401 + control-agent-server api GET "/api/conversations/$RQ/model_router" --quiet --check routes len-eq 3 --check routes.2.name eq reviewer + ``` + 404 with the profile's name, 400 for a route name already in the table + (whether it came from the wrap or from a profile), 400 for a name that + breaks the profile rules, 422 without `profile_name`, 404 for the unknown + id and 401 without a key; `RQ` still has its three routes. +- **Pin a route and return to auto (`F09.model-router-select`).** On `RQ`, + then on `RM`. + + Requires: `F09.model-router-add-route` + ```sh + control-agent-server ws start "/sockets/events/$RQ" --name qa-f09-router-select --duration 120 + control-agent-server api PATCH "/api/conversations/$RQ/model_router" --json '{"selection": {"mode": "manual", "manual_route_name": "reviewer"}}' \ + --expect 200 --check enabled eq true --check selection.mode eq manual --check selection.manual_route_name eq reviewer \ + --check routes len-eq 3 --save F09.model-router-select/manual + control-agent-server api GET "/api/conversations/$RQ/model_router" --quiet --check selection.manual_route_name eq reviewer + control-agent-server api GET "/api/conversations/$RQ" --quiet --check agent.llm.router_name eq base_router \ + --check agent.llm.selection.mode eq manual --check agent.llm.selection.manual_route_name eq reviewer --save F09.model-router-select/get + control-agent-server state cat "server/workspace/conversations/${RQ//-/}/base_state.json" --max-chars 200 \ + --check agent.llm.selection.manual_route_name eq reviewer + control-agent-server ws read qa-f09-router-select --kinds ConversationStateUpdateEvent --contains '"manual_route_name": "reviewer"' \ + --expect-min 1 --wait 20 --save F09.model-router-select/frames + control-agent-server ws stop qa-f09-router-select + control-agent-server api PATCH "/api/conversations/$RQ/model_router" --json '{"selection": {"mode": "auto"}}' \ + --expect 200 --check selection.mode eq auto --check selection.manual_route_name eq null --check routes len-eq 3 --save F09.model-router-select/auto + control-agent-server state cat "server/workspace/conversations/${RQ//-/}/base_state.json" --max-chars 200 --check agent.llm.selection.mode eq auto + control-agent-server api PATCH "/api/conversations/$RM/model_router" --json '{"selection": {"mode": "manual", "manual_route_name": "secondary"}}' \ + --expect 200 --check selection.manual_route_name eq secondary + control-agent-server api GET "/api/conversations/$RM" --quiet --check agent.llm.router_name eq multimodal_router \ + --check agent.llm.selection.manual_route_name eq secondary + ``` + The `PATCH` answers the new status; `GET .../model_router`, the router's + `selection` in `GET`, `base_state.json` and a `ConversationStateUpdateEvent` + with key `agent` all carry the pin, and `auto` clears it + (`manual_route_name: null`). The router and its routes are kept: on `RM` + the pin applies to the `MultimodalRouter`, which stays one. +- **Selection errors (`F09.model-router-select-errors`).** Pin `RQ` to + `qa-f09-route-a`, then try every refused selection; finally a + conversation `RP` whose LLM is not a router. + + Requires: `F09.model-router-add-route` + ```sh + control-agent-server api PATCH "/api/conversations/$RQ/model_router" --json '{"selection": {"mode": "manual", "manual_route_name": "qa-f09-route-a"}}' --expect 200 + control-agent-server api PATCH "/api/conversations/$RQ/model_router" --json '{"selection": {"mode": "manual", "manual_route_name": "qa-f09-nope"}}' \ + --expect 400 --check detail eq "LLM route 'qa-f09-nope' was not found. Available routes: default, qa-f09-route-a, reviewer" \ + --save F09.model-router-select-errors/unknown-route + control-agent-server api PATCH "/api/conversations/$RQ/model_router" --json '{"selection": {"mode": "manual"}}' \ + --expect 422 --check detail.0.msg contains 'manual_route_name is required in manual routing mode' + control-agent-server api PATCH "/api/conversations/$RQ/model_router" --json '{"selection": {"mode": "auto", "manual_route_name": "default"}}' \ + --expect 422 --check detail.0.msg contains 'manual_route_name is only valid in manual routing mode' + control-agent-server api PATCH "/api/conversations/$RQ/model_router" --json '{"selection": {"mode": "fast"}}' --expect 422 --check detail.0.loc.2 eq mode + control-agent-server api PATCH "/api/conversations/$RQ/model_router" --json '{}' --expect 422 --check detail.0.loc.1 eq selection + control-agent-server api PATCH /api/conversations/00000000-0000-4000-8000-000000000000/model_router --json '{"selection": {"mode": "auto"}}' --expect 404 + control-agent-server api PATCH "/api/conversations/$RQ/model_router" --auth none --json '{"selection": {"mode": "auto"}}' --expect 401 + control-agent-server api GET "/api/conversations/$RQ/model_router" --quiet --check selection.mode eq manual --check selection.manual_route_name eq qa-f09-route-a + RP=$(control-agent-server conversation start --no-autotitle --body-json '{"agent": {"kind": "Agent", "tools": [], "llm": {"model": "openai/qa-f09-route-plain", "api_key": "sk-qa-f09-route-plain", "base_url": "http://127.0.0.1:9/v1"}}}' --print-id) + control-agent-server api PATCH "/api/conversations/$RP/model_router" --json '{"selection": {"mode": "auto"}}' \ + --expect 400 --check detail eq 'Current agent LLM is not a RouterLLM.' --save F09.model-router-select-errors/not-a-router + control-agent-server api GET "/api/conversations/$RP/model_router" --quiet --check enabled eq false + ``` + An unknown route is 400 and lists the routes; the selection shapes the + model refuses are 422 (a manual pin without a route, an `auto` that names + one, an unknown `mode`, no `selection`); the unknown id is 404 and no key + 401. `RQ` stays pinned to `qa-f09-route-a`. A `PATCH` never creates a + router: on `RP` it is 400 and `RP` stays plain (adding a route is what + wraps an LLM). +- **switch_llm and switch_profile replace the router (`F09.model-router-switch-replaces`).** + `switch_profile` on `RM` (a pinned `MultimodalRouter` with three routes), + `switch_llm` on `RQ2` (a wrapped router), then a route add on `RQ2`. + + Requires: `F09.model-router-select` + ```sh + control-agent-server api POST "/api/conversations/$RM/switch_profile" --json '{"profile_name": "qa-f09-route-a"}' \ + --expect 200 --check success eq true --save F09.model-router-switch-replaces/switch-profile + control-agent-server api GET "/api/conversations/$RM/model_router" --check enabled eq false --check routes len-eq 0 --save F09.model-router-switch-replaces/status + control-agent-server api GET "/api/conversations/$RM" --quiet --check agent.llm.model eq openai/qa-f09-route-a \ + --check agent.llm.usage_id eq profile:qa-f09-route-a --check agent.llm.router_name missing --check agent.llm.llms_for_routing missing + control-agent-server api PATCH "/api/conversations/$RM/model_router" --json '{"selection": {"mode": "auto"}}' \ + --expect 400 --check detail eq 'Current agent LLM is not a RouterLLM.' + control-agent-server api POST "/api/conversations/$RQ2/switch_llm" \ + --json '{"llm": {"model": "openai/qa-f09-route-switched", "api_key": "sk-qa-f09-route-switched", "base_url": "http://127.0.0.1:9/v1", "usage_id": "qa-f09-route-switched"}}' \ + --expect 200 --check success eq true + control-agent-server api GET "/api/conversations/$RQ2/model_router" --quiet --check enabled eq false + control-agent-server api POST "/api/conversations/$RQ2/model_router/routes" --json '{"profile_name": "qa-f09-route-b"}' \ + --expect 200 --check routes len-eq 2 --check routes.0.name eq default --check routes.0.model eq openai/qa-f09-route-switched \ + --check routes.1.name eq qa-f09-route-b --save F09.model-router-switch-replaces/rewrap + ``` + Both switches answer `{"success": true}` and install the posted or saved + LLM as the agent's whole LLM: the router, its routes and its pin are gone + (`enabled: false`, no `router_name` or `llms_for_routing` in `GET`), and + a selection change is 400 as on any plain conversation. To change models + and keep the router, pin a route instead. A route add after the switch + builds a new `base_router` around the switched LLM (`default`), without + the routes the old router had (`main`, `qa-f09-route-a`). +- **Model router on an ACP conversation (`F09.model-router-acp-refused`).** + An ACP conversation `RA` that has not run. + + Requires: `F09.model-router-status` + ```sh + RA=$(control-agent-server conversation start --no-autotitle --body-json '{"agent_settings": {"agent_kind": "acp", "acp_server": "claude-code", "acp_model": "sonnet"}}' --print-id) + control-agent-server api GET "/api/conversations/$RA/model_router" --check enabled eq false --check routes len-eq 0 + control-agent-server api POST "/api/conversations/$RA/model_router/routes" --json '{"profile_name": "qa-f09-route-a"}' \ + --expect 400 --check detail contains switch_acp_model --save F09.model-router-acp-refused/add-route + control-agent-server api PATCH "/api/conversations/$RA/model_router" --json '{"selection": {"mode": "auto"}}' \ + --expect 400 --check detail contains switch_acp_model --save F09.model-router-acp-refused/select + control-agent-server api GET "/api/conversations/$RA" --quiet --check agent.kind eq ACPAgent --check agent.llm.usage_id eq acp-managed \ + --check agent.llm.llms_for_routing missing --check agent.acp_model eq sonnet + ``` + The ACP agent's placeholder LLM is not a router (`enabled: false`); a + route add and a selection change are both refused with the same 400 as + `switch_llm` (`switch_llm and switch_profile are not supported for ACP + conversations; use switch_acp_model instead.`), and the placeholder stays. +- **Model router from the Python SDK (`F09.model-router-sdk`).** A fresh + plain conversation `RS`, reattached from an SDK program. + + Requires: `F09.model-router-status` + ```sh + RS=$(control-agent-server conversation start --no-autotitle --body-json '{"agent": {"kind": "Agent", "tools": [], "llm": {"model": "openai/qa-f09-route-sdk", "api_key": "sk-qa-f09-route-sdk", "base_url": "http://127.0.0.1:9/v1"}}}' --print-id) + mkdir -p "$AGENT_SERVER_VERIFY_RUN/fixtures/qa-f09-router-sdk" + control-agent-server exec --env RS="$RS" --env WD="$AGENT_SERVER_VERIFY_RUN/fixtures/qa-f09-router-sdk" --expect-output SDK_MODEL_ROUTER_OK \ + --save F09.model-router-sdk/program -- .venv/bin/python - <<'PY' + import os + import uuid + + import httpx + + from openhands.sdk import RemoteConversation, RemoteWorkspace + from openhands.sdk.llm import ModelRouterSelection, ModelRouterStatus + + ws = RemoteWorkspace(host=os.environ["AGENT_SERVER_URL"], api_key=os.environ["SESSION_API_KEY"], working_dir=os.environ["WD"]) + conv = RemoteConversation.attach(ws, uuid.UUID(os.environ["RS"]), visualizer=None) + status = conv.get_model_router() + assert isinstance(status, ModelRouterStatus) and status.enabled is False and status.routes == [], status + status = conv.add_model_router_route("qa-f09-route-a", base_route_name="base") + assert [r.name for r in status.routes] == ["base", "qa-f09-route-a"], status + status = conv.add_model_router_route("qa-f09-route-b", route_name="reviewer") + assert [r.model for r in status.routes] == ["openai/qa-f09-route-sdk", "openai/qa-f09-route-a", "openai/qa-f09-route-b"], status + status = conv.switch_routed_llm("reviewer") + assert status.selection == ModelRouterSelection(mode="manual", manual_route_name="reviewer"), status + status = conv.resume_auto_llm_routing() + assert status.selection.mode == "auto" and status.selection.manual_route_name is None, status + status = conv.update_model_router_selection(ModelRouterSelection(mode="manual", manual_route_name="qa-f09-route-a")) + assert status.selection.manual_route_name == "qa-f09-route-a", status + for call, code in ((lambda: conv.switch_routed_llm("qa-f09-nope"), 400), (lambda: conv.add_model_router_route("qa-f09-nope"), 404)): + try: + call() + raise AssertionError("a refused call succeeded") + except httpx.HTTPStatusError as e: + assert e.response.status_code == code, e + print("SDK_MODEL_ROUTER_OK") + conv.close() + PY + control-agent-server api GET "/api/conversations/$RS/model_router" --quiet --check selection.manual_route_name eq qa-f09-route-a \ + --check routes len-eq 3 --check routes.0.name eq base --save F09.model-router-sdk/get + ``` + Every method returns a `ModelRouterStatus` matching what the server + stored: the wrap under `base_route_name` `base`, the named route, the pin, + the return to `auto` and the explicit selection. The unknown route and + the unknown profile raise `httpx.HTTPStatusError` 400 and 404 (the SDK + logs and re-raises), and the server keeps the `qa-f09-route-a` pin. +- **TypeScript model-router methods (`F09.ts-client-model-router`).** + `ConversationClient` and `ConversationManager` from the built client + (rebuilt when `clients/typescript/src` is newer than `dist`), on a fresh + plain conversation `RTS`. + + Requires: `F09.model-router-status` + ```sh + test -d clients/typescript/node_modules/typescript || (cd clients/typescript && npm ci --silent) + if [ ! -f clients/typescript/dist/index.js ] || \ + [ -n "$(find clients/typescript/src clients/typescript/package.json -newer clients/typescript/dist/index.js -print -quit)" ]; then + (cd clients/typescript && npm run build --silent) + fi + RTS=$(control-agent-server conversation start --no-autotitle --body-json '{"agent": {"kind": "Agent", "tools": [], "llm": {"model": "openai/qa-f09-route-ts", "api_key": "sk-qa-f09-route-ts", "base_url": "http://127.0.0.1:9/v1"}}}' --print-id) + F="$AGENT_SERVER_VERIFY_RUN/fixtures" + mkdir -p "$F" + cat > "$F/qa-f09-ts-router.mjs" <<'JS' + import assert from 'node:assert/strict'; + import { resolve } from 'node:path'; + import { pathToFileURL } from 'node:url'; + + const dist = (name) => import(pathToFileURL(resolve('clients/typescript/dist', name)).href); + const { ConversationClient } = await dist('clients.js'); + const { ConversationManager } = await dist('index.js'); + const [cid] = process.argv.slice(2); + const host = process.env.AGENT_SERVER_URL; + const apiKey = process.env.SESSION_API_KEY; + const client = new ConversationClient({ host, apiKey }); + const manager = new ConversationManager({ host, apiKey }); + let status = await client.getModelRouter(cid); + assert.equal(status.enabled, false); + assert.deepEqual(status.routes, []); + status = await client.addModelRouterRoute(cid, 'qa-f09-route-a'); + assert.equal(status.enabled, true); + assert.deepEqual(status.routes.map((r) => r.name), ['default', 'qa-f09-route-a']); + status = await manager.addModelRouterRoute(cid, 'qa-f09-route-b', 'reviewer'); + assert.deepEqual(status.routes.map((r) => r.name), ['default', 'qa-f09-route-a', 'reviewer']); + status = await client.switchModelRouter(cid, 'reviewer'); + assert.deepEqual(status.selection, { mode: 'manual', manual_route_name: 'reviewer' }); + status = await manager.resumeAutoModelRouting(cid); + assert.equal(status.selection.mode, 'auto'); + status = await manager.switchModelRouter(cid, 'qa-f09-route-a'); + assert.equal(status.selection.manual_route_name, 'qa-f09-route-a'); + status = await client.resumeAutoModelRouting(cid); + assert.equal(status.selection.mode, 'auto'); + await client.updateModelRouter(cid, { mode: 'manual', manual_route_name: 'reviewer' }); + assert.equal((await manager.getModelRouter(cid)).selection.manual_route_name, 'reviewer'); + await assert.rejects(client.switchModelRouter(cid, 'qa-f09-nope'), (e) => e.status === 400); + await assert.rejects(client.addModelRouterRoute(cid, 'qa-f09-nope'), (e) => e.status === 404); + await assert.rejects(client.addModelRouterRoute(cid, 'qa-f09-route-a'), (e) => e.status === 400); + await assert.rejects(new ConversationClient({ host }).getModelRouter(cid), (e) => e.status === 401); + assert.equal((await client.getModelRouter(cid)).routes.length, 3); + console.log('TS_MODEL_ROUTER_OK'); + JS + control-agent-server exec --expect-output TS_MODEL_ROUTER_OK --save F09.ts-client-model-router/program -- node "$F/qa-f09-ts-router.mjs" "$RTS" + control-agent-server api GET "/api/conversations/$RTS" --quiet --check agent.llm.router_name eq base_router \ + --check agent.llm.selection.manual_route_name eq reviewer --check agent.llm.llms_for_routing.reviewer.model eq openai/qa-f09-route-b + ``` + Every call resolves with the server's `ModelRouterStatus`: the wrap + (`default` plus the profile's route), the named route from + `ConversationManager`, pins and returns to `auto` through both classes, + and `updateModelRouter` with an explicit selection; the unknown route + rejects with an `HttpError` 400, the unknown profile 404, the duplicate + route 400 and a keyless client 401, and `GET` shows the router pinned to + `reviewer`. (`RemoteConversation`'s methods of the same names post the + same bodies to the same routes.) +- **Turns on a pinned route (`F09.model-router-run`).** A conversation `RT` + from the saved settings runs one turn on `deepseek-flash`; then the + `deepseek-flash` profile is added as route `flash`, pinned, and a turn + runs; then routing returns to `auto` and a turn runs. The block re-saves + the DeepSeek preset (it is idempotent) so the bullet runs on its own. + + Needs: `llm` + ```sh + control-agent-server llm preset deepseek + RT=$(control-agent-server conversation start --tools none --no-autotitle --prompt 'Reply with the single word: ok' --wait --until finished --timeout 240 --print-id) + DEF1=$(control-agent-server api GET "/api/conversations/$RT" --field stats.usage_to_metrics.default.accumulated_token_usage.prompt_tokens) + test "$DEF1" -gt 0 + control-agent-server api POST "/api/conversations/$RT/model_router/routes" --json '{"profile_name": "deepseek-flash", "route_name": "flash"}' \ + --expect 200 --check routes.0.name eq default --check routes.1.name eq flash + control-agent-server api PATCH "/api/conversations/$RT/model_router" --json '{"selection": {"mode": "manual", "manual_route_name": "flash"}}' \ + --expect 200 --check last_route_name eq null + control-agent-server conversation send "$RT" --text 'Reply with the single word: ok' --wait --until finished,error --timeout 240 + control-agent-server api GET "/api/conversations/$RT" --until-ok 60 --quiet --check execution_status eq finished \ + --check 'stats.usage_to_metrics.profile:deepseek-flash.accumulated_token_usage.prompt_tokens' gt 0 \ + --check stats.usage_to_metrics.default.accumulated_token_usage.prompt_tokens eq "$DEF1" --save F09.model-router-run/manual-stats + control-agent-server api GET "/api/conversations/$RT/model_router" --check last_route_name eq flash --check selection.manual_route_name eq flash \ + --save F09.model-router-run/manual-status + FL1=$(control-agent-server api GET "/api/conversations/$RT" --field 'stats.usage_to_metrics.profile:deepseek-flash.accumulated_token_usage.prompt_tokens') + control-agent-server api PATCH "/api/conversations/$RT/model_router" --json '{"selection": {"mode": "auto"}}' \ + --expect 200 --check selection.mode eq auto --check last_route_name eq flash + control-agent-server conversation send "$RT" --text 'Reply with the single word: ok' --wait --until finished,error --timeout 240 + control-agent-server api GET "/api/conversations/$RT" --until-ok 60 --quiet --check execution_status eq finished \ + --check stats.usage_to_metrics.default.accumulated_token_usage.prompt_tokens gt "$DEF1" \ + --check 'stats.usage_to_metrics.profile:deepseek-flash.accumulated_token_usage.prompt_tokens' eq "$FL1" --save F09.model-router-run/auto-stats + control-agent-server api GET "/api/conversations/$RT/model_router" --check last_route_name eq default --check selection.mode eq auto + ``` + The pinned turn finishes on `flash`: its tokens are recorded under + `profile:deepseek-flash` while `default` does not grow, and + `last_route_name` becomes `flash` (only a call sets it; a `PATCH` keeps + the last one). After the return to `auto`, the base router routes to its + first route: the next turn's tokens go to `default`, `flash` does not + grow, and `last_route_name` is `default`. +- **A route pinned before the first run (`F09.model-router-before-first-run`), known bug #5639.** + Start from a placeholder agent whose model is unreachable, add the + `deepseek-flash` profile as route `flash` and pin it before anything + ran, then run. + + Requires: `F09.model-router-run` + Needs: `llm` + ```sh + RB=$(control-agent-server conversation start --placeholder-agent --tools none --no-autotitle --print-id) + control-agent-server api POST "/api/conversations/$RB/model_router/routes" --json '{"profile_name": "deepseek-flash", "route_name": "flash"}' --expect 200 + control-agent-server api PATCH "/api/conversations/$RB/model_router" --json '{"selection": {"mode": "manual", "manual_route_name": "flash"}}' --expect 200 + control-agent-server conversation send "$RB" --text 'Reply with the single word: ok' --wait --until finished,error --timeout 240 + control-agent-server api GET "/api/conversations/$RB" --quiet --check execution_status eq finished + control-agent-server api GET "/api/conversations/$RB/model_router" --quiet --check last_route_name eq flash + control-agent-server api GET "/api/conversations/$RB" --until-ok 20 --quiet \ + --check 'stats.usage_to_metrics.profile:deepseek-flash.accumulated_token_usage.prompt_tokens' gt 0 --save F09.model-router-before-first-run/stats # bug: status eq 200 && checks.0.actual eq null + ``` + The run finishes and `last_route_name` is `flash`, which only the pinned + route can do (the placeholder points at a closed port). Expected: its + tokens are recorded under `profile:deepseek-flash`, as they are when the + route is added after the first message (`F09.model-router-run`). Today + `stats.usage_to_metrics` is `{}`: the route add and the pin register the + LLM before the stats subscribe on the first message + ([#5639](https://github.com/OpenHands/software-agent-sdk/issues/5639)). +- **Restart (`F09.model-router-restart`).** Restart the server and read the + router conversations back. + + Requires: `F09.model-router-select-errors`, `F09.model-router-switch-replaces` + ```sh + control-agent-server restart + control-agent-server api GET "/api/conversations/$RQ/model_router" --check enabled eq true --check selection.mode eq manual \ + --check selection.manual_route_name eq qa-f09-route-a --check routes len-eq 3 --check routes.2.name eq reviewer --save F09.model-router-restart/rq + control-agent-server api GET "/api/conversations/$RQ" --quiet --check agent.llm.router_name eq base_router \ + --check agent.llm.llms_for_routing.reviewer.usage_id eq profile:qa-f09-route-b + control-agent-server api PATCH "/api/conversations/$RQ/model_router" --json '{"selection": {"mode": "manual", "manual_route_name": "reviewer"}}' \ + --expect 200 --check selection.manual_route_name eq reviewer + control-agent-server api GET "/api/conversations/$RM/model_router" --quiet --check enabled eq false + control-agent-server api GET "/api/conversations/$RQ2/model_router" --quiet --check routes len-eq 2 --check routes.1.name eq qa-f09-route-b + ``` + `RQ` comes back as a `base_router` with its three routes and the + `qa-f09-route-a` pin, and accepts a new pin; `RM`, whose router + `switch_profile` replaced, is still plain, and `RQ2` still has the router + built after its `switch_llm`. ## Gotchas @@ -961,6 +1398,19 @@ Preconditions: for LLMs registered after the agent initialized (`F09.switch-llm-before-first-run`); send one message (or create with an initial message) before switching when costs matter. +- The model-router routes act on the conversation's agent LLM only when it + is a `RouterLLM`: `GET .../model_router` answers `enabled: false` for any + other LLM, `PATCH` is 400 there, and only `POST .../model_router/routes` + turns a plain LLM into a router. The router's own `model` is a placeholder + (`base_router`, `multimodal_router`), and its usage is never recorded under + the router: each route keeps its LLM's `usage_id` (`default` for a wrapped + agent LLM, `profile:<name>` for an added profile). A route is a copy of the + profile at the time it was added (later profile edits do not reach it), + and adding the same profile twice under two route names reuses the + conversation's cached `profile:<name>` LLM. An empty `route_name` falls back + to the profile name. A route added or a pin set before the first message + is not counted in `stats` (`F09.model-router-before-first-run`), like any + LLM registered then. - The agent is initialized by the first message (`conversation send --no-run` is enough) or run, not by the create; ACP agents only at the first run, which is when the subprocess starts. diff --git a/.agents/skills/verify-agent-server/references/feature-map/F19-mcp.md b/.agents/skills/verify-agent-server/references/feature-map/F19-mcp.md index d0293488db2..c78782be5b3 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F19-mcp.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F19-mcp.md @@ -62,15 +62,15 @@ Routes: `POST /api/mcp/test`, `POST /api/mcp/oauth/start`, - `F19.oauth-inline-writeback`: an OAuth server passed inline on a conversation's agent (not in settings) whose access token has expired is refreshed when the conversation connects, keeps its tools, and the rotated tokens are posted with the run's session key to the webhook's `/mcp-oauth-state`, not stored in settings. - `F19.oauth-stored-state`: the returned `oauth_state` used as `auth.state` connects without consent (test, and start without `authorization_url`), and stored through the settings route it is redacted, encrypted on disk and decrypted to the real token. - `F19.disabled-server`: a conversation started from the settings exposes the tools of enabled servers (including the OAuth server from its stored tokens) and neither exposes nor spawns a disabled server. -- `F19.conversation-tool`: a model-driven conversation calls the configured stdio server's `qa_echo` tool, the `ObservationEvent` (REST and WebSocket) carries `QA_MCP_ECHO:hi`, and deleting the conversation stops the stdio child. +- `F19.conversation-tool`: a model-driven conversation calls the configured stdio server's `qa_echo` tool, the `ObservationEvent` (REST and WebSocket) carries `QA_MCP_ECHO:hi`, and deleting the conversation stops the stdio child; an embedded resource's text and a structured-only result reach the observation (known bug #6993: both are dropped), and a single server that advertises `terminal` next to the built-in terminal gets it as `<server>_terminal` while both run (known bug #6992: a server that joins after the startup budget loses all its tools, so its other tools are `not found`; one ready within the budget makes the first message 500 `Duplicate tool names found: {'terminal'}`). - `F19.oauth-refresh-writeback`: when a stored OAuth server's access token has expired, the next conversation refreshes it, keeps the server's tools, and writes the rotated token and new expiry back into the settings (encrypted on disk) without posting them to the webhook. - `F19.conversation-secrets-at-rest`: a conversation started from the settings persists its agent's MCP credentials (a disabled server's `env`, OAuth tokens) encrypted, and no MCP credential this family used appears in plaintext under the conversations directory or in the server log. - `F19.restart`: stored MCP servers and their credentials survive a restart (plaintext GET and the SDK's `RemoteWorkspace.get_mcp_config()`), while in-memory OAuth jobs are gone (404). - `F19.conversation-mcp-add`: `POST /api/conversations/{conversation_id}/mcp` on a conversation whose agent is initialized starts the server before it answers 200 `{"success": true}`, `GET /api/conversations/{conversation_id}` and `base_state.json` show it in `agent.mcp_config`, the model's next request offers its tools as `<server>_<tool>`, a call returns the tool's result, deleting the conversation stops the child, and `/server_info` advertises `conversation_mcp_install_v1` and `conversation_configuration_additions_v1`. - `F19.conversation-mcp-add-before-init`: on a conversation without a first message the route only records the server (no process, fast answer), with `env` values encrypted by the settings API decrypted for the child and stored encrypted in `base_state.json`; the first message connects it like any configured server. - `F19.conversation-mcp-add-errors`: an unknown conversation is 404, a server name the agent already has and an SSE server are 400 with the reason, a server without `command`/`url`, an unknown field and a non-UUID id are 422, none of them changes the agent's `mcp_config`, and a disabled server is stored without being spawned. -- `F19.conversation-mcp-add-start-failure`: a server that cannot start (a command that exits at once) is refused with the route's 400 and is not added (known bug [#6926](https://github.com/OpenHands/software-agent-sdk/issues/6926), fix in [#6928](https://github.com/OpenHands/software-agent-sdk/pull/6928): 500 Internal Server Error with `exception: Connection closed`). -- `F19.conversation-mcp-add-restart`: a server added at runtime is still in the conversation's agent after a restart, and the tools the agent already called keep working under the same names (known bug [#6927](https://github.com/OpenHands/software-agent-sdk/issues/6927), fix in [#6929](https://github.com/OpenHands/software-agent-sdk/pull/6929): with one server the tools come back unprefixed, so `qa-f19-rs_qa_echo` is `Tool ... not found`). +- `F19.conversation-mcp-add-start-failure`: a server that cannot start (a command that exits at once) is refused with the route's 400 naming the server and the cause (`Could not start MCP server <name>: Connection closed`, since [#6928](https://github.com/OpenHands/software-agent-sdk/pull/6928)) and is not added. +- `F19.conversation-mcp-add-restart`: a server added at runtime is still in the conversation's agent after a restart, and the tools the agent already called keep working under the same names (known bug [#6927](https://github.com/OpenHands/software-agent-sdk/issues/6927), closed 2026-11-18 with no fix merged; fix proposed in [#6929](https://github.com/OpenHands/software-agent-sdk/pull/6929): with one server the tools come back unprefixed, so `qa-f19-rs_qa_echo` is `Tool ... not found`). - `F19.conversation-mcp-slow-start`: a conversation whose stdio server delays its handshake by 8 s is created within a few seconds, the server is spawned, and the agent's first model request waits for it and offers its tools; the agent's call returns its result. - `F19.conversation-mcp-late-failure`: a server that fails after the startup budget costs only its own tools: the conversation starts at once, the first step waits for the failure, the other server's tools are offered, the run finishes and the log names the failed server. @@ -140,10 +140,13 @@ Preconditions: `F19.oauth-refresh-writeback` read it), `doctor` ok, `$DEEPSEEK_API_KEY` set (only `F19.conversation-tool` uses the model), and `jq`, `curl` and `pgrep` on `PATH`. -- The CLI's stdio fixture (`fixture mcp-server`, tool `qa_echo`) plus three +- The CLI's stdio fixture (`fixture mcp-server`, tool `qa_echo`) plus four recipe-local fixtures written below (no CLI verb exists for them): a stdio server with `qa_env`, `qa_sleep` and `qa_fail` whose first argument delays - the MCP handshake, an HTTP FastMCP server with `read_subject` and `qa_wait` + the MCP handshake, a stdio server whose results are not plain text + (`qa_resource`, one embedded resource; `qa_structured_only`, only + structured content) and that also advertises `terminal` and `qa_sibling`, + an HTTP FastMCP server with `read_subject` and `qa_wait` in three modes (`none`, `bearer` with the static token `qa-f19-bearer`, and `oauth` with FastMCP's `InMemoryOAuthProvider`, which approves every authorization request), and an @@ -192,6 +195,49 @@ Preconditions: raise ValueError("QA_MCP_FAIL") + if __name__ == "__main__": + mcp.run() + EOF + cat > "$F19/qa_f19_payload.py" <<'EOF' + """F19 stdio MCP fixture: non-text results, and a tool named like the built-in terminal; argv[1] delays the MCP handshake.""" + import sys + import time + + from fastmcp import FastMCP + from fastmcp.tools import ToolResult + from mcp import types + + if len(sys.argv) > 1: + time.sleep(float(sys.argv[1])) + + mcp = FastMCP("qa-f19-payload") + + + @mcp.tool(output_schema=None) + def qa_resource() -> ToolResult: + """Return one embedded text resource.""" + text = types.TextResourceContents(uri="file:///qa-f19/payload.json", mimeType="application/json", text='{"payload": "QA_F19_RESOURCE_TEXT"}') + return ToolResult(content=[types.EmbeddedResource(type="resource", resource=text)]) + + + @mcp.tool(output_schema=None) + def qa_structured_only() -> ToolResult: + """Return structured content without content blocks.""" + return ToolResult(content=[], structured_content={"payload": "QA_F19_STRUCTURED"}) + + + @mcp.tool(name="terminal") + def qa_shadow_terminal(command: str) -> str: + """An MCP tool with the built-in terminal tool's name.""" + return "QA_F19_MCP_TERMINAL:" + command + + + @mcp.tool() + def qa_sibling() -> str: + """A tool next to the shadowing one.""" + return "QA_F19_SIBLING" + + if __name__ == "__main__": mcp.run() EOF @@ -274,6 +320,7 @@ Preconditions: print("SDK_MCP_ENV_OK=" + str(token == "qa-f19-env-secret" and server.enabled is False)) EOF TOOLS_SRV=$(jq -nc --arg py "$PY" --arg s "$F19/qa_f19_tools.py" '{command: $py, args: [$s]}') + PAYLOAD_SRV=$(jq -nc --arg py "$PY" --arg s "$F19/qa_f19_payload.py" '{command: $py, args: [$s]}') RUN_URL=$(control-agent-server config | jq -r .url) free_port() { "$PY" -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])'; } BEARER_PORT=$(free_port) @@ -988,6 +1035,56 @@ Preconditions: `qa-f19_qa_echo`) carries `QA_MCP_ECHO:hi` in the REST event list and as a WebSocket frame, the stdio child exists during the conversation and is gone after the delete. +- **Non-text tool results (`F19.conversation-tool`), known bug #6993.** + The payload fixture as the agent's only server; an `llm-stub` model calls + `qa_resource`, then `qa_structured_only`, and finishes. + ```sh + PL_STUB=$(control-agent-server fixture llm-stub --name qa-f19-pl-stub --step 'tool:qa_resource:{}' --step 'tool:qa_structured_only:{}' --step 'tool:finish:{"message":"QA_F19_PL_DONE"}' --print-path) + PL_AGENT=$(jq -nc --arg b "$PL_STUB/v1" --argjson s "$PAYLOAD_SRV" '{llm: {model: "openai/qa-stub", api_key: "qa-stub-key", base_url: $b, num_retries: 0}, tools: [], mcp_config: {"qa-f19-payload": $s}}') + PLCID=$(control-agent-server conversation start --body-json "$(jq -nc --argjson a "$PL_AGENT" '{agent: $a}')" --no-autotitle --prompt 'Read the payloads.' --wait --timeout 90 --print-id) + control-agent-server conversation events "$PLCID" --kinds ObservationEvent --contains "Tool 'qa_structured_only' executed" --expect-kind ObservationEvent + control-agent-server conversation events "$PLCID" --kinds ObservationEvent --contains QA_F19_STRUCTURED --expect-kind ObservationEvent --save F19.conversation-tool/structured-only # bug: matching eq 0 && kinds.ObservationEvent eq 3 + control-agent-server conversation events "$PLCID" --kinds ObservationEvent --contains QA_F19_RESOURCE_TEXT --expect-kind ObservationEvent --save F19.conversation-tool/resource # bug: matching eq 0 && kinds.ObservationEvent eq 3 + control-agent-server api DELETE "/api/conversations/$PLCID" --expect 200 --quiet + ``` + Both calls ran (the `qa_structured_only` observation exists), and each + observation should carry the tool's payload (both checks are marked; the + first one stops the bullet today). Today the structured-only + call's observation is only `[Tool 'qa_structured_only' executed.]`, and the + resource call's only `[Tool 'qa_resource' executed.]`: the SDK's + `MCPToolObservation.from_call_tool_result` keeps text and image blocks and + never reads `structured_content`, so neither the model nor an event + consumer sees the payload. +- **A server tool named like a built-in (`F19.conversation-tool`), known bug #6992.** + The agent has the built-in `terminal`, and the payload fixture (which also + advertises `terminal`) is its only MCP server, with its handshake delayed + by 3 s so it always joins after the 1 s startup budget (see Gotchas): the + path a slow-starting server takes, whatever the machine's load. The + `llm-stub` model runs the built-in terminal, then `qa_sibling`. + ```sh + SH_STUB=$(control-agent-server fixture llm-stub --name qa-f19-sh-stub --step 'tool:terminal:{"command":"echo QA_F19_BUILTIN_TERMINAL"}' --step 'tool:qa_sibling:{}' --step 'tool:finish:{"message":"QA_F19_SH_DONE"}' --print-path) + SH_AGENT=$(jq -nc --arg b "$SH_STUB/v1" --argjson s "$PAYLOAD_SRV" '{llm: {model: "openai/qa-stub", api_key: "qa-stub-key", base_url: $b, num_retries: 0}, tools: [{name: "terminal"}], mcp_config: {"qa-f19-shadow": ($s + {args: ($s.args + ["3"])})}}') + SHCID=$(control-agent-server conversation start --body-json "$(jq -nc --argjson a "$SH_AGENT" '{agent: $a}')" --no-autotitle --print-id) + trap 'control-agent-server api DELETE "/api/conversations/$SHCID" --expect 200,404 >/dev/null || true' EXIT + control-agent-server api POST "/api/conversations/$SHCID/events" --json '{"role": "user", "content": [{"type": "text", "text": "Use both tools."}], "run": true}' \ + --expect 200 --save F19.conversation-tool/shadow-first-message + control-agent-server conversation wait "$SHCID" --until finished --timeout 90 + control-agent-server conversation events "$SHCID" --kinds ObservationEvent --contains QA_F19_BUILTIN_TERMINAL --expect-kind ObservationEvent + control-agent-server conversation events "$SHCID" --kinds ObservationEvent --contains QA_F19_SIBLING --expect-kind ObservationEvent \ + --save F19.conversation-tool/shadow-sibling # bug: matching eq 0 && kinds.AgentErrorEvent eq 1 + control-agent-server sink read --name qa-f19-sh-stub --contains '"name": "qa-f19-shadow_terminal"' --expect-min 1 + ``` + The built-in terminal runs (the control). The server's tools should join + as `qa-f19-shadow_terminal` and `qa_sibling`, so the sibling call returns + `QA_F19_SIBLING`. Today the late join is refused as a whole (the log says + `Could not add MCP tools of qa-f19-shadow: Dynamically advertised MCP + tools conflict with existing runtime tools: ['terminal']`), the model is + offered no tool of that server, and the sibling call ends in an + `AgentErrorEvent` `Tool 'qa_sibling' not found`; the run still finishes. + A server that is ready within the budget hits the same collision during + initialization instead, and the first message is 500 `Duplicate tool + names found: {'terminal'}` with the conversation left `idle`; that + variant depends on start-up speed, so the recipe does not drive it. - **Stored OAuth token refresh (`F19.oauth-refresh-writeback`).** Expire the stored `qa-oauth` token with a sparse patch (client-side expiry only), then let a new conversation connect. @@ -1153,18 +1250,19 @@ Preconditions: Each refusal leaves the agent's `mcp_config` as it was; the disabled server is stored with `enabled: false` and its command never runs; deleting the conversation stops the child the add started. -- **A server that cannot start (`F19.conversation-mcp-add-start-failure`), known bug #6926.** +- **A server that cannot start (`F19.conversation-mcp-add-start-failure`).** The command exits at once, so the MCP connection closes during the - handshake. The route's documented refusal is 400 (`Invalid MCP server - update`); today the exception escapes as a 500. + handshake. The SDK raises `MCPServerStartError` and the route answers its + documented refusal, 400 (`Invalid MCP server update`). ```sh SF_AGENT='{"llm": {"model": "openai/qa-offline", "api_key": "qa-offline-key", "base_url": "http://127.0.0.1:9/v1", "num_retries": 0}, "tools": []}' FCID=$(control-agent-server conversation start --body-json "$(jq -nc --argjson a "$SF_AGENT" '{agent: $a}')" --no-autotitle --prompt 'Never run.' --no-run --print-id) - control-agent-server api POST "/api/conversations/$FCID/mcp" --json '{"mcp_servers":{"qa-f19-dies":{"command":"/bin/true"}}}' --expect 400 --save F19.conversation-mcp-add-start-failure/add # bug: status eq 500 && response.body.exception eq 'Connection closed' + control-agent-server api POST "/api/conversations/$FCID/mcp" --json '{"mcp_servers":{"qa-f19-dies":{"command":"/bin/true"}}}' --expect 400 \ + --check detail eq 'Could not start MCP server qa-f19-dies: Connection closed' --save F19.conversation-mcp-add-start-failure/add control-agent-server api GET "/api/conversations/$FCID" --max-chars 300 --check agent.mcp_config.qa-f19-dies missing --save F19.conversation-mcp-add-start-failure/after ``` - The server is not added either way (`agent.mcp_config` keeps no - `qa-f19-dies`); only the status is wrong. + The 400's `detail` names the server and the cause, and the server is not + added (`agent.mcp_config` keeps no `qa-f19-dies`). - **Added servers after a restart (`F19.conversation-mcp-add-restart`), known bug #6927.** The stub calls the added tool, finishes, and after a restart and a new message calls the same tool again. diff --git a/.agents/skills/verify-agent-server/references/feature-map/F21-agent-profiles.md b/.agents/skills/verify-agent-server/references/feature-map/F21-agent-profiles.md index cbbafa7a738..461dae47f1a 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F21-agent-profiles.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F21-agent-profiles.md @@ -60,6 +60,7 @@ Routes: `GET /api/agent-profiles`, `GET /api/agent-profiles/{name}`, - `F21.dangling-llm-report`: how materialize reports a dangling LLM reference must match what the server's seed code documents: the documented `dangling_llm_profile_ref` field (the one the launch's 422 carries) is in the materialize body, or the documentation stops promising it. - `F21.ts-payload-types`: the TypeScript client's exported `AgentProfileSummary`, `AgentProfileDiagnostics`, `OpenHandsAgentProfile` and `ACPAgentProfile` types accept the payloads the server sends, as its generated contract does: every field the server sends is declared and every required field is sent. - `F21.json-suffix-identity`: saving a name that ends in `.json` twice must keep one `id` and bump `revision`, like any other name, and deleting it while active must move the pointer off it (to `default`) (or the name must be refused with 422 and nothing stored). +- `F21.sandbox-metadata`: both profile kinds take an optional `sandbox: {"spec_id": "..."}` (a sandbox spec the provisioning host offers): it is stored and served back by GET, absent means `null`, an empty or missing `spec_id`, an extra key or a non-object is 422 and saves nothing; it never enters the resolved launch (materialize gives the same `resolved_settings` as the same profile without it), and a conversation still starts from such a profile on a server that does not provision sandboxes. ## How to get to it (agent POV) @@ -74,7 +75,9 @@ Routes: `GET /api/agent-profiles`, `GET /api/agent-profiles/{name}`, `POST /api/agent-profiles/{name}/rename` (`{"new_name": ...}`), `POST /api/agent-profiles/{profile_id}/activate` (the uuid, not the name), `POST /api/agent-profiles/{name}/materialize` (no body or `{}` for the - stored profile, `{"profile": {...}}` for a draft). Every recipe below + stored profile, `{"profile": {...}}` for a draft). A profile body may + carry `sandbox: {"spec_id": ...}` (`ProfileSandboxSettings`, the + same name in the TypeScript client). Every recipe below drives these routes directly. - REST (other families, used here as second views or setup): `GET /api/settings` and `PATCH /api/settings` (`active_agent_profile_id`, @@ -700,6 +703,60 @@ printf -- '---\nname: qa-f21-skill\ndescription: QA F21 user skill\n---\nQA_F21_ the text), and the stored profile is unchanged. An unknown key and a 32769-character value (the limit is 32768) are 422, and the conversation count does not move. +- **Sandbox metadata (`F21.sandbox-metadata`).** Self-contained: an LLM + profile of its own (no key, never called), an OpenHands and an ACP + profile with a sandbox spec, a twin without one, the refused shapes, two + previews and a conversation start; everything it creates is deleted at + the end. + ```sh + SBX_SPEC=registry.example/agent-server:qa-f21 + control-agent-server llm set --profile qa-f21-sbx-llm --model openai/qa-f21-sbx --base-url http://127.0.0.1:9/v1 \ + --no-api-key --no-validate --no-activate --num-retries 0 + control-agent-server api POST /api/agent-profiles/qa-f21-sbx --expect 201 \ + --json "{\"llm_profile_ref\": \"qa-f21-sbx-llm\", \"tools\": [], \"sandbox\": {\"spec_id\": \"$SBX_SPEC\"}}" + SBX=$(control-agent-server api GET /api/agent-profiles/qa-f21-sbx --check profile.sandbox.spec_id eq "$SBX_SPEC" \ + --save F21.sandbox-metadata/get --field profile.id) + control-agent-server state cat home/.openhands/agent-profiles/qa-f21-sbx.json --check sandbox.spec_id eq "$SBX_SPEC" + control-agent-server api POST /api/agent-profiles/qa-f21-sbx-acp --expect 201 \ + --json "{\"agent_kind\": \"acp\", \"acp_server\": \"codex\", \"sandbox\": {\"spec_id\": \"$SBX_SPEC\"}}" + control-agent-server api GET /api/agent-profiles/qa-f21-sbx-acp --check profile.agent_kind eq acp \ + --check profile.sandbox.spec_id eq "$SBX_SPEC" --save F21.sandbox-metadata/acp + control-agent-server api POST /api/agent-profiles/qa-f21-nosbx --expect 201 --json '{"llm_profile_ref": "qa-f21-sbx-llm", "tools": []}' + control-agent-server api GET /api/agent-profiles/qa-f21-nosbx --check profile.sandbox eq null + control-agent-server api POST /api/agent-profiles/qa-f21-sbx-bad --json '{"llm_profile_ref": "qa-f21-sbx-llm", "sandbox": {"spec_id": ""}}' \ + --expect 422 --check detail.0.type eq string_too_short --check detail.0.loc contains spec_id --save F21.sandbox-metadata/empty-spec + control-agent-server api POST /api/agent-profiles/qa-f21-sbx-bad --json '{"llm_profile_ref": "qa-f21-sbx-llm", "sandbox": {}}' \ + --expect 422 --check detail.0.type eq missing --check detail.0.loc contains spec_id + control-agent-server api POST /api/agent-profiles/qa-f21-sbx-bad --json '{"llm_profile_ref": "qa-f21-sbx-llm", "sandbox": {"spec_id": "img", "size": "large"}}' \ + --expect 422 --check detail.0.type eq extra_forbidden --check detail.0.loc contains size + control-agent-server api POST /api/agent-profiles/qa-f21-sbx-bad --json '{"llm_profile_ref": "qa-f21-sbx-llm", "sandbox": "img"}' \ + --expect 422 --check detail.0.type eq model_type --check detail.0.loc contains sandbox + control-agent-server api GET /api/agent-profiles/qa-f21-sbx-bad --expect 404 + control-agent-server state ls 'home/.openhands/agent-profiles/qa-f21-sbx-bad*' --expect-count 0 + SBX_DIR="$AGENT_SERVER_VERIFY_RUN/fixtures/qa-f21-sbx" + mkdir -p "$SBX_DIR" + control-agent-server api POST /api/agent-profiles/qa-f21-sbx/materialize --expect 200 --quiet --check valid eq true \ + --check resolved_settings.llm.model eq openai/qa-f21-sbx --raw-out "$SBX_DIR/with.json" --save F21.sandbox-metadata/materialize + control-agent-server api POST /api/agent-profiles/qa-f21-nosbx/materialize --expect 200 --quiet --check valid eq true \ + --raw-out "$SBX_DIR/without.json" + jq -e -s --arg spec "$SBX_SPEC" '[.[] | .resolved_settings | del(.agent_context.current_datetime)] | (.[0] == .[1]) and ((.[0] | tostring | contains($spec)) | not)' \ + "$SBX_DIR/with.json" "$SBX_DIR/without.json" + SBX_CONV=$(control-agent-server api POST /api/conversations \ + --json "{\"workspace\": {\"working_dir\": \"$SBX_DIR\"}, \"agent_profile_id\": \"$SBX\", \"autotitle\": false}" \ + --expect 201 --check launched_agent_profile.agent_profile_id eq "$SBX" --check agent.llm.model eq openai/qa-f21-sbx \ + --check execution_status eq idle --save F21.sandbox-metadata/conversation --field id) + control-agent-server api DELETE "/api/conversations/$SBX_CONV" --expect 200 + control-agent-server api DELETE /api/agent-profiles/qa-f21-sbx --expect 200 + control-agent-server api DELETE /api/agent-profiles/qa-f21-sbx-acp --expect 200 + control-agent-server api DELETE /api/agent-profiles/qa-f21-nosbx --expect 200 + control-agent-server api DELETE /api/profiles/qa-f21-sbx-llm --expect 200 + ``` + GET and the stored file carry the spec id on both kinds and `null` on + the twin; the four bad shapes are 422 at `sandbox` or `spec_id` and leave + no file. The two previews are valid and their `resolved_settings` are + identical (the preview's clock aside) and never mention the spec; the + conversation starts `idle` from the profile on this local server, which + provisions no sandbox. - **TypeScript client (`F21.ts-client`).** Every `AgentProfilesClient` method from the built client (built here when missing), plus the same client reached through `AgentServerClient.agentProfiles`. @@ -950,11 +1007,14 @@ printf -- '---\nname: qa-f21-skill\ndescription: QA F21 user skill\n---\nQA_F21_ SUMMARY=$(jq -c '.profiles[0] | with_entries(.value = true)' "$D/list.json") OPENHANDS=$(jq -c '.profile | with_entries(.value = true)' "$D/openhands.json") ACPKEYS=$(jq -c '.profile | with_entries(.value = true)' "$D/acp.json") + test "$(jq -c '[.profile | has("sandbox")]' "$D/openhands.json" "$D/acp.json" | paste -sd,)" = '[true],[true]' + CONTRACT_OPENHANDS=$(jq -c '.profile | del(.sandbox) | with_entries(.value = true)' "$D/openhands.json") + CONTRACT_ACP=$(jq -c '.profile | del(.sandbox) | with_entries(.value = true)' "$D/acp.json") DIAG=$(jq -c 'with_entries(.value = true)' "$D/diagnostics.json") DIST="$PWD/clients/typescript/dist" printf '%s\n' "import type { AgentProfileInfo, OpenHandsAgentProfile, AcpAgentProfile, AgentProfileDiagnostics } from '$DIST/generated/agent-server-schema.js';" \ "import type { Shape } from './shape.js';" "export const summary: Shape<AgentProfileInfo> = $SUMMARY;" \ - "export const openhands: Shape<OpenHandsAgentProfile> = $OPENHANDS;" "export const acp: Shape<AcpAgentProfile> = $ACPKEYS;" \ + "export const openhands: Shape<OpenHandsAgentProfile> = $CONTRACT_OPENHANDS;" "export const acp: Shape<AcpAgentProfile> = $CONTRACT_ACP;" \ "export const diagnostics: Shape<AgentProfileDiagnostics> = $DIAG;" > "$D/contract.ts" printf '%s\n' "import type { AgentProfileSummary, OpenHandsAgentProfile, ACPAgentProfile, AgentProfileDiagnostics } from '$DIST/index.js';" \ "import type { Shape } from './shape.js';" "export const summary: Shape<AgentProfileSummary> = $SUMMARY;" \ @@ -972,7 +1032,10 @@ printf -- '---\nname: qa-f21-skill\ndescription: QA F21 user skill\n---\nQA_F21_ `disabled_skills`, `meta_profile_ref` and `enable_classify_and_switch_llm_tool` (and no `skills`, which a save refuses with 422 `extra_forbidden`), and `ACPAgentProfile` declares - `acp_startup_timeout`. The client's unit test + `acp_startup_timeout`. Both profile kinds also carry `sandbox` (`null` + when unset, `F21.sandbox-metadata`), which the exported types declare; + the generated contract is built from the published server schema and may + not list it yet, so `contract.ts` checks every other key. The client's unit test `agent-server-contract-types.test.ts` pins the same key sets against the generated contract. `public_skills` is declared on `OpenHandsAgentProfile` and `ACPAgentProfile`, and `acp_effort` (which the server's ACP profile carries) on both the diff --git a/.agents/skills/verify-agent-server/references/feature-map/F27-hooks-subagents-tools.md b/.agents/skills/verify-agent-server/references/feature-map/F27-hooks-subagents-tools.md index fa88b77259f..0bf5a34510e 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F27-hooks-subagents-tools.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F27-hooks-subagents-tools.md @@ -37,7 +37,7 @@ Routes: `GET /api/tools/`, `GET /api/tools/catalog`, `POST /api/sub-agents`, - `F27.hooks-formats`: a snake_case `hooks.json` keeps its matcher, timeout and `async` flag, a PascalCase key next to it is normalized, a `prompt` hook keeps its prompt, and a matcher left out becomes `*`. - `F27.hooks-absent`: no `project_dir`, a misspelled field (`projectDir`), a missing directory, a directory without `.openhands/hooks.json`, a `hooks.json` only under `.agents/`, and an empty `{}` file all return 200 with `hook_config: null`. - `F27.hooks-invalid`: unparsable JSON, an unknown event (PascalCase or snake_case), both spellings of one event, and a command hook without a command return 200 with `hook_config: null` and log `Failed to load hooks`. -- `F27.hooks-conversation`: the config `POST /api/hooks` returns, passed as `hook_config` to a DeepSeek conversation in that project, runs: a `HookExecutionEvent` with the hook's stdout appears over REST and the WebSocket replay, and the conversation keeps its `hook_config` across a restart. +- `F27.hooks-conversation`: the config `POST /api/hooks` returns, passed as `hook_config` to a DeepSeek conversation in that project, runs: a `HookExecutionEvent` with the hook's stdout appears over REST and the WebSocket replay, and the conversation keeps its `hook_config` across a restart; a command hook sees the conversation's secrets in its environment, as the agent's terminal does (known bug #6966: the hook's environment has none). - `F27.hooks-not-automatic`: a conversation in the same project started without `hook_config` runs no hook, although the project has `hooks.json` and the agent used the terminal. - `F27.hooks-block`: a `PreToolUse` hook that exits 2 stops the agent's terminal call (a `HookExecutionEvent` with `blocked: true`, a `UserRejectObservation` from the hook with its stderr as the reason, no terminal observation, no file) and records the action in `blocked_actions` until the agent consumes it; a `UserPromptSubmit` hook that exits 2 records the queued message in `blocked_messages`, and running finishes at once without a model call. - `F27.subagents-builtin`: without project and user agents (switched off, or a `project_dir` that does not exist) the catalog is exactly the four built-ins (`bash-runner`, `code-explorer`, `general-purpose`, `web-researcher`) with `level: builtin` and `is_builtin: true`, `mcp_servers` mirrors `mcp_config`, and `load_builtin: false` empties it. @@ -531,6 +531,29 @@ Preconditions: echo QA_PROJECT_HOOK`, the command's output in `stdout` and `exit_code: 0` is in the event log and in the WebSocket replay. The conversation still reports the same `hook_config` after the restart. +- **Hooks and the conversation's secrets (`F27.hooks-conversation`), known bug #6966.** + A conversation with the secret `QA_F27_HOOK_SECRET`, a `PreToolUse` hook + on `terminal`, and an `llm-stub` model whose terminal call runs the same + probe; both print `SET` or `UNSET`, never the value. + ```sh + HS_PROBE='test -n "$QA_F27_HOOK_SECRET" && echo QA_F27_%s_SECRET_SET || echo QA_F27_%s_SECRET_UNSET' + HS_TERMINAL_CMD=$(printf "$HS_PROBE" TERMINAL TERMINAL) + HS_HOOK_CMD=$(printf "$HS_PROBE" HOOK HOOK) + HS_STEP=$(jq -nc --arg c "$HS_TERMINAL_CMD" '{command: $c}') + HS_STUB=$(control-agent-server fixture llm-stub --name qa-f27-hs-stub --step "tool:terminal:$HS_STEP" --step 'tool:finish:{"message":"QA_F27_HS_DONE"}' --print-path) + HS_BODY=$(jq -nc --arg b "$HS_STUB/v1" --arg c "$HS_HOOK_CMD" \ + '{agent: {llm: {model: "openai/qa-stub", api_key: "qa-stub-key", base_url: $b, num_retries: 0}, tools: [{name: "terminal"}]}, hook_config: {pre_tool_use: [{matcher: "terminal", hooks: [{command: $c}]}]}}') + HSCID=$(control-agent-server conversation start --body-json "$HS_BODY" --secret QA_F27_HOOK_SECRET=qa-f27-hook-secret-value \ + --no-autotitle --prompt 'Check the secret.' --wait --timeout 90 --print-id) + control-agent-server conversation events "$HSCID" --kinds ObservationEvent --contains QA_F27_TERMINAL_SECRET_SET --expect-kind ObservationEvent + control-agent-server conversation events "$HSCID" --kinds HookExecutionEvent --contains '"stdout": "QA_F27_HOOK_SECRET_SET\n"' \ + --expect-kind HookExecutionEvent --full --save F27.hooks-conversation/hook-secret # bug: matching eq 0 && kinds.HookExecutionEvent eq 1 + ``` + The terminal call sees the secret (`QA_F27_TERMINAL_SECRET_SET`, the + control), and so should the hook. Today the hook's one + `HookExecutionEvent` has `stdout` `QA_F27_HOOK_SECRET_UNSET`: the SDK's + `HookExecutor` builds the hook environment from `sanitized_env()` and + never reads the conversation's secrets. - **Hooks are not loaded implicitly (`F27.hooks-not-automatic`).** The same project and prompt, without `hook_config`. ```sh diff --git a/.agents/skills/verify-agent-server/references/feature-map/F30-openai-gateway.md b/.agents/skills/verify-agent-server/references/feature-map/F30-openai-gateway.md index 6e868a0cd3e..d47e5490edd 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F30-openai-gateway.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F30-openai-gateway.md @@ -49,6 +49,9 @@ Routes: `GET /v1/models`, `POST /v1/chat/completions`, `POST /v1/responses`, `PO - `F30.chat-run-limit-429`: when `max_concurrent_runs` is full a new gateway conversation is refused with 429 and nothing is created; the same request succeeds once a slot is free. - `F30.chat-reuse-run-limit-429`: on a continued conversation a 429 also leaves nothing behind, so a client retry is safe. - `F30.chat-timeout-504`: a run longer than the fixed 120 s answers 504 `Agent run timed out`, and the run keeps going server-side. +- `F30.chat-timeout-no-retry`: the official OpenAI Python client with its default retries sends a run that outlives the gateway's 120 s exactly once: the post-send 504 carries `X-Should-Retry: false` and the conversation id, the client raises `InternalServerError` (504) after one HTTP request, and the conversation holds one user message and one model call. +- `F30.inner-gateway-presend-504`: an inner gateway turn that cannot start before its 120 s deadline (the conversation's direct run is still going) answers 504 `Agent run timed out` with `X-Should-Retry: true` and no conversation header, and saves no message. +- `F30.inner-gateway-no-sender`: the inner gateway route refuses a body with `sender_conversation_id` with 422 `A gateway turn is not sent by another conversation` before running anything. - `F30.responses-basic`: `POST /v1/responses` on a real model returns a completed `response` with one `output_text` message equal to the agent's final answer, echoes `instructions` and `metadata`, and puts the instructions in the system suffix. - `F30.responses-stateless-replay`: every responses call is a new conversation (the conversation header is ignored); replayed items become one user message wrapped in `<message role="...">` markers, and developer items go to the system suffix. - `F30.responses-errors`: `stream`, `store: true` and `previous_response_id` are 400 before anything else is checked, bad input is 400 or 422, an unknown model 404, and nothing is created. @@ -923,6 +926,99 @@ control-agent-server api GET /api/settings --check conversation_settings.confirm The 504 arrives after 115 to 140 s with the reason in `detail`. The run is not cancelled: right after the 504 the conversation is still `running` on its single model call, so the bullet deletes it. +- **The default OpenAI client sends a timed-out turn once (`F30.chat-timeout-no-retry`).** + The same hanging stub as the timeout bullet, behind its own profile, and + a program using `openai` with its default `max_retries` (2); an `httpx` + request hook counts the HTTP requests the client really sends. Two + minutes: the server's own timer is under test. + ```sh + SLOW_SDK=$(control-agent-server fixture llm-stub --name qa-f30-slow-sdk --step hang:170 --print-path) + control-agent-server llm set --profile qa-f30-slow-sdk --model openai/qa-stub --base-url "$SLOW_SDK/v1" \ + --no-api-key --no-validate --no-activate --num-retries 0 --llm-timeout 300 + NR_ID=$(python3 -c 'import uuid; print(uuid.uuid4())') + control-agent-server exec --openai --timeout 600 --expect-output QA_F30_NO_RETRY_OK --save F30.chat-timeout-no-retry/program -- \ + .venv/bin/python - "$NR_ID" <<'PY' + import sys + import time + + import openai + from openai import DefaultHttpxClient, OpenAI + + sent = [] + client = OpenAI(timeout=300, http_client=DefaultHttpxClient(event_hooks={"request": [sent.append]})) + assert client.max_retries == 2, client.max_retries + started = time.monotonic() + try: + client.chat.completions.create( + model="openhands_qa-f30-slow-sdk", + messages=[{"role": "user", "content": "QA_F30_NO_RETRY"}], + extra_headers={"X-OpenHands-ServerConversation-ID": sys.argv[1]}, + ) + raise SystemExit("expected a 504") + except openai.InternalServerError as exc: + headers = exc.response.headers + print(exc.status_code, headers.get("x-should-retry"), headers.get("x-openhands-serverconversation-id"), len(sent), round(time.monotonic() - started)) + assert exc.status_code == 504, exc.status_code + assert headers.get("x-should-retry") == "false", dict(headers) + assert headers.get("x-openhands-serverconversation-id") == sys.argv[1], dict(headers) + assert len(sent) == 1, len(sent) + print("QA_F30_NO_RETRY_OK") + PY + control-agent-server api GET "/api/conversations/$NR_ID/events/search" --query source=user --check items len-eq 1 \ + --check items.0.llm_message.content.0.text eq QA_F30_NO_RETRY --save F30.chat-timeout-no-retry/user-messages + control-agent-server api GET "/api/conversations/$NR_ID" --check execution_status eq running --quiet + control-agent-server sink read --name qa-f30-slow-sdk --expect-min 1 --expect-max 1 + control-agent-server api DELETE "/api/conversations/$NR_ID" --expect 200 + control-agent-server api DELETE /api/profiles/qa-f30-slow-sdk --expect 200 + ``` + The client raises `InternalServerError` with status 504, + `X-Should-Retry: false` and the id it chose, after one HTTP request, + although its default would retry a 5xx twice. The conversation holds the + one user message, the stub saw one model call, and the run is still + going, so the bullet deletes it. +- **A turn that cannot start in time is retryable (`F30.inner-gateway-presend-504`).** + A conversation whose direct run hangs in its model call; the inner + gateway turn on it waits for that run to end and gives up at its + deadline without saving its message. Two minutes. + ```sh + PRE_SLOW=$(control-agent-server fixture llm-stub --name qa-f30-presend --step hang:170 --print-path) + PRE_AGENT="{\"llm\": {\"model\": \"openai/qa-stub\", \"api_key\": \"qa-stub-key\", \"base_url\": \"$PRE_SLOW/v1\", \"num_retries\": 0, \"timeout\": 300}, \"tools\": []}" + PRE=$(control-agent-server conversation start --body-json "{\"agent\": $PRE_AGENT}" --no-autotitle --print-id) + control-agent-server api POST "/api/conversations/$PRE/events" \ + --json '{"role":"user","content":[{"type":"text","text":"QA_F30_DIRECT"}],"run":true}' --expect 200 + control-agent-server conversation wait "$PRE" --until running --timeout 30 + control-agent-server sink read --name qa-f30-presend --expect-min 1 + T0=$SECONDS + control-agent-server api POST "/api/conversations/$PRE/events/_gateway/run" --timeout 200 \ + --json '{"role":"user","content":[{"type":"text","text":"QA_F30_PRESEND"}],"run":true}' \ + --expect 504 --check detail eq 'Agent run timed out' --check-header x-should-retry eq true \ + --check-header x-openhands-serverconversation-id missing --expect-max-ms 140000 --save F30.inner-gateway-presend-504/timeout + test $((SECONDS - T0)) -ge 115 + control-agent-server api GET "/api/conversations/$PRE/events/count" --query body=QA_F30_PRESEND --check . eq 0 \ + --save F30.inner-gateway-presend-504/no-message + control-agent-server api GET "/api/conversations/$PRE/events/count" --query source=user --check . eq 1 + control-agent-server sink read --name qa-f30-presend --expect-min 1 --expect-max 1 + control-agent-server api DELETE "/api/conversations/$PRE" --expect 200 + ``` + The 504 arrives after 115 to 140 s with `X-Should-Retry: true` and + without a conversation header; the gateway's message was never saved + (only the direct run's user message is in the log) and no second model + call was made, so a client retry is safe. +- **No sender on a gateway turn (`F30.inner-gateway-no-sender`).** The + field that marks a message from another conversation, on the inner + route. + ```sh + NS_FROM=$(control-agent-server conversation start --no-autotitle --tools none --print-id) + NS_TO=$(control-agent-server conversation start --no-autotitle --tools none --print-id) + control-agent-server api POST "/api/conversations/$NS_TO/events/_gateway/run" \ + --json "{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"QA_F30_NS\"}],\"run\":true,\"sender_conversation_id\":\"$NS_FROM\"}" \ + --expect 422 --check detail eq 'A gateway turn is not sent by another conversation' --save F30.inner-gateway-no-sender/refused + control-agent-server api GET "/api/conversations/$NS_TO/events/count" --check . eq 0 + control-agent-server api GET "/api/conversations/$NS_TO" --check execution_status eq idle --quiet + control-agent-server api DELETE "/api/conversations/$NS_TO" --expect 200 + control-agent-server api DELETE "/api/conversations/$NS_FROM" --expect 200 + ``` + 422 at once; the target's log is still empty and it never ran. - **Responses (`F30.responses-basic`).** One real-model call with instructions, metadata and an ignored client tool. `RID` names its conversation. diff --git a/.agents/skills/verify-agent-server/references/feature-map/README.md b/.agents/skills/verify-agent-server/references/feature-map/README.md index 3a5131ed0fa..7fcaa36b8f5 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/README.md +++ b/.agents/skills/verify-agent-server/references/feature-map/README.md @@ -10,7 +10,7 @@ recipe. Maintenance baseline: main@0275e714df62d3dd66eb51d518dd1da953ec2c16 (2026-11-17). The next maintenance pass starts from this commit, a pass proposes the next baseline in its PR, and merging that PR accepts it. -The map has **1047 sub-features**. +The map has **1068 sub-features**. ## Baseline preconditions @@ -68,12 +68,12 @@ them honest: [../maintenance.md](../maintenance.md). | F01 | [Server status, info and contract](F01-server-status.md) | liveness, health, readiness, server info and its `/` alias, uptime and idle time, OpenAPI document, the static-files mode, DEBUG tracebacks, tool preloading | `GET /alive`, `/health`, `/ready`, `/server_info`, `/`, `/openapi.json` | baseline | 11 | | F02 | [Authentication and workspace sessions](F02-auth-and-sessions.md) | session keys on HTTP and on WebSockets (first frame, deprecated query and header forms, close codes), several and rotated keys, no-auth and legacy modes, the default bind address, the workspace session cookie, CORS | `X-Session-API-Key`, `POST`/`DELETE /api/auth/workspace-session`, WebSocket auth on every socket | recipes: tmux | 22 | | F03 | [Deferred init (warm-pool dormant mode)](F03-deferred-init.md) | warm-pool dormant mode: the gate on `/api`, `/v1`, `/app-backends` and every socket (checked before auth), init key auth, validation and redaction, rollback, one-shot init, and every field the init body delivers (keys, secret key, paths, worktree root, run limit, CORS, webhooks, env, web URL) | `GET`/`POST /api/init`, `X-Init-API-Key` | git; launch `--no-auth --deferred-init` | 26 | -| F04 | [Conversation lifecycle and catalog](F04-conversation-lifecycle.md) | create from settings, an agent profile or an inline agent; client ids, parents, worktrees, start-time secrets, 422 redaction; get, batch get, search, count; PATCH; final response; DELETE; restart | `POST`/`GET`/`PATCH`/`DELETE /api/conversations[/{id}]`, `/search`, `/count`, `/agent_final_response`, `/children` | llm, git | 37 | +| F04 | [Conversation lifecycle and catalog](F04-conversation-lifecycle.md) | create from settings, an agent profile or an inline agent; client ids, parents, worktrees, start-time secrets, 422 redaction; get, batch get, search, count; PATCH; final response; DELETE; restart | `POST`/`GET`/`PATCH`/`DELETE /api/conversations[/{id}]`, `/search`, `/count`, `/agent_final_response`, `/children` | llm, git | 40 | | F05 | [Run, pause, interrupt and status transitions](F05-run-pause-interrupt.md) | run, pause and interrupt during model and tool calls; status transitions on every view; stuck detection and max_iterations stops; 409 and 429; graceful and SIGKILL restarts | `POST /api/conversations/{id}/run`, `/pause`, `/interrupt` | llm, tmux; launch `--config-json '{"max_concurrent_runs": 1}'` | 32 | -| F06 | [Conversation events and the events WebSocket](F06-conversation-events.md) | appending messages, event search, count, get and batch get (GET with a body, or the browser-safe POST) with filters and paging, the events WebSocket (auth, snapshot, replay, live frames), history across restart | `POST`/`GET /api/conversations/{id}/events[...]`, `POST /api/conversations/{id}/events/batch`, `WS /sockets/events/{id}` | llm; launch `--env TZ=UTC` | 31 | +| F06 | [Conversation events and the events WebSocket](F06-conversation-events.md) | appending messages, event search, count, get and batch get (GET with a body, or the browser-safe POST) with filters and paging, the events WebSocket (auth, snapshot, replay, live frames), history across restart | `POST`/`GET /api/conversations/{id}/events[...]`, `POST /api/conversations/{id}/events/batch`, `WS /sockets/events/{id}` | llm; launch `--env TZ=UTC` | 33 | | F07 | [Session WebSocket protocol](F07-session-socket.md) | the resumable session socket: sync, durable and transient frames, seq replay, inbound messages, error and close codes, streaming deltas, backpressure, subscriber cap | `WS /sockets/session/{id}` | llm | 27 | | F08 | [Confirmation policy, security analyzer and conversation secrets](F08-confirmation-security-secrets.md) | confirmation policies and accept/reject, the security analyzer, runtime conversation secrets (static, lookup, masking, encryption, no cipher), and what survives a restart | `POST /api/conversations/{id}/confirmation_policy`, `/security_analyzer`, `/secrets`, `/events/respond_to_confirmation` | llm, tmux | 26 | -| F09 | [Model, profile and ACP switching; runtime plugins](F09-model-switching-plugins.md) | swap the agent LLM, switch to a saved profile, switch an ACP model, load a plugin at runtime; stats, hooks and restart persistence | `POST /api/conversations/{id}/switch_llm`, `/switch_profile`, `/switch_acp_model`, `/load_plugin` | llm, tmux, node, network | 37 | +| F09 | [Model, profile and ACP switching; runtime plugins](F09-model-switching-plugins.md) | swap the agent LLM, switch to a saved profile, switch an ACP model, load a plugin at runtime; stats, hooks and restart persistence | `POST /api/conversations/{id}/switch_llm`, `/switch_profile`, `/switch_acp_model`, `/load_plugin` | llm, tmux, node, network | 49 | | F10 | [Fork, navigate, condense and ask_agent](F10-fork-navigate-condense-ask.md) | fork, navigate between branches, condense with a real summarizer, ask the agent a side question; state after restart | `POST /api/conversations/{id}/fork`, `/navigate`, `/condense`, `/ask_agent` | llm | 26 | | F11 | [Goal loop](F11-goals.md) | goal loops: start, LLM-judged completion and caps, stop, resume, supersede, interrupt, failures, busy and capacity limits | `POST /api/conversations/{id}/goal`, `/goal/stop`, `/goal/resume` | llm, tmux; launch `--config-json '{"max_concurrent_runs": 1}'` | 25 | | F12 | [Conversation runtime info, sandbox pause and credential bindings](F12-runtime-and-credentials.md) | runtime info and reprovision, prepare-for-sandbox-pause, credential bindings for ACP agents, managed LLM key refresh, the Docker runtime mode's routes and contract | `GET /api/conversations/{id}/runtime`, `POST .../runtime/reprovision`, `POST /api/conversations/prepare-for-sandbox-pause`, `PUT .../credential-bindings/{name}`, Docker mode `DELETE .../runtime`, `POST .../runtime/credentials` | llm, node, network | 31 | @@ -85,7 +85,7 @@ them honest: [../maintenance.md](../maintenance.md). | F18 | [Server settings, settings schemas and the secrets store](F18-settings-and-secrets.md) | persisted settings with redacted, plaintext and encrypted exposure, sparse PATCH, settings schemas, the named secrets store, encryption at rest, cipher-key changes | `GET`/`PATCH /api/settings`, `/api/settings/agent-schema`, `/conversation-schema`, `/api/settings/secrets[/{name}]` | git, node | 32 | | F19 | [MCP server settings, connection test and OAuth](F19-mcp.md) | MCP server settings CRUD, connection tests (stdio and HTTP), install-time OAuth jobs and token refresh, a model-driven MCP tool call, MCP servers added to a running conversation, conversations that start before a slow MCP server is ready | `/api/settings/mcp/{key}`, `POST /api/mcp/test`, `/api/mcp/oauth/*`, `POST /api/conversations/{id}/mcp` | llm; launch `--webhook-sink` | 41 | | F20 | [LLM profiles](F20-llm-profiles.md) | LLM profiles: save, read, validate against real and stubbed providers, activate, rename, delete, name rules, limits, store contention, restart | `/api/profiles[/{name}]`, `/validate`, `/activate`, `/rename` | llm, node | 37 | -| F21 | [Agent profiles](F21-agent-profiles.md) | agent profiles (OpenHands and ACP kinds): CRUD, default seed, activation, rename, delete, materialize, conversations started from a profile, store contention | `/api/agent-profiles[/{name}]`, `/rename`, `/{id}/activate`, `/materialize` | llm | 32 | +| F21 | [Agent profiles](F21-agent-profiles.md) | agent profiles (OpenHands and ACP kinds): CRUD, default seed, activation, rename, delete, materialize, conversations started from a profile, store contention | `/api/agent-profiles[/{name}]`, `/rename`, `/{id}/activate`, `/materialize` | llm | 33 | | F22 | [Meta-profiles (model router)](F22-meta-profiles.md) | meta-profiles (model router): save, read, activate, delete, settings reflection, routing in a conversation | `/api/meta-profiles[/{name}]`, `/activate` | llm, node | 24 | | F23 | [LLM catalog and provider connections](F23-llm-catalog-and-connections.md) | provider and model catalogs, verified models, provider connections (create, rotate, delete, references from profiles), live ACP discovery and errors | `GET /api/llm/providers`, `/api/llm/models[/verified]`, `/api/llm/provider-connections[/{id}]`, `POST /api/acp/models` | llm, node, network | 28 | | F24 | [OpenAI ChatGPT subscription device flow](F24-openai-subscription.md) | the ChatGPT subscription device flow: models, status, device start and poll, logout, upstream failures | `/api/llm/subscription/openai/*` | node, network | 20 | @@ -94,7 +94,7 @@ them honest: [../maintenance.md](../maintenance.md). | F27 | [Hooks, sub-agents and tool catalogs](F27-hooks-subagents-tools.md) | hooks and sub-agent catalogs, blocking hooks in a conversation, the tool registry and catalog, client tools, forwarded agent definitions, custom tool modules | `POST /api/hooks`, `POST /api/sub-agents`, `GET /api/tools/`, `/api/tools/catalog` | llm, git, tmux, chromium | 33 | | F28 | [Canvas apps: install, catalog and backend lifecycle](F28-canvas-extensions.md) | Canvas app install, catalog, enable, uninstall, bundle and icon; backend approval, prepare, start, stop, logs, data, failure states, restart | `/api/canvas-extensions/install`, `/api/canvas-extensions/installed[/{name}][/backend...]` | git, node; launch `--env TZ=UTC` | 30 | | F29 | [Canvas App backend bridge (session, HTTP and WebSocket proxy)](F29-app-backend-bridge.md) | the app-backend bridge: session cookies, origin and host checks, proxied HTTP methods and WebSockets, revocation | `/app-backends/{name}/session`, `/app-backends/{name}[/{path}]`, `WS /app-backends/{name}[/{path}]` | git; launch `--canvas-ingress` | 29 | -| F30 | [OpenAI-compatible gateway (/v1)](F30-openai-gateway.md) | the OpenAI-compatible gateway: models, chat completions (plain and streaming), responses, conversation reuse, usage, the official OpenAI SDK | `GET /v1/models`, `POST /v1/chat/completions`, `/v1/responses` | llm, tmux | 31 | +| F30 | [OpenAI-compatible gateway (/v1)](F30-openai-gateway.md) | the OpenAI-compatible gateway: models, chat completions (plain and streaming), responses, conversation reuse, usage, the official OpenAI SDK | `GET /v1/models`, `POST /v1/chat/completions`, `/v1/responses` | llm, tmux | 34 | | F31 | [Persistence, restart recovery, leases and limits](F31-persistence-and-limits.md) | state that survives graceful and SIGKILL restarts, crash recovery, conversation ownership leases and takeover between servers, idle eviction, run capacity, config file vs environment precedence | behavior only: `restart`, `restart --hard`, a second server on shared storage | llm, tmux | 24 | | F32 | [Event webhooks and product telemetry](F32-webhooks-and-telemetry.md) | event webhooks (event batches, conversation updates, headers, buffering, flush, limits, retries) and product telemetry (off by default, exporter lifecycle, sanitized payloads, consent) | behavior only: `webhooks` / `OH_WEBHOOKS_*`, `OH_TELEMETRY_*` | llm; launch `--webhook-sink` | 30 | | F33 | [Python SDK remote client (RemoteConversation, RemoteWorkspace)](F33-sdk-remote-client.md) | the Python SDK in this checkout against the server: RemoteWorkspace commands, files, git and settings readers; RemoteConversation create, run, callbacks, streaming, reattach, fork, confirmation, pause, interrupt, stop hooks, client tools, completion logs | behavior only: `Workspace(host=...)`, `RemoteWorkspace`, `RemoteConversation` | llm, tmux, git | 43 | diff --git a/.agents/skills/verify-sdk/SKILL.md b/.agents/skills/verify-sdk/SKILL.md index de462773fcf..6a07c44be05 100644 --- a/.agents/skills/verify-sdk/SKILL.md +++ b/.agents/skills/verify-sdk/SKILL.md @@ -39,7 +39,10 @@ export SDK_VERIFY_RUN=$(control-sdk new --llm deepseek-flash --field run) # live A run has its own HOME, `OH_PERSISTENCE_DIR`, conversation store, workspace and tmux server, and drops the caller's `LLM_*` and `OH_*` variables, so it never touches a developer's `~/.openhands`. Live runs read `DEEPSEEK_API_KEY` -when a model is called; without it they exit 3 and report `blocked`. +when a model is called, or, with `LLM_BASE_URL` and `LLM_API_KEY` set, send +`deepseek-flash` through that LiteLLM proxy (the run carries the pair as +`SDK_VERIFY_LLM_BASE_URL`/`SDK_VERIFY_LLM_API_KEY`); without either they exit 3 +and report `blocked`. ## Doctor @@ -80,7 +83,8 @@ sdk-map run --all --out "$SDK_VERIFY_HOME/full" # the whole map A bullet passes when every block it ran exits 0; exit 3 means blocked. A block that needs a real model (a live run, `--model-env`, an example script or the -integration runner) runs only with `DEEPSEEK_API_KEY` set and is skipped whole +integration runner) runs only with `DEEPSEEK_API_KEY` (or the proxy's +`LLM_API_KEY`) set and is skipped whole otherwise. An integration run passes only when its `results.json` shows every requested test succeeded, because the runner exits 0 on failed tests. Each block runs in its own process group, which is killed when the block ends or diff --git a/.agents/skills/verify-sdk/programs/S03/model_router.py b/.agents/skills/verify-sdk/programs/S03/model_router.py new file mode 100644 index 00000000000..3c58dfb840d --- /dev/null +++ b/.agents/skills/verify-sdk/programs/S03/model_router.py @@ -0,0 +1,374 @@ +"""A conversation whose agent LLM is a `RouterLLM`: route status, manual and +automatic selection, routes added from saved profiles, and what `switch_llm` +and `switch_profile` do to a router. Each route is a real `LLM` answering from +its own local endpoint (`S02/fake_provider.py`), so the program can tell which +route served each turn. Profiles go to the run's default store +(`$OH_PERSISTENCE_DIR/profiles`), as a user's would. + +Usage: model_router.py MODE + selection status, switch_routed_llm / resume_auto_llm_routing in agent + runs, refused selections, persistence and resume + add-route add_model_router_route: wrap a plain LLM, extend a router, + run on the added route, refused routes + switch-replaces switch_profile / switch_llm on a router conversation, and the + router's own copy-returning switch methods + before-first-run a route added and selected before the first message: the + run's usage in conversation_stats +""" + +import json +import sys +import uuid + +from pydantic import SecretStr, ValidationError +from S02.fake_provider import FakeProvider +from sdk_verify import conversation_dir, emit, run_paths + +from openhands.sdk import LLM, Agent, Conversation, LLMProfileStore +from openhands.sdk.llm import ( + ModelRouterSelection, + ModelRouterStatus, + ModelRouterUpdateRequest, + RouterLLM, +) +from openhands.sdk.llm.router import MultimodalRouter + + +mode = sys.argv[1] +paths = run_paths() +WORKSPACE = str(paths["workspace"]) +PRICE = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002} # 0.14 per call +out: dict = {"ok": True, "mode": mode} + + +def fake_llm(server: FakeProvider, usage_id: str, model: str) -> LLM: + return LLM( + model=model, + base_url=server.url, + api_key=SecretStr("sk-test"), + usage_id=usage_id, + num_retries=1, + **PRICE, + ) + + +def attempt(fn): + try: + return fn(), None + except Exception as exc: + return None, f"{type(exc).__name__}: {exc}"[:200] + + +def status_of(conversation) -> dict: + llm = conversation.agent.llm + if not isinstance(llm, RouterLLM): + return ModelRouterStatus(enabled=False).model_dump(mode="json") + return llm.status().model_dump(mode="json") + + +def turn(conversation, text: str) -> None: + conversation.send_message(text) + conversation.run() + + +def costs(conversation) -> dict: + return { + k: round(v.accumulated_cost, 4) + for k, v in conversation.conversation_stats.usage_to_metrics.items() + } + + +store = LLMProfileStore() + +if mode == "selection": + cid = uuid.uuid4() + with ( + FakeProvider([{"finish": "A1"}, {"finish": "A2"}, {"finish": "A3"}]) as a, + FakeProvider([{"finish": "B1"}, {"finish": "B2"}]) as b, + ): + router = RouterLLM( + usage_id="router", + llms_for_routing={ + "first": fake_llm(a, "route-first", "openai/gpt-4.1"), + "second": fake_llm(b, "route-second", "openai/gpt-4.1-mini"), + }, + ) + conversation = Conversation( + agent=Agent(llm=router, tools=[]), + workspace=WORKSPACE, + persistence_dir=str(paths["conversations"]), + conversation_id=cid, + visualizer=None, + delete_on_close=False, + ) + out["initial"] = status_of(conversation) + turn(conversation, "Task one.") + out["after_auto_turn"] = status_of(conversation) + conversation.switch_routed_llm("second") + out["after_manual"] = status_of(conversation) + out["router_kept"] = type(conversation.agent.llm).__name__ + out["state_agent_follows"] = ( + conversation.state.agent.llm is conversation.agent.llm + ) + turn(conversation, "Task two.") + out["after_manual_turn"] = status_of(conversation) + persisted = json.loads( + (conversation_dir(str(cid)) / "base_state.json").read_text() + )["agent"]["llm"] + out["persisted"] = { + "router_name": persisted.get("router_name"), + "selection": persisted.get("selection"), + "routes": sorted(persisted.get("llms_for_routing", {})), + } + _, out["unknown_route_error"] = attempt( + lambda: conversation.switch_routed_llm("missing") + ) + out["after_unknown_route"] = status_of(conversation)["selection"] + conversation.resume_auto_llm_routing() + turn(conversation, "Task three.") + out["after_resume_turn"] = status_of(conversation) + out["requests"] = {"first": len(a.requests), "second": len(b.requests)} + out["costs"] = costs(conversation) + conversation.update_model_router_selection( + ModelRouterSelection(mode="manual", manual_route_name="second") + ) + out["persisted_before_resume"] = json.loads( + (conversation_dir(str(cid)) / "base_state.json").read_text() + )["agent"]["llm"].get("selection") + conversation.close() + # Resume the persisted conversation with a fresh, auto-routing router: + # on resume the program's agent (and its LLM) wins over the stored one. + fresh = RouterLLM( + usage_id="router", + llms_for_routing={ + "first": fake_llm(a, "route-first", "openai/gpt-4.1"), + "second": fake_llm(b, "route-second", "openai/gpt-4.1-mini"), + }, + ) + resumed = Conversation( + agent=Agent(llm=fresh, tools=[]), + workspace=WORKSPACE, + persistence_dir=str(paths["conversations"]), + conversation_id=cid, + visualizer=None, + delete_on_close=False, + ) + out["resumed"] = status_of(resumed) + out["resumed_type"] = type(resumed.agent.llm).__name__ + resumed.close() + # Selection shapes the model refuses, and a plain-LLM conversation. + _, out["manual_without_route"] = attempt( + lambda: ModelRouterSelection(mode="manual") + ) + _, out["auto_with_route"] = attempt( + lambda: ModelRouterSelection(mode="auto", manual_route_name="first") + ) + _, out["update_request_without_selection"] = attempt( + lambda: ModelRouterUpdateRequest.model_validate({}) + ) + out["validation_error_type"] = ValidationError.__name__ + plain = Conversation( + agent=Agent(llm=LLM(model="openai/gpt-4.1", usage_id="plain"), tools=[]), + workspace=WORKSPACE, + visualizer=None, + ) + _, out["plain_switch_error"] = attempt(lambda: plain.switch_routed_llm("first")) + _, out["plain_resume_error"] = attempt(plain.resume_auto_llm_routing) + out["plain_status"] = status_of(plain) + plain.close() + +elif mode == "add-route": + with ( + FakeProvider([{"finish": "A1"}, {"finish": "A2"}]) as a, + FakeProvider([{"finish": "R1"}]) as r, + FakeProvider([]) as late, + ): + store.save( + "qa-reviewer", + fake_llm(r, "reviewer", "openai/gpt-4.1-mini"), + include_secrets=True, + ) + conversation = Conversation( + agent=Agent(llm=fake_llm(a, "agent", "openai/gpt-4.1"), tools=[]), + workspace=WORKSPACE, + visualizer=None, + ) + turn(conversation, "Task one.") + returned = conversation.add_model_router_route( + "qa-reviewer", route_name="reviewer" + ) + llm = conversation.agent.llm + out["returned"] = returned.model_dump(mode="json") + out["agent_llm_type"] = type(llm).__name__ + out["route_usage_ids"] = { + name: route.usage_id for name, route in llm.llms_for_routing.items() + } + out["registry"] = sorted(conversation.llm_registry.list_usage_ids()) + out["registry_holds_route"] = ( + conversation.llm_registry.get("profile:qa-reviewer") + is llm.llms_for_routing["reviewer"] + ) + turn(conversation, "Task two (auto).") + out["after_auto_turn"] = status_of(conversation) + conversation.switch_routed_llm("reviewer") + turn(conversation, "Task three (reviewer).") + out["after_reviewer_turn"] = status_of(conversation) + out["requests"] = {"agent": len(a.requests), "reviewer": len(r.requests)} + out["costs"] = costs(conversation) + _, out["duplicate_route_error"] = attempt( + lambda: conversation.add_model_router_route( + "qa-reviewer", route_name="default" + ) + ) + _, out["missing_profile_error"] = attempt( + lambda: conversation.add_model_router_route("qa-no-such-profile") + ) + # The route is a snapshot: editing the saved profile does not change it. + store.save( + "qa-reviewer", + fake_llm(late, "reviewer", "openai/gpt-4o-mini"), + include_secrets=True, + ) + out["route_model_after_profile_edit"] = conversation.agent.llm.llms_for_routing[ + "reviewer" + ].model + out["routes_after_errors"] = [ + r["name"] for r in status_of(conversation)["routes"] + ] + conversation.close() + # A route added to an existing MultimodalRouter keeps its router type, and + # base_route_name names the wrapped LLM only when there is no router yet. + multimodal = Conversation( + agent=Agent( + llm=MultimodalRouter( + usage_id="mm", + llms_for_routing={ + "primary": fake_llm(a, "mm-primary", "openai/gpt-4o-mini"), + "secondary": fake_llm(a, "mm-secondary", "openai/gpt-4.1-mini"), + }, + ), + tools=[], + ), + workspace=WORKSPACE, + visualizer=None, + ) + extended = multimodal.add_model_router_route( + "qa-reviewer", base_route_name="ignored" + ) + out["multimodal_type"] = type(multimodal.agent.llm).__name__ + out["multimodal_routes"] = [route.name for route in extended.routes] + multimodal.close() + named = Conversation( + agent=Agent(llm=fake_llm(a, "agent", "openai/gpt-4.1"), tools=[]), + workspace=WORKSPACE, + visualizer=None, + ) + out["named_base_routes"] = [ + route.name + for route in named.add_model_router_route( + "qa-reviewer", base_route_name="main" + ).routes + ] + named.close() + +elif mode == "switch-replaces": + with ( + FakeProvider([{"finish": "B1"}]) as b, + FakeProvider([{"finish": "C1"}]) as c, + FakeProvider([]) as idle, + ): + store.save( + "qa-plain-b", + fake_llm(b, "plain-b", "openai/gpt-4.1-mini"), + include_secrets=True, + ) + + def router_conversation() -> Conversation: + router = RouterLLM( + usage_id="router", + llms_for_routing={ + "first": fake_llm(idle, "route-first", "openai/gpt-4.1"), + "second": fake_llm(idle, "route-second", "openai/gpt-4o-mini"), + }, + ).with_manual_route("second") + return Conversation( + agent=Agent(llm=router, tools=[]), workspace=WORKSPACE, visualizer=None + ) + + by_profile = router_conversation() + by_profile.switch_profile("qa-plain-b") + out["after_switch_profile"] = { + "type": type(by_profile.agent.llm).__name__, + "model": by_profile.agent.llm.model, + "usage_id": by_profile.agent.llm.usage_id, + "status": status_of(by_profile), + } + _, out["select_after_switch_profile"] = attempt( + lambda: by_profile.switch_routed_llm("second") + ) + turn(by_profile, "Task on B.") + by_profile.close() + + by_object = router_conversation() + by_object.switch_llm(fake_llm(c, "plain-c", "openai/gpt-4.1-nano")) + out["after_switch_llm"] = { + "type": type(by_object.agent.llm).__name__, + "model": by_object.agent.llm.model, + "status": status_of(by_object), + } + turn(by_object, "Task on C.") + by_object.close() + out["requests"] = { + "b": len(b.requests), + "c": len(c.requests), + "routes": len(idle.requests), + } + + # The router's own switch methods return a copy and leave the original. + router = RouterLLM( + llms_for_routing={ + "first": fake_llm(idle, "first", "openai/gpt-4.1"), + "second": fake_llm(idle, "second", "openai/gpt-4o-mini"), + } + ) + pinned = router.switch_llm("second") + out["router_switch_llm"] = { + "copy": pinned is not router, + "original": router.selection.model_dump(mode="json"), + "copy_selection": pinned.selection.model_dump(mode="json"), + "active_route_name": pinned.active_route_name, + "resumed": pinned.resume_auto_routing().selection.model_dump(mode="json"), + } + _, out["router_switch_unknown"] = attempt(lambda: router.switch_llm("missing")) + _, out["router_with_route_duplicate"] = attempt( + lambda: router.with_route("first", fake_llm(idle, "dup", "openai/gpt-4.1")) + ) + +elif mode == "before-first-run": + with ( + FakeProvider([]) as dead, + FakeProvider([{"finish": "R1"}]) as r, + ): + store.save( + "qa-early-route", + fake_llm(r, "early", "openai/gpt-4.1-mini"), + include_secrets=True, + ) + conversation = Conversation( + agent=Agent(llm=fake_llm(dead, "agent", "openai/gpt-4.1"), tools=[]), + workspace=WORKSPACE, + visualizer=None, + ) + conversation.add_model_router_route("qa-early-route", route_name="early") + conversation.switch_routed_llm("early") + turn(conversation, "Task on the early route.") + out["status"] = conversation.state.execution_status.value + out["requests"] = {"base": len(dead.requests), "route": len(r.requests)} + out["last_route_name"] = status_of(conversation)["last_route_name"] + out["costs"] = costs(conversation) + conversation.close() + +else: + raise SystemExit(f"unknown mode {mode}") + +emit(out) diff --git a/.agents/skills/verify-sdk/programs/S04/control.py b/.agents/skills/verify-sdk/programs/S04/control.py index 71701bd7dca..87990c96341 100644 --- a/.agents/skills/verify-sdk/programs/S04/control.py +++ b/.agents/skills/verify-sdk/programs/S04/control.py @@ -18,6 +18,7 @@ import threading import time +from pydantic import Field from S01.sleeper import TIMELINE # registers "SleeperTool" from sdk_verify import emit, run_paths, script_messages @@ -46,10 +47,13 @@ def sleeper(label: str, seconds: float = 0.6) -> dict: class SlowLLM(TestLLM): - """A scripted model that takes `delay` seconds per call (sync and async).""" + """A scripted model that takes `delay` seconds per call (sync and async). + + The agent's `llm` is serialized as its own class (SerializeAsAny, #6789), so + the event is kept out of the persisted state.""" delay: float = 0.0 - in_flight: threading.Event | None = None + in_flight: threading.Event | None = Field(default=None, exclude=True) def completion(self, *args, **kwargs): if self.in_flight is not None: diff --git a/.agents/skills/verify-sdk/programs/S04/resolution.py b/.agents/skills/verify-sdk/programs/S04/resolution.py new file mode 100644 index 00000000000..02285f61e4c --- /dev/null +++ b/.agents/skills/verify-sdk/programs/S04/resolution.py @@ -0,0 +1,192 @@ +"""The resolution judge: is anything left for the user after a turn? + +Usage: resolution.py verdicts|judge + +- verdicts: `verdict_from_judgments` on fixed judgment sets, no model. +- judge: `resolution_judge(fallback, store)` on an empty profile store saves + the `resolution_judge` profile from the fallback LLM, then + `judge_resolution` asks it the questions through a local OpenAI-compatible + endpoint that answers with fixed JSON; a second call with another fallback + reuses the saved profile; a cipher store keeps the key encrypted on disk; a + reply that is not JSON raises. + +The endpoint is `S16/fake_openai.py`, so no key or network is needed. +""" + +import asyncio +import json +import sys + +from pydantic import SecretStr +from S16 import fake_openai +from sdk_verify import emit, run_paths + +from openhands.sdk import LLM +from openhands.sdk.conversation.resolution import ( + RESOLUTION_JUDGE_PROFILE_NAME, + RESOLUTION_QUESTIONS, + ResolutionState, + judge_resolution, + resolution_judge, + verdict_from_judgments, +) +from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.utils.cipher import Cipher + + +MODE = sys.argv[1] +LOW = { + "asks_user": 0.05, + "hands_off_review": 0.05, + "offers_choice": 0.05, + "asks_to_check": 0.05, + "reports_problem": 0.05, +} + + +def verdict(judgments: dict[str, float]) -> dict: + v = verdict_from_judgments(judgments, model="qa-judge") + return { + "owed": v.owed, + "probability": round(v.probability, 4), + "resolved": v.resolved, + "needs_user": v.needs_user, + "note": v.note(), + "model": v.model, + } + + +if MODE == "verdicts": + emit( + { + "questions": sorted(RESOLUTION_QUESTIONS), + "done": verdict({"handled": 0.95, "task_done": 0.9, **LOW}), + "asks": verdict( + {"handled": 0.95, "task_done": 0.9, **LOW, "asks_user": 0.8} + ), + "review": verdict( + {"handled": 0.95, "task_done": 0.9, **LOW, "hands_off_review": 0.9} + ), + "unfinished": verdict({"handled": 0.95, "task_done": 0.4, **LOW}), + "unclear": verdict( + {"handled": 0.95, "task_done": 0.9, **LOW, "offers_choice": 0.3} + ), + "problem": verdict( + {"handled": 0.95, "task_done": 0.9, **LOW, "reports_problem": 0.7} + ), + "out_of_range": verdict( + {"handled": 7, "task_done": "x", **LOW} # type: ignore[dict-item] + ), + } + ) + sys.exit(0) + +REPLY = json.dumps( + { + "handled": {"probability": 0.97}, + "task_done": {"probability": 0.92}, + "asks_user": {"probability": 0.9}, + "hands_off_review": {"probability": 0.02}, + "offers_choice": {"probability": 0.02}, + "asks_to_check": {"probability": 0.02}, + "reports_problem": {"probability": 0.02}, + } +) +STATE = ResolutionState( + request="Rename the setting.", + reply="Renamed it. Should I also update the docs?", + actions=["edit settings.py"], + task="Clean up the settings module.", +) + + +def fallback(base_url: str, model: str, key: str) -> LLM: + return LLM( + model=model, + api_key=SecretStr(key), + base_url=base_url, + usage_id="agent", + num_retries=3, + timeout=600, + ) + + +base = run_paths()["run"] / "profiles" / "resolution" +plain_dir, cipher_dir = base / "plain", base / "cipher" +plain_dir.mkdir(parents=True, exist_ok=True) +cipher_dir.mkdir(parents=True, exist_ok=True) +url = fake_openai.serve(REPLY) +store = LLMProfileStore(plain_dir) +before = store.list() + +judge = resolution_judge(fallback(url, "openai/qa-judge-a", "qa-key-a"), store) +first = asyncio.run(judge_resolution(judge, STATE)) +saved = store.load(RESOLUTION_JUDGE_PROFILE_NAME) +request = fake_openai.requests[-1] +prompt = json.dumps(request.get("messages")) + +again = resolution_judge(fallback(url, "openai/qa-judge-b", "qa-key-b"), store) + +cipher = Cipher("qa-s04-resolution-secret") +cipher_store = LLMProfileStore(cipher_dir) +resolution_judge( + fallback(url, "openai/qa-judge-c", "qa-key-c"), cipher_store, cipher=cipher +) +cipher_file = (cipher_dir / f"{RESOLUTION_JUDGE_PROFILE_NAME}.json").read_text() +cipher_loaded = cipher_store.load(RESOLUTION_JUDGE_PROFILE_NAME, cipher=cipher) + +bad_dir = base / "bad" +bad_dir.mkdir(parents=True, exist_ok=True) +bad_url = fake_openai.serve("I think it is resolved.") +bad_judge = resolution_judge( + fallback(bad_url, "openai/qa-judge-bad", "qa-key-bad"), LLMProfileStore(bad_dir) +) +calls_before_bad = len(fake_openai.requests) +try: + asyncio.run(judge_resolution(bad_judge, STATE)) + bad_error = None +except Exception as exc: # what a caller sees + bad_error = f"{type(exc).__name__}: {exc}" + +emit( + { + "profiles_before": before, + "profiles_after": store.list(), + "saved": { + "model": saved.model, + "base_url": saved.base_url, + "api_key_kept": saved.api_key is not None + and saved.api_key.get_secret_value() == "qa-key-a", + "usage_id": saved.usage_id, + }, + "judge": { + "model": judge.llm.model, + "usage_id": judge.llm.usage_id, + "llm_timeout": judge.llm.timeout, + "llm_num_retries": judge.llm.num_retries, + "timeout": judge.timeout, + "num_retries": judge.num_retries, + }, + "verdict": { + "owed": first.owed, + "resolved": first.resolved, + "needs_user": first.needs_user, + "probability": round(first.probability, 4), + "model": first.model, + "judgments": sorted(first.judgments), + }, + "request": { + "model": request.get("model"), + "has_state": "Rename the setting." in prompt, + "names_questions": all(name in prompt for name in RESOLUTION_QUESTIONS), + }, + "reused_model": again.llm.model, + "cipher": { + "plain_key_on_disk": "qa-key-c" in cipher_file, + "loaded_key_matches": cipher_loaded.api_key is not None + and cipher_loaded.api_key.get_secret_value() == "qa-key-c", + }, + "bad_reply_error": bad_error, + "bad_reply_requests": len(fake_openai.requests) - calls_before_bad, + } +) diff --git a/.agents/skills/verify-sdk/programs/S04/sender.py b/.agents/skills/verify-sdk/programs/S04/sender.py new file mode 100644 index 00000000000..710800dd53b --- /dev/null +++ b/.agents/skills/verify-sdk/programs/S04/sender.py @@ -0,0 +1,71 @@ +"""A message one conversation sends another, marked with its sender. + +Usage: sender.py + +Two persisted conversations; the second gets one message from the user and +one sent with `sender=conversation_sender(first.id)`. The program reports the +helper's value and the `sender` of both saved messages, in memory and after a +reload from `persistence_dir`. Scripted: no model call is made. +""" + +import uuid + +from sdk_verify import emit, make_llm, run_paths + +from openhands.sdk import Agent, Conversation +from openhands.sdk.conversation.types import ( + CONVERSATION_SENDER_PREFIX, + conversation_sender, +) +from openhands.sdk.event import MessageEvent + + +paths = run_paths() +store = paths["run"] / "sender-store" + + +def open_conversation(conversation_id=None): + return Conversation( + agent=Agent(llm=make_llm(script=[{"finish": "done"}]), tools=[]), + workspace=str(paths["workspace"]), + persistence_dir=str(store), + conversation_id=conversation_id, + visualizer=None, + ) + + +def user_messages(conversation) -> list[dict]: + with conversation.state: + return [ + { + "text": e.llm_message.content[0].text, + "sender": e.sender, + } + for e in conversation.state.events + if isinstance(e, MessageEvent) and e.source == "user" + ] + + +sender_conv = open_conversation() +receiver = open_conversation() +receiver_id = receiver.id +value = conversation_sender(sender_conv.id) +receiver.send_message("QA_S04_FROM_USER") +receiver.send_message("QA_S04_FROM_CONVERSATION", sender=value) +in_memory = user_messages(receiver) +receiver.close() +reloaded = open_conversation(receiver_id) +after_reload = user_messages(reloaded) + +emit( + { + "prefix": CONVERSATION_SENDER_PREFIX, + "value": value, + "value_is_prefix_plus_hex": value + == CONVERSATION_SENDER_PREFIX + sender_conv.id.hex, + "hex_parses": uuid.UUID(value.removeprefix(CONVERSATION_SENDER_PREFIX)) + == sender_conv.id, + "in_memory": in_memory, + "after_reload": after_reload, + } +) diff --git a/.agents/skills/verify-sdk/programs/S06/condense.py b/.agents/skills/verify-sdk/programs/S06/condense.py index 4192fccb467..6e18c179e27 100644 --- a/.agents/skills/verify-sdk/programs/S06/condense.py +++ b/.agents/skills/verify-sdk/programs/S06/condense.py @@ -15,6 +15,8 @@ reset (retries, then gives up). soft size pressure with nothing worth forgetting (minimum_progress) leaves the run alone; an explicit request resets hard. + empty-summary size pressure while the summary LLM answers with no text + (""), then with only whitespace (" "). reasons condensation reasons and their combination, on a View taken from a real conversation (the API a custom condenser author uses). pipeline PipelineCondenser and NoOpCondenser inside conversations. @@ -361,6 +363,26 @@ def summary_reached_agent(): out["hard_offset"] = conds[-1].summary_offset if conds else None conversation.close() +elif mode == "empty-summary": + for name, blank in (("empty", ""), ("blank", " ")): + usage_id = f"condenser-{name}" + condenser = summarizer([blank] * 6, usage_id=usage_id) + conversation = conversation_for(echo_steps(8) + [{"finish": "done"}], condenser) + error = run(conversation) + conds = condensations(conversation) + out[name] = { + "error": error, + "status": conversation.state.execution_status.value, + "summary_calls": len(calls(usage_id)), + "condensations": len(conds), + "empty_summary_condensations": sum( + not (c.summary or "").strip() for c in conds + ), + "forgotten": sum(len(c.forgotten_event_ids) for c in conds), + "summaries": [c.summary for c in conds], + } + conversation.close() + elif mode == "reasons": script = echo_steps(12) + [{"finish": "done"}] conversation = conversation_for(script, None) diff --git a/.agents/skills/verify-sdk/programs/S09/conversation.py b/.agents/skills/verify-sdk/programs/S09/conversation.py index d63c9565cc7..9257c605872 100644 --- a/.agents/skills/verify-sdk/programs/S09/conversation.py +++ b/.agents/skills/verify-sdk/programs/S09/conversation.py @@ -10,6 +10,12 @@ startup a working server next to one that exits at startup, then the failing server alone. disabled only a disabled server (its command would write a marker file). + payload one server with --payload-tools; the agent calls `resource` (an + EmbeddedResource block) and `structured_only` (structuredContent + without content blocks). + shadow Agent(tools=[terminal]) with one server that also advertises a + tool named `terminal` (--shadow-terminal) next to `sibling_check`; + the agent calls the built-in terminal, then `sibling_check`. The recorder keeps every LLM call's tool schemas; the program prints facts. """ @@ -21,7 +27,7 @@ from recorder import calls, recording_llm from sdk_verify import conversation_dir, emit, run_paths -from openhands.sdk import Agent, Conversation +from openhands.sdk import Agent, Conversation, Tool from openhands.sdk.event import ObservationEvent from openhands.sdk.mcp import MCPServer @@ -208,4 +214,48 @@ def results(conversation): out["marker_written"] = marker.exists() conversation.close() +elif mode == "payload": + conversation, error = converse( + {"verify": server("--payload-tools")}, + [call("resource"), call("structured_only"), {"finish": "done"}], + ) + out["error"] = error + out["status"] = conversation.state.execution_status.value + rows = {r["tool"]: r for r in results(conversation)} + out["resource_text"] = rows.get("resource", {}).get("text") + out["structured_only_text"] = rows.get("structured_only", {}).get("text") + conversation.close() + +elif mode == "shadow": + from openhands.tools.terminal import TerminalTool + + agent = Agent( + llm=recording_llm( + [ + call("terminal", command="echo builtin-terminal"), + call("sibling_check"), + {"finish": "done"}, + ] + ), + tools=[Tool(name=TerminalTool.name)], + mcp_config={"ext": server("--shadow-terminal")}, + ) + conversation = Conversation( + agent=agent, + workspace=str(paths["workspace"]), + persistence_dir=str(paths["conversations"]), + visualizer=None, + ) + error = None + try: + conversation.send_message("use the tools") + conversation.run() + except Exception as exc: + error = f"{type(exc).__name__}: {exc}"[:300] + out["error"] = error + out["status"] = conversation.state.execution_status.value + out["tools"] = sorted(conversation.agent.tools_map) + out["texts"] = [r["text"] for r in results(conversation)] + conversation.close() + emit(out) diff --git a/.agents/skills/verify-sdk/programs/S09/mcp_server.py b/.agents/skills/verify-sdk/programs/S09/mcp_server.py index 4e1b8a8c8cb..1cef02b9c85 100644 --- a/.agents/skills/verify-sdk/programs/S09/mcp_server.py +++ b/.agents/skills/verify-sdk/programs/S09/mcp_server.py @@ -3,7 +3,7 @@ Usage: mcp_server.py [stdio | http PORT | sse PORT] [--name NAME] [--slow-start SECONDS] [--fail-start] [--oauth [--no-metadata]] [--fail-first-call] - [--handshake-only] + [--handshake-only] [--payload-tools] [--shadow-terminal] --fail-first-call (http only) answers the first tools/call with HTTP 503, a transient gateway error. @@ -19,6 +19,10 @@ right after answering), register_extra / remove_extra (change the tool list and send notifications/tools/list_changed while answering), register_late (adds a tool and sends the notification on the session after answering). +--payload-tools adds `resource` (one EmbeddedResource text block) and +`structured_only` (no content blocks, only structuredContent). +--shadow-terminal adds a tool named `terminal`, as the built-in tool is, and +`sibling_check`. """ import asyncio @@ -205,6 +209,46 @@ async def remove_extra() -> str: return "removed" +if "--payload-tools" in args: + from fastmcp.tools import ToolResult + + @mcp.tool(output_schema=None) + def resource() -> ToolResult: + """Return one embedded text resource and nothing else.""" + return ToolResult( + content=[ + mcp_types.EmbeddedResource( + type="resource", + resource=mcp_types.TextResourceContents( + uri="file:///verify/payload.json", + mimeType="application/json", + text='{"payload": "VERIFY_RESOURCE_TEXT"}', + ), + ) + ] + ) + + @mcp.tool(output_schema=None) + def structured_only() -> ToolResult: + """Return structured content without any content block.""" + return ToolResult( + content=[], structured_content={"payload": "VERIFY_STRUCTURED"} + ) + + +if "--shadow-terminal" in args: + + @mcp.tool(name="terminal") + def shadow_terminal(command: str) -> str: + """An MCP tool that has the built-in terminal tool's name.""" + return f"mcp terminal: {command}" + + @mcp.tool() + def sibling_check() -> str: + """A tool next to the shadowing one.""" + return "sibling ok" + + # OAuth endpoints for --oauth: authorization-server metadata (unless # --no-metadata) and two token endpoints that record which one was hit. if "--oauth" in args: diff --git a/.agents/skills/verify-sdk/programs/S09/runtime.py b/.agents/skills/verify-sdk/programs/S09/runtime.py index 9cc154cba6e..b2a21492d62 100644 --- a/.agents/skills/verify-sdk/programs/S09/runtime.py +++ b/.agents/skills/verify-sdk/programs/S09/runtime.py @@ -27,7 +27,7 @@ from openhands.sdk import Agent from openhands.sdk.conversation.impl.local_conversation import LocalConversation from openhands.sdk.event import AgentErrorEvent, ObservationEvent -from openhands.sdk.mcp import MCPServer +from openhands.sdk.mcp import MCPError, MCPServer, MCPServerStartError mode = sys.argv[1] @@ -98,7 +98,15 @@ def mcp_tools(conv): out["sse_error"] = refusal( conv, {"remote": MCPServer(url="http://127.0.0.1:9/sse", transport="sse")} ) - out["start_failure_error"] = refusal(conv, {"broken": server("--fail-start")}) + try: + conv.add_mcp_servers({"broken": server("--fail-start")}) + out["start_failure_error"] = None + except Exception as exc: + out["start_failure_error"] = f"{type(exc).__name__}: {exc}"[:200] + out["start_failure_is_mcp_error"] = isinstance(exc, MCPError) + if isinstance(exc, MCPServerStartError): + out["start_failure_server_name"] = exc.server_name + out["start_failure_cause"] = type(exc.__cause__).__name__ out["disabled_error"] = refusal( conv, {"off": MCPServer(command="sh", args=["-c", "exit 1"], enabled=False)} ) diff --git a/.agents/skills/verify-sdk/programs/S10/hook_secrets.py b/.agents/skills/verify-sdk/programs/S10/hook_secrets.py new file mode 100644 index 00000000000..55a45a3b5c4 --- /dev/null +++ b/.agents/skills/verify-sdk/programs/S10/hook_secrets.py @@ -0,0 +1,57 @@ +"""A command hook in a conversation that has secrets. + +The conversation holds the secret VERIFY_HOOK_SECRET. A PreToolUse hook on +terminal and the agent's own terminal call both test whether the variable is +set in their environment, without printing it. The program reports what each +saw. +""" + +from sdk_verify import emit, make_llm, run_paths + +from openhands.sdk import Agent, Conversation, Tool +from openhands.sdk.event import HookExecutionEvent, ObservationEvent +from openhands.sdk.hooks import HookConfig +from openhands.tools.terminal import TerminalTool + + +paths = run_paths() +probe = 'test -n "$VERIFY_HOOK_SECRET" && echo {0}_SECRET_SET || echo {0}_SECRET_UNSET' +llm = make_llm( + [ + {"tool": "terminal", "args": {"command": probe.format("TERMINAL")}}, + {"finish": "done"}, + ] +) +conversation = Conversation( + agent=Agent(llm=llm, tools=[Tool(name=TerminalTool.name)]), + workspace=str(paths["workspace"]), + persistence_dir=str(paths["conversations"]), + secrets={"VERIFY_HOOK_SECRET": "verify-hook-secret-value"}, + hook_config=HookConfig.model_validate( + { + "pre_tool_use": [ + {"matcher": "terminal", "hooks": [{"command": probe.format("HOOK")}]} + ] + } + ), + visualizer=None, +) +conversation.send_message("check the secret") +conversation.run() +events = list(conversation.state.events) +hooks = [e for e in events if isinstance(e, HookExecutionEvent)] +observations = [ + e for e in events if isinstance(e, ObservationEvent) and e.tool_name == "terminal" +] +emit( + { + "ok": True, + "status": conversation.state.execution_status.value, + "hook_runs": len(hooks), + "hook_stdout": hooks[0].stdout.strip() if hooks else None, + "terminal_text": observations[0].observation.text[:200] + if observations + else None, + } +) +conversation.close() diff --git a/.agents/skills/verify-sdk/programs/S11/task_fail_closed.py b/.agents/skills/verify-sdk/programs/S11/task_fail_closed.py new file mode 100644 index 00000000000..e6e9cc7aa7d --- /dev/null +++ b/.agents/skills/verify-sdk/programs/S11/task_fail_closed.py @@ -0,0 +1,176 @@ +"""Sub-agent confirmation when nothing can confirm, and the parent's security +analyzer. + +1. task tool, no handler: `careful` (permission_mode: always_confirm) runs + `touch closed-task.txt`. +2. DelegateExecutor, no handler (wrapped in a small `delegate` tool, as in + S11/delegate.py): `careful_worker` (permission_mode: always_confirm) runs + `touch closed-delegate.txt`. +3. task tool with a recording handler that rejects: the parent has + ConfirmRisky and a recording analyzer (LOW for `task`, HIGH for anything + else); `inheritor` (no permission_mode) runs `touch risky.txt`. + +Reports which files exist, which actions the parent's analyzer was asked +about, and the handler's calls. +""" + +from s11_helpers import scripted, task_observations +from sdk_verify import emit, run_paths + +from openhands.sdk import Agent, Conversation, Tool +from openhands.sdk.security.analyzer import SecurityAnalyzerBase +from openhands.sdk.security.confirmation_policy import ConfirmRisky +from openhands.sdk.security.risk import SecurityRisk +from openhands.sdk.subagent import AgentDefinition, register_agent +from openhands.sdk.tool import ToolDefinition, register_tool +from openhands.tools.delegate.definition import DelegateAction, DelegateObservation +from openhands.tools.delegate.impl import DelegateExecutor +from openhands.tools.task import TaskToolSet +from openhands.tools.terminal import TerminalTool + + +ws = run_paths()["workspace"] +ANALYZED: list[dict] = [] +HANDLER_CALLS: list[dict] = [] + + +class RecordingAnalyzer(SecurityAnalyzerBase): + def security_risk(self, action): # type: ignore[override] + command = getattr(action.action, "command", None) + ANALYZED.append({"tool": action.tool_name, "command": command}) + return SecurityRisk.LOW if action.tool_name == "task" else SecurityRisk.HIGH + + +def reject(task_id: str, pending: list) -> bool: + HANDLER_CALLS.append({"task_id": task_id, "tools": [a.tool_name for a in pending]}) + return False + + +class RejectingTaskToolSet(TaskToolSet): + @classmethod + def create(cls, conv_state, **params): # type: ignore[override] + return TaskToolSet.create(conv_state, confirmation_handler=reject, **params) + + +class DelegateTool(ToolDefinition[DelegateAction, DelegateObservation]): + @classmethod + def create(cls, conv_state): # type: ignore[override] # noqa: ARG003 + return [ + cls( + action_type=DelegateAction, + observation_type=DelegateObservation, + description="Spawn named sub-agents, then delegate tasks to them.", + executor=DelegateExecutor(max_children=2), + ) + ] + + +register_tool("rejecting_task_tool_set", RejectingTaskToolSet) +register_tool("delegate", DelegateTool) + + +def toucher(name: str, file: str, permission_mode: str | None): + register_agent( + name, + lambda _llm: Agent( + llm=scripted( + [ + {"tool": "terminal", "args": {"command": f"touch {file}"}}, + {"finish": f"{name} done"}, + ], + name, + ), + tools=[Tool(name=TerminalTool.name)], + ), + AgentDefinition( + name=name, + description=f"Touches {file}.", + tools=["terminal"], + permission_mode=permission_mode, + ), + ) + + +toucher("careful", "closed-task.txt", "always_confirm") +toucher("careful_worker", "closed-delegate.txt", "always_confirm") +toucher("inheritor", "risky.txt", None) + + +def conversation_for(tool: str, script: list): + return Conversation( + agent=Agent(llm=scripted(script, "parent"), tools=[Tool(name=tool)]), + workspace=str(ws), + visualizer=None, + ) + + +# 1. The task tool without a handler. +plain = conversation_for( + TaskToolSet.name, + [ + {"tool": "task", "args": {"prompt": "touch", "subagent_type": "careful"}}, + {"finish": "delegated"}, + ], +) +plain.send_message("Delegate to careful.") +plain.run() +task_obs = task_observations(plain.state.events) +plain.close() + +# 2. DelegateExecutor without a handler. +delegating = conversation_for( + "delegate", + [ + { + "tool": "delegate", + "args": { + "command": "spawn", + "ids": ["w"], + "agent_types": ["careful_worker"], + }, + }, + { + "tool": "delegate", + "args": {"command": "delegate", "tasks": {"w": "touch the file"}}, + }, + {"finish": "delegated"}, + ], +) +delegating.send_message("Delegate to a careful worker.") +delegating.run() +delegate_texts = [ + e.observation.text[:300] + for e in delegating.state.events + if type(getattr(e, "observation", None)).__name__ == "DelegateObservation" +] +delegating.close() + +# 3. The parent's ConfirmRisky and analyzer; a handler that rejects. +risky = conversation_for( + "rejecting_task_tool_set", + [ + {"tool": "task", "args": {"prompt": "touch", "subagent_type": "inheritor"}}, + {"finish": "delegated"}, + ], +) +risky.set_security_analyzer(RecordingAnalyzer()) +risky.set_confirmation_policy(ConfirmRisky()) +risky.send_message("Delegate to inheritor.") +risky.run() +risky_status = risky.state.execution_status.value +risky.close() + +emit( + { + "ok": True, + "task_file": (ws / "closed-task.txt").exists(), + "task_observations": task_obs, + "delegate_file": (ws / "closed-delegate.txt").exists(), + "delegate_observations": delegate_texts, + "risky_status": risky_status, + "analyzed_tools": [a["tool"] for a in ANALYZED], + "sub_action_analyzed": any(a["tool"] == "terminal" for a in ANALYZED), + "handler_calls": HANDLER_CALLS, + "risky_file": (ws / "risky.txt").exists(), + } +) diff --git a/.agents/skills/verify-sdk/programs/S11/task_resume_session.py b/.agents/skills/verify-sdk/programs/S11/task_resume_session.py new file mode 100644 index 00000000000..fa6494b19a8 --- /dev/null +++ b/.agents/skills/verify-sdk/programs/S11/task_resume_session.py @@ -0,0 +1,107 @@ +"""Resuming a sub-agent task after the parent conversation is reopened. + +Usage: task_resume_session.py create | resume CONVERSATION_ID | retype + +create a persisted parent delegates to `echoer` (task_00000001), finishes + and closes; prints the parent's conversation id. +resume a new process reopens that parent by id (a new task tool and task + manager) and its model calls task(resume="task_00000001"). +retype in one session, the parent delegates to `echoer`, then resumes + task_00000001 naming another type (`other`, which has no tools). + +Each echoer turn has its own scripted LLM; the program reports the task +observations and whether the sub-agent was asked again in the conversation +it started in. +""" + +import sys +import uuid +from pathlib import Path + +from s11_helpers import calls_for, scripted, task_observations +from sdk_verify import emit, run_paths + +from openhands.sdk import Agent, Conversation, Tool +from openhands.sdk.subagent import AgentDefinition, register_agent +from openhands.tools.task import TaskToolSet +from openhands.tools.terminal import TerminalTool + + +mode = sys.argv[1] +paths = run_paths() +echoer_scripts = [ + [{"finish": "said hello"}], + [{"finish": "you asked me to say hello"}], +] +register_agent( + "echoer", + lambda _llm: Agent( + llm=scripted(echoer_scripts.pop(0), "echoer"), + tools=[Tool(name=TerminalTool.name)], + ), + AgentDefinition(name="echoer", description="Echoes.", tools=["terminal"]), +) +register_agent( + "other", + lambda _llm: Agent(llm=scripted([{"finish": "other"}], "other"), tools=[]), + AgentDefinition(name="other", description="No tools.", tools=[]), +) + + +def parent(script, conversation_id=None): + options = {} + if conversation_id is not None: + options["conversation_id"] = uuid.UUID(conversation_id) + return Conversation( + agent=Agent( + llm=scripted(script, "parent"), tools=[Tool(name=TaskToolSet.name)] + ), + workspace=str(paths["workspace"]), + persistence_dir=str(paths["conversations"]), + visualizer=None, + **options, + ) + + +def task(**args): + return {"tool": "task", "args": {"prompt": "say hello", **args}} + + +if mode == "create": + conversation = parent([task(subagent_type="echoer"), {"finish": "first done"}]) +elif mode == "resume": + conversation = parent( + [ + task(subagent_type="echoer", resume="task_00000001"), + {"finish": "second done"}, + ], + sys.argv[2], + ) +elif mode == "retype": + conversation = parent( + [ + task(subagent_type="echoer"), + task(subagent_type="other", resume="task_00000001"), + {"finish": "done"}, + ] + ) +else: + raise SystemExit(f"unknown mode {mode}") + +conversation.send_message("Delegate.") +conversation.run() +subagents = Path(conversation.state.persistence_dir) / "subagents" +out = { + "ok": True, + "mode": mode, + "id": str(conversation.id), + "execution_status": conversation.state.execution_status.value, + "observations": task_observations(conversation.state.events), + "echoer_calls": len(calls_for("echoer")), + "other_calls": len(calls_for("other")), + "subagent_dirs": sorted(p.name for p in subagents.glob("*")) + if subagents.is_dir() + else [], +} +conversation.close() +emit(out) diff --git a/.agents/skills/verify-sdk/programs/S14/terminal_flags.py b/.agents/skills/verify-sdk/programs/S14/terminal_flags.py new file mode 100644 index 00000000000..03a1a072941 --- /dev/null +++ b/.agents/skills/verify-sdk/programs/S14/terminal_flags.py @@ -0,0 +1,73 @@ +"""What a terminal observation says about a timed-out command, and a command +that starts with a Bash comment. + +Usage: terminal_flags.py subprocess|tmux + +`Tool(name="terminal", params={"terminal_type": MODE, +"no_change_timeout_seconds": 2})`. A scripted agent runs: + control sleep 2 && printf '%s-%s\n' synthetic status-output (the + output, not the command line, holds synthetic-status-output) + comment the same command after a `# ...` comment line + hard sleep 5 with timeout=1 (hard limit), then C-c + idle sleep 30 with no timeout: no new output for 2 s, then C-c +The program reports each call's exit code, `timeout` flag, text and how long +it took (action to observation). +""" + +import sys +from datetime import datetime + +from S14.toolkit import converse, tool_results +from sdk_verify import emit + +from openhands.sdk import Tool +from openhands.sdk.event import ActionEvent, ObservationEvent +from openhands.tools.terminal import TerminalTool + + +mode = sys.argv[1] +params = {"terminal_type": mode, "no_change_timeout_seconds": 2} +command = "sleep 2 && printf '%s-%s\\n' synthetic status-output" +steps = [ + ("control", {"command": command, "timeout": 12}), + ( + "comment", + {"command": f"# Read-only: print a status line.\n{command}", "timeout": 12}, + ), + ("hard", {"command": "sleep 5", "timeout": 1}), + ("hard_stop", {"command": "C-c", "is_input": True}), + ("idle", {"command": "sleep 30"}), + ("idle_stop", {"command": "C-c", "is_input": True}), + ("after", {"command": "echo after-ok"}), +] +script = [{"tool": "terminal", "args": args} for _, args in steps] + [ + {"finish": "done"} +] +conversation = converse([Tool(name=TerminalTool.name, params=params)], script) +rows = [r for r in tool_results(conversation) if r["tool"] == "terminal"] +events = list(conversation.state.events) + + +def seconds(event) -> float: + return datetime.fromisoformat(event.timestamp).timestamp() + + +actions = {e.id: e for e in events if isinstance(e, ActionEvent)} +durations = [ + round(seconds(e) - seconds(actions[e.action_id]), 2) + for e in events + if isinstance(e, ObservationEvent) and e.tool_name == "terminal" +] +named = {} +for (name, _), row, took in zip(steps, rows, durations): + named[name] = { + "exit_code": row.get("exit_code"), + "timeout": row.get("timeout"), + "is_error": row["is_error"], + "text": row["text"][:600], + "seconds": took, + } +status = conversation.state.execution_status.value +conversation.close() + +emit({"ok": True, "mode": mode, "execution_status": status, "steps": named}) diff --git a/.agents/skills/verify-sdk/programs/S19/profile_sandbox.py b/.agents/skills/verify-sdk/programs/S19/profile_sandbox.py new file mode 100644 index 00000000000..5cbe428124d --- /dev/null +++ b/.agents/skills/verify-sdk/programs/S19/profile_sandbox.py @@ -0,0 +1,112 @@ +"""Sandbox launch metadata on agent profiles. + +Usage: profile_sandbox.py + +`sandbox=ProfileSandboxSettings(spec_id=...)` on both profile variants: +validation, a round trip through `AgentProfileStore`, an older payload without +the field, the refused shapes, and `resolve()` giving the same settings with +and without it. No model call is made. +""" + +from pydantic import SecretStr, ValidationError +from sdk_verify import emit + +from openhands.sdk import LLM +from openhands.sdk.launch import LaunchStores, resolve +from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.profiles import ( + AgentProfileStore, + OpenHandsAgentProfile, + ProfileSandboxSettings, + validate_agent_profile, +) + + +SPEC = "registry.example/agent-server:qa-s19" +out: dict = {"ok": True} + +old = validate_agent_profile( + {"schema_version": 4, "name": "old", "llm_profile_ref": "fast"} +) +out["old_payload_sandbox"] = old.sandbox +out["old_dump_sandbox"] = old.model_dump(mode="json").get("sandbox", "absent") + +variants = {} +for kind in ("openhands", "acp"): + payload: dict = { + "agent_kind": kind, + "name": f"qa-{kind}", + "sandbox": {"spec_id": SPEC}, + } + if kind == "openhands": + payload["llm_profile_ref"] = "fast" + else: + payload["acp_server"] = "codex" + profile = validate_agent_profile(payload) + variants[kind] = { + "type": type(profile).__name__, + "spec_id": profile.sandbox.spec_id if profile.sandbox else None, + "is_settings_type": isinstance(profile.sandbox, ProfileSandboxSettings), + "round_trip": validate_agent_profile(profile.model_dump(mode="json")) + == profile, + } +out["variants"] = variants + +refused = {} +for label, sandbox in { + "empty_spec": {"spec_id": ""}, + "no_spec": {}, + "extra_key": {"spec_id": "img", "size": "large"}, + "string": "img", +}.items(): + try: + validate_agent_profile( + {"name": "bad", "llm_profile_ref": "fast", "sandbox": sandbox} + ) + refused[label] = "accepted" + except ValidationError as exc: + refused[label] = exc.errors()[0]["type"] +out["refused"] = refused + +agents = AgentProfileStore() +agents.save( + OpenHandsAgentProfile( + name="qa-sbx", + llm_profile_ref="fast", + sandbox=ProfileSandboxSettings(spec_id=SPEC), + ) +) +loaded = agents.load("qa-sbx") +out["store"] = { + "spec_id": loaded.sandbox.spec_id if loaded.sandbox else None, + "on_disk_spec": (agents.base_dir / "qa-sbx.json").read_text().count(SPEC), +} + +llms = LLMProfileStore() +llms.save( + "fast", + LLM(model="openai/fast-model", api_key=SecretStr("sk-fast"), usage_id="agent"), + include_secrets=True, +) +stores = LaunchStores(llm_profiles=llms, mcp_config={}, skills=lambda: []) + + +def resolved_dump(profile: OpenHandsAgentProfile) -> dict: + settings = resolve(profile, stores).settings.model_dump(mode="json") + settings["agent_context"].pop("current_datetime", None) + return settings + + +plain = resolved_dump(OpenHandsAgentProfile(name="plain", llm_profile_ref="fast")) +with_sandbox = resolved_dump( + OpenHandsAgentProfile( + name="plain", + llm_profile_ref="fast", + sandbox=ProfileSandboxSettings(spec_id=SPEC), + ) +) +out["resolve"] = { + "same_settings": plain == with_sandbox, + "settings_mention_spec": SPEC in str(with_sandbox), +} +emit(out) diff --git a/.agents/skills/verify-sdk/programs/sdk_verify.py b/.agents/skills/verify-sdk/programs/sdk_verify.py index a799e838768..01738183160 100644 --- a/.agents/skills/verify-sdk/programs/sdk_verify.py +++ b/.agents/skills/verify-sdk/programs/sdk_verify.py @@ -13,8 +13,10 @@ emit({"ok": True, "events": 3}) `make_llm()` without a script returns the run's real model (`control-sdk new ---llm deepseek-flash`), reading the key from DEEPSEEK_API_KEY at call time. A -missing key exits 3, which `control-sdk` and `sdk-map run` report as blocked. +--llm deepseek-flash`), reading the key from DEEPSEEK_API_KEY at call time, or, +with $LLM_BASE_URL and $LLM_API_KEY set, deepseek-flash through that LiteLLM +proxy (as `control-agent-server llm preset deepseek` does). A missing key exits +3, which `control-sdk` and `sdk-map run` report as blocked. Keys are never printed or written. """ @@ -43,6 +45,32 @@ }, } +# With $LLM_BASE_URL and $LLM_API_KEY set, these models go through that LiteLLM +# proxy instead, with the model name it serves. +PROXY_MODELS: dict[str, str] = { + "deepseek-flash": "litellm_proxy/deepseek-v4.1-flash", +} +PROXY_ENVS = ( + ("SDK_VERIFY_LLM_BASE_URL", "SDK_VERIFY_LLM_API_KEY"), + ("LLM_BASE_URL", "LLM_API_KEY"), +) + + +def model_spec(name: str) -> dict[str, str] | None: + """The run model's model, key variable and base URL: the proxy route when + $LLM_BASE_URL and $LLM_API_KEY are set and it serves the model, otherwise + the provider's own.""" + spec = MODELS.get(name) + if spec is None: + return None + # A run drops the caller's LLM_* variables; control-sdk carries the proxy + # into it as SDK_VERIFY_LLM_BASE_URL and SDK_VERIFY_LLM_API_KEY. + for url_env, key_env in PROXY_ENVS: + base_url = os.environ.get(url_env) + if name in PROXY_MODELS and base_url and os.environ.get(key_env): + return {"model": PROXY_MODELS[name], "env": key_env, "base_url": base_url} + return spec + def emit(result: dict[str, Any]) -> None: """Print the program's result as the last line of output.""" @@ -154,10 +182,12 @@ def make_llm( f"run model is {name!r}: pass a script, or create the run with" f" `control-sdk new --llm {next(iter(MODELS))}`" ) - spec = MODELS[name] + spec = model_spec(name) or MODELS[name] key = os.environ.get(spec["env"]) if not key: blocked(f"set {spec['env']} for {name}") + if spec["env"] != MODELS[name]["env"]: + kwargs.setdefault("base_url", spec["base_url"]) return LLM( model=spec["model"], api_key=SecretStr(key or ""), usage_id=usage_id, **kwargs ) @@ -165,7 +195,8 @@ def make_llm( def has_model_key() -> bool: name = run_config().get("llm", "scripted") - return name in MODELS and bool(os.environ.get(MODELS[name]["env"])) + spec = model_spec(name) + return spec is not None and bool(os.environ.get(spec["env"])) # -------------------------------------------------------------------------- diff --git a/.agents/skills/verify-sdk/references/feature-map/README.md b/.agents/skills/verify-sdk/references/feature-map/README.md index 88958026d44..0dd0b1a8c75 100644 --- a/.agents/skills/verify-sdk/references/feature-map/README.md +++ b/.agents/skills/verify-sdk/references/feature-map/README.md @@ -10,9 +10,9 @@ agent-server map and follows its layout. Family IDs here are `Snn`, so they never collide with that map's `Fnn`. -Maintenance baseline: main@0275e714df62d3dd66eb51d518dd1da953ec2c16 (2026-11-17). The next maintenance pass starts from this commit, a pass proposes the next baseline in its PR, and merging that PR accepts it. +Maintenance baseline: main@141423921fdaeb08cd5f53464a69e860da70b66f (2026-11-19). The next maintenance pass starts from this commit, a pass proposes the next baseline in its PR, and merging that PR accepts it. -The map has **496 sub-features**, **34 of them untested** (no unit or cross +The map has **504 sub-features**, **36 of them untested** (no unit or cross test exercises the behavior; most of those have an example script). ## What each sub-feature links @@ -38,9 +38,13 @@ the way a user's program does (see [recipes.md](../recipes.md)). ## Baseline preconditions - `uv sync --dev` has run in the repository root. -- Live bullets: `DEEPSEEK_API_KEY` is exported. `control-sdk new --llm - deepseek-flash` runs use it, and `control-sdk exec --model-env` hands it to - examples and the integration runner as `LLM_API_KEY`/`LLM_MODEL`. +- Live bullets: `DEEPSEEK_API_KEY` is exported, or `LLM_BASE_URL` and + `LLM_API_KEY` name a LiteLLM proxy that serves `deepseek-v4.1-flash` (then + `deepseek-flash` goes through the proxy, as `control-agent-server llm preset + deepseek` does; `deepseek-pro` still needs `DEEPSEEK_API_KEY`). `control-sdk + new --llm deepseek-flash` runs use it, and `control-sdk exec --model-env` + hands it to examples and the integration runner as + `LLM_API_KEY`/`LLM_MODEL`/`LLM_BASE_URL`. - Run commands from the repository root. - `control-sdk` and `sdk-map` are on `PATH`: `export PATH="$PWD/.agents/skills/verify-sdk/scripts:$PATH"`. @@ -73,8 +77,8 @@ an interactive login are labeled **blocked**. |---|---|---|---|---|---|---| | S01 | [Agent step loop](S01-agent-loop.md) | step handling (tool calls, text, empty replies), iteration and budget limits, stuck detector, parallel tools, messages sent mid-run, image input, `ask_agent`, structured output, recovery from bad tool calls and context overflow | `Conversation(max_iteration_per_run=, stuck_detection=, max_budget_per_run=)`, `Agent(tool_concurrency_limit=)`, `ask_agent()`, `response_schema` | llm, tmux, network | 24 | 1 | | S02 | [LLM core](S02-llm-core.md) | completion, responses and `generate`; streaming; reasoning and thinking blocks; native and prompted tool calling; metrics, stats and pricing; retries and error mapping; caching, token counts, image inlining, completion logs | `LLM(...)`, `llm.completion`/`responses`, `Conversation(token_callbacks=)`, `llm.metrics` | llm, network | 28 | 0 | -| S03 | [LLM registry, routing, fallback, profiles and model switching](S03-llm-routing-profiles.md) | `LLMRegistry`, multimodal and random routers, `FallbackStrategy`, `LLMProfileStore`, `switch_profile`/`switch_llm`, the switch-LLM tool, `route_task_to_model`, subscription login | `LLMRegistry`, `MultimodalRouter`, `FallbackStrategy`, `LLMProfileStore`, `LLM.subscription_login()` | llm, network | 26 | 3 | -| S04 | [Local conversation lifecycle](S04-local-conversation.md) | factory and ids, persistence and resume, callbacks, close, pause, `arun` and interrupt, secrets, visualizers, fork and branches, goal loop (and its stop on a pending `ask_user` question, `GoalAsked`), final response, stats, title and tags | `Conversation(persistence_dir=, callbacks=, visualizer=, secrets=)`, `run`/`pause`/`arun`/`interrupt`/`fork`/`run_goal` | llm, tmux, network, uvx | 31 | 3 | +| S03 | [LLM registry, routing, fallback, profiles and model switching](S03-llm-routing-profiles.md) | `LLMRegistry`, multimodal and random routers, `FallbackStrategy`, `LLMProfileStore`, `switch_profile`/`switch_llm`, the switch-LLM tool, `route_task_to_model`, subscription login | `LLMRegistry`, `MultimodalRouter`, `FallbackStrategy`, `LLMProfileStore`, `LLM.subscription_login()` | llm, network | 30 | 4 | +| S04 | [Local conversation lifecycle](S04-local-conversation.md) | factory and ids, persistence and resume, callbacks, close, pause, `arun` and interrupt, secrets, visualizers, fork and branches, goal loop (and its stop on a pending `ask_user` question, `GoalAsked`), final response, stats, title and tags | `Conversation(persistence_dir=, callbacks=, visualizer=, secrets=)`, `run`/`pause`/`arun`/`interrupt`/`fork`/`run_goal` | llm, tmux, network, uvx | 34 | 4 | | S05 | [Events and the event log](S05-events.md) | event model, JSON round trip, conversion to LLM messages and OpenAI formats, unmatched actions, the `EventLog`, error classification, completion and token events | `Event.model_validate_json`, `events_to_messages`, `conversation.state.events` | llm, tmux | 18 | 1 | | S06 | [Agent context, system prompt, condensers and memory](S06-context-condensers.md) | `AgentContext`, system prompt sections and vendor files, persona, persistent memory, the condenser `View`, `LLMSummarizingCondenser`, `condense()`, overflow condensation, no-op and pipeline condensers | `Agent(agent_context=, system_prompt=, condenser=)`, `AgentContext(load_memory=)`, `conversation.condense()` | llm, tmux | 24 | 1 | | S07 | [Skills](S07-skills.md) | skill sources (programmatic, Markdown, AGENTS.md, SKILL.md, user, project, public, installed), keyword, task and path triggers, `<available_skills>`, `invoke_skill`, inline commands, disabling, install and marketplace, the `OH_PUBLIC_SKILLS_ENABLED` switch | `AgentContext(skills=, load_*_skills=, disabled_skills=)`, `.agents/skills/`, `install_skill`, `OH_PUBLIC_SKILLS_ENABLED` | llm, git, network | 25 | 0 | @@ -89,7 +93,7 @@ an interactive login are labeled **blocked**. | S16 | [Planning, task tracking, oracle, ToM, workflow, reaction, message-edit, mention and ask-user tools](S16-planning-workflow-tools.md) | task tracker, planning preset and PLAN.md editor, plan-then-execute, `ask_oracle`, `tom_consult`, sleeptime compute, the workflow tool, emoji reactions (`react`/`unreact`, `conversation.react`), message edits (`edit_message`, `MessageEditEvent`, `message_text`), `ask_user` questions and answers, conversation mention links and the `find_conversations`/`read_conversation` tools (server-backed, F38) | `Tool(name="task_tracker" \| "ask_oracle" \| "tom_consult" \| "workflow_tool_set" \| "react" \| "unreact" \| "edit_message" \| "ask_user" \| "find_conversations" \| "read_conversation")`, `get_planning_agent`, `LocalConversation.react`, `LocalConversation.answer_ask`, `conversation_mention` | llm | 30 | 6 | | S17 | [Custom tools, client-defined tools and the tool registry](S17-custom-and-client-tools.md) | custom tool classes and tool sets, registration and naming, tool schemas, injected fields, catalog seal, module qualnames, client tools, `execute_tool` | `register_tool`, `Tool(name, params)`, `Conversation(client_tools=)`, `execute_tool` | llm | 24 | 1 | | S18 | [Workspaces and sandbox backends](S18-workspace-backends.md) | the `Workspace` factory, `LocalWorkspace`, `RemoteWorkspace`, and the Docker, API-remote, Cloud, Apptainer and agent-sandbox backends (start, health, commands, files, pause, cleanup) | `Workspace(...)`, `LocalWorkspace`, `openhands.workspace.*Workspace` | docker, apptainer, cloud, k8s, network, llm, git | 24 | 3 | -| S19 | [Agent settings, persisted schema migrations and agent profiles](S19-settings.md) | `OpenHandsAgentSettings`, ACP settings, schema migrations v0 to v8, settings diffs and export, the persisted-settings compatibility gate, agent profiles and their store | `openhands.sdk.settings`, `openhands.sdk.profiles`, `~/.openhands/agent-profiles/` | llm, tmux, network | 25 | 0 | +| S19 | [Agent settings, persisted schema migrations and agent profiles](S19-settings.md) | `OpenHandsAgentSettings`, ACP settings, schema migrations v0 to v8, settings diffs and export, the persisted-settings compatibility gate, agent profiles and their store | `openhands.sdk.settings`, `openhands.sdk.profiles`, `~/.openhands/agent-profiles/` | llm, tmux, network | 26 | 0 | | S20 | [Observability, tracing and logging](S20-observability.md) | enabling tracing, Laminar and OTLP backends, conversation, step, tool and LLM spans, sub-agent trace links, automation helpers, the logger | `LMNR_*`/`OTEL_*` env vars, `Conversation(observability_*=)`, `get_logger` | llm, network | 22 | 5 | | S21 | [ACP agent (in-process)](S21-acp-agent.md) | construction, provider registry, handshake, turns and tool-call events, permission auto-approval, timeouts and cancel, session resume, model switching, usage, MCP forwarding, secrets, cleanup | `ACPAgent(acp_command=...)`, `switch_acp_model`, `ACPAgentSettings` | node, network, llm | 27 | 2 | diff --git a/.agents/skills/verify-sdk/references/feature-map/S03-llm-routing-profiles.md b/.agents/skills/verify-sdk/references/feature-map/S03-llm-routing-profiles.md index e6cef258c63..acbe2f0d995 100644 --- a/.agents/skills/verify-sdk/references/feature-map/S03-llm-routing-profiles.md +++ b/.agents/skills/verify-sdk/references/feature-map/S03-llm-routing-profiles.md @@ -1,6 +1,6 @@ # LLM registry, routing, fallback, profiles and model switching -A Python program that uses more than one model meets this family. `LLMRegistry` holds `LLM`s by `usage_id`, gives each one independent metrics, and notifies subscribers; every `LocalConversation` has its own `conversation.llm_registry`, into which it registers every LLM the agent reaches (the agent's, the condenser's, router children), and `conversation_stats` tracks them all, per conversation: an `LLM` object used by two conversations counts each one's usage in that conversation's stats and the total in `llm.metrics`. A `RouterLLM` (`MultimodalRouter`, `RandomRouter`) is itself an `LLM` that picks one of `llms_for_routing` per call; `MultimodalRouter` sends a request with images, or one too large for the secondary model, to `primary` and everything else to `secondary`. `LLM(fallback_strategy=FallbackStrategy(fallback_llms=[...]))` retries a call that failed with a transient error on saved profiles, in order, and records their cost under the caller's usage. `LLMProfileStore` saves and loads named `LLM` configurations as JSON (default `~/.openhands/profiles`, secrets masked unless asked for). A running conversation can change model with `conversation.switch_profile(name)` or `conversation.switch_llm(llm)`, the agent can do it itself with the `switch_llm` tool, or a classifier LLM can pick a profile through the `route_task_to_model` tool configured by a meta-profile. `LLM.subscription_login(vendor="openai", model=...)` returns an `LLM` backed by a ChatGPT Plus/Pro subscription through OAuth instead of an API key. +A Python program that uses more than one model meets this family. `LLMRegistry` holds `LLM`s by `usage_id`, gives each one independent metrics, and notifies subscribers; every `LocalConversation` has its own `conversation.llm_registry`, into which it registers every LLM the agent reaches (the agent's, the condenser's, router children), and `conversation_stats` tracks them all, per conversation: an `LLM` object used by two conversations counts each one's usage in that conversation's stats and the total in `llm.metrics`. A `RouterLLM` (`MultimodalRouter`, `RandomRouter`) is itself an `LLM` that picks one of `llms_for_routing` per call; `MultimodalRouter` sends a request with images, or one too large for the secondary model, to `primary` and everything else to `secondary`. The router owns its route table and its selection: in `auto` mode it routes by its policy (the base `RouterLLM` uses its first route) and records the route it chose as `last_route_name`, in `manual` mode every call goes to the pinned route; `router.status()` (a `ModelRouterStatus`) reports both, and on a conversation whose agent LLM is a router `conversation.switch_routed_llm(route)`, `resume_auto_llm_routing()` and `update_model_router_selection(ModelRouterSelection(...))` change the selection without replacing the router, while `conversation.add_model_router_route(profile)` adds a saved profile as a route (wrapping a plain LLM in a `RouterLLM` first). `LLM(fallback_strategy=FallbackStrategy(fallback_llms=[...]))` retries a call that failed with a transient error on saved profiles, in order, and records their cost under the caller's usage. `LLMProfileStore` saves and loads named `LLM` configurations as JSON (default `~/.openhands/profiles`, secrets masked unless asked for). A running conversation can change model with `conversation.switch_profile(name)` or `conversation.switch_llm(llm)`, the agent can do it itself with the `switch_llm` tool, or a classifier LLM can pick a profile through the `route_task_to_model` tool configured by a meta-profile. `LLM.subscription_login(vendor="openai", model=...)` returns an `LLM` backed by a ChatGPT Plus/Pro subscription through OAuth instead of an API key. Source: `openhands-sdk/openhands/sdk/llm/llm_registry.py`, `openhands-sdk/openhands/sdk/llm/router/`, `openhands-sdk/openhands/sdk/llm/fallback_strategy.py`, `openhands-sdk/openhands/sdk/llm/llm_profile_store.py`, `openhands-sdk/openhands/sdk/llm/meta_profile_store.py`, `openhands-sdk/openhands/sdk/llm/provider_connection_store.py`, `openhands-sdk/openhands/sdk/llm/auth/`, `openhands-sdk/openhands/sdk/llm/llm.py`, `openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py`, `openhands-sdk/openhands/sdk/conversation/conversation_stats.py`, `openhands-sdk/openhands/sdk/tool/builtins/switch_llm.py`, `openhands-sdk/openhands/sdk/tool/builtins/classify_and_switch_llm.py`, `openhands-sdk/openhands/sdk/settings/model.py` @@ -19,6 +19,10 @@ Examples: `examples/01_standalone_sdk/05_use_llm_registry.py`, `examples/01_stan - `S03.multimodal-router-text`: text-only turns go to `secondary`, and turns whose token count exceeds the secondary's `effective_max_input_tokens` go to `primary`. Unit: **none (untested)**. Example: `examples/01_standalone_sdk/19_llm_routing.py` (text turns before and after the image turn). Real-LLM: none. - `S03.router-validation`: `MultimodalRouter` without a `primary` or `secondary` key and any `RouterLLM` with empty `llms_for_routing` fail validation. Unit: **none (untested)**. Example: none. Real-LLM: none. - `S03.random-router`: `RandomRouter` picks a random entry of `llms_for_routing` per call. Unit: **none (untested)**. Example: none. Real-LLM: none. +- `S03.model-router-selection`: on a conversation whose agent LLM is a `RouterLLM`, `router.status()` reports `enabled`, the `selection` (`auto`, no route) and every route's `name`, `model` and `label`; an `auto` turn goes to the first route and sets `last_route_name`; `conversation.switch_routed_llm(route)` pins a route (the agent keeps the router, `state.agent` follows, `base_state.json` stores the selection) and the next turn goes there; an unknown route raises `ValueError` and keeps the selection; `resume_auto_llm_routing()` returns to the first route; `ModelRouterSelection` refuses `manual` without a route and `auto` with one; a plain-LLM conversation raises `ValueError: Current agent LLM is not a RouterLLM.`; a resumed conversation runs on the router the program passes, not the stored selection. Unit: `tests/sdk/llm/test_router_llm.py`, `tests/sdk/conversation/test_switch_model.py::test_switch_routed_llm_updates_agent_and_state`, `::test_resume_auto_llm_routing_updates_agent_and_state`, `::test_switch_routed_llm_survives_state_roundtrip`, `::test_switch_routed_llm_rejects_non_router`. Example: none. Real-LLM: none. +- `S03.model-router-add-route`: `conversation.add_model_router_route(profile, route_name=None, base_route_name="default")` wraps a plain agent LLM in a `RouterLLM` (the old LLM becomes route `base_route_name`) or extends an existing router (keeping its class, e.g. `MultimodalRouter`), registers the profile's LLM as `profile:<name>`, returns the new `ModelRouterStatus` and stays in `auto` mode; once selected the route serves the turn and its cost is recorded under `profile:<name>`; a duplicate route name raises `ValueError`, a missing profile `FileNotFoundError`, and a later edit of the saved profile does not change the route. Unit: `tests/sdk/conversation/test_switch_model.py::test_add_model_router_route_wraps_plain_llm`, `::test_add_model_router_route_extends_existing_router`, `::test_add_model_router_route_rejects_duplicate_name`, `::test_add_model_router_route_rejects_missing_profile`, `::test_added_model_router_route_survives_profile_change`. Example: none. Real-LLM: none. +- `S03.model-router-switch-replaces`: `conversation.switch_profile(name)` and `switch_llm(llm)` on a router conversation install the plain LLM in place of the router (status `enabled: false`, route selection then raises `ValueError`), and the next turn runs on it; the router's own `switch_llm(route)` / `with_manual_route`, `resume_auto_routing()` and `with_route` return changed copies and leave the original alone, refusing an unknown or duplicate route with `ValueError`. Unit: `tests/sdk/llm/test_router_llm.py::test_router_manual_switch_returns_copy_and_routes_to_key`, `::test_router_with_route_rejects_duplicate_key`. Example: none. Real-LLM: none. +- `S03.model-router-before-first-run`: a route added with `add_model_router_route` and selected before the conversation's first message serves the first turn, and that turn's cost is recorded in `conversation_stats` under the route's `profile:<name>`. Unit: **none (untested)**. Example: none. Real-LLM: none. - `S03.fallback-transient`: after the primary exhausts its retries on a transient error (rate limit, timeout, connection, service unavailable, transient HTTP status) the fallback profiles are tried in order; the first success is returned; if all fail the primary's error is raised; each new call starts on the primary again. Chat and Responses, sync and async. Unit: `tests/sdk/llm/test_llm_fallback.py`. Example: `examples/01_standalone_sdk/39_llm_fallback.py`. Real-LLM: none. - `S03.fallback-skip`: non-transient errors (bad request, auth, context window) and budget denials do not fall back; quota exhaustion fails over without retrying the primary. Unit: `tests/sdk/llm/test_llm_fallback.py::test_non_transient_error_skips_fallback`, `::test_budget_denial_stops_retries_and_fallbacks`, `::test_quota_exhaustion_fails_over_without_retries`. Example: none. Real-LLM: none. - `S03.fallback-metrics`: cost and token usage of a fallback call are recorded in the caller's usage session (the primary's `llm.metrics`, and the primary's `usage_id` in `conversation_stats` when the call belongs to a conversation), and each cost entry's `model` names the model that served the call, so a fallback's entry carries the fallback model; caller kwargs and the conversation call context are forwarded to the fallback. Unit: `tests/sdk/llm/test_llm_fallback.py::test_metrics_merged_from_fallback`, `::test_fallback_forwards_caller_kwargs`, `::test_completion_fallback_receives_call_context`, `tests/sdk/llm/test_llm_fallback_sessions.py::test_fallback_preserves_caller_accounting`. Example: `examples/01_standalone_sdk/39_llm_fallback.py`. Real-LLM: none. @@ -40,7 +44,7 @@ Examples: `examples/01_standalone_sdk/05_use_llm_registry.py`, `examples/01_stan ## How to get to it (agent POV) - `LLMRegistry()` from `openhands.sdk`: `add`, `get`, `remove`, `list_usage_ids`, `subscribe`; `conversation.llm_registry` is the conversation's own instance. `conversation.conversation_stats.usage_to_metrics[usage_id]` is the conversation's ledger for a role and `llm.metrics` the LLM object's own. Covers `S03.registry-*`, `S03.agent-llm-discovery`, `S03.session-per-conversation`. -- `MultimodalRouter(usage_id=..., llms_for_routing={"primary": llm_a, "secondary": llm_b})` from `openhands.sdk.llm.router`, passed as `Agent(llm=router)`; `RandomRouter` from `openhands.sdk.llm.router.impl.random`; subclass `RouterLLM` and implement `select_llm(messages) -> key` for a custom router. Covers `S03.*router*`. +- `MultimodalRouter(usage_id=..., llms_for_routing={"primary": llm_a, "secondary": llm_b})` from `openhands.sdk.llm.router`, passed as `Agent(llm=router)`; `RandomRouter` from `openhands.sdk.llm.router.impl.random`; subclass `RouterLLM` and override `select_llm(messages) -> route name` for a custom router (the base class routes to its first route). Router state: `router.status()`, `router.selection`, `router.last_selected_route_name`, `router.active_route_name`, and the copy-returning `with_selection`, `with_manual_route` / `switch_llm(route)`, `with_auto_routing` / `resume_auto_routing()`, `with_route(name, llm)`; models `ModelRouterSelection`, `ModelRouteInfo`, `ModelRouterStatus`, `ModelRouterUpdateRequest` from `openhands.sdk.llm`. On `LocalConversation`: `switch_routed_llm(route)`, `resume_auto_llm_routing()`, `update_model_router_selection(selection)`, `add_model_router_route(profile, route_name=None, base_route_name="default")` (the agent server's `/model_router` routes and `RemoteConversation`'s methods of the same names call these; that side is `F09.model-router-*` in the agent-server map). Covers `S03.*router*`. - `LLM(..., fallback_strategy=FallbackStrategy(fallback_llms=["p1", "p2"], profile_store_dir=...))` with `FallbackStrategy` from `openhands.sdk.llm`. Covers `S03.fallback-*`. - `LLMProfileStore(base_dir=None)` from `openhands.sdk`: `save(name, llm, include_secrets=False, cipher=None)`, `load(name, cipher=None)`, `list()`, `list_summaries()`, `delete()`, `rename()`; default directory `$OH_PERSISTENCE_DIR/profiles` or `~/.openhands/profiles`. Covers `S03.profile-*`, `S03.provider-connections`. - `conversation.switch_profile(name)`, `conversation.switch_llm(llm)` on `LocalConversation`; `conversation.switch_acp_model(model)` is the ACP-agent counterpart (ACP family). Covers `S03.switch-profile`, `S03.switch-llm-object`, `S03.switch-keeps-usage`. @@ -116,6 +120,70 @@ Every model in the scripted bullets is a real `LLM` answering from its own local --check errors.empty_random eq ValidationError --check errors.valid missing control-sdk program S03/routing --arg random --check calls eq 20 --check from_a gt 0 --check from_b gt 0 ``` +- **Route selection on a router conversation (`S03.model-router-selection`).** A persisted conversation whose agent LLM is a base `RouterLLM` with routes `first` (`openai/gpt-4.1`) and `second` (`openai/gpt-4.1-mini`): an `auto` turn, `switch_routed_llm("second")` and a turn, an unknown route, `resume_auto_llm_routing()` and a turn; then `update_model_router_selection(manual second)`, close, and resume with a fresh `auto` router; refused selections and a plain-LLM conversation. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S03/model_router --arg selection \ + --check initial.enabled eq true --check initial.selection.mode eq auto --check initial.last_route_name eq null \ + --check initial.routes.0.name eq first --check initial.routes.1.model eq openai/gpt-4.1-mini --check initial.routes.1.label eq second \ + --check after_auto_turn.last_route_name eq first \ + --check after_manual.selection.manual_route_name eq second --check router_kept eq RouterLLM --check state_agent_follows eq true \ + --check after_manual_turn.last_route_name eq second \ + --check persisted.router_name eq base_router --check persisted.selection.mode eq manual --check persisted.selection.manual_route_name eq second \ + --check unknown_route_error contains "LLM route 'missing' was not found" --check after_unknown_route.manual_route_name eq second \ + --check after_resume_turn.selection.mode eq auto --check after_resume_turn.last_route_name eq first \ + --check requests.first eq 2 --check requests.second eq 1 --check costs.route-first eq 0.28 --check costs.route-second eq 0.14 \ + --check persisted_before_resume.manual_route_name eq second --check resumed_type eq RouterLLM \ + --check resumed.selection.mode eq auto --check resumed.last_route_name eq null \ + --check manual_without_route contains 'manual_route_name is required' --check auto_with_route contains 'only valid in manual' \ + --check update_request_without_selection contains 'Field required' \ + --check plain_switch_error eq 'ValueError: Current agent LLM is not a RouterLLM.' --check plain_resume_error contains 'not a RouterLLM' \ + --check plain_status.enabled eq false --check plain_status.routes len-eq 0 + ``` + Three turns, served first, second, first: the agent's LLM stays a `RouterLLM` through every selection change, and costs land under each route's own `usage_id`. `base_state.json` stores the router (`router_name`, routes, `selection`); resuming with a fresh router runs on that router's `auto` selection, because on resume the program's agent, LLM included, wins over the stored one (`AgentBase.load` only checks the agent class and tools), so a program that wants the stored pin passes a router that carries it. +- **Routes from saved profiles (`S03.model-router-add-route`).** A conversation on model A (`usage_id` `agent`) runs a turn, adds the saved profile `qa-reviewer` (model R) as route `reviewer`, runs an `auto` turn, pins `reviewer` and runs again; then a duplicate route, a missing profile and an edit of the profile; then a `MultimodalRouter` conversation and a plain one with `base_route_name="main"`. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S03/model_router --arg add-route \ + --check agent_llm_type eq RouterLLM --check returned.enabled eq true --check returned.selection.mode eq auto \ + --check returned.routes.0.name eq default --check returned.routes.0.model eq openai/gpt-4.1 \ + --check returned.routes.1.name eq reviewer --check returned.routes.1.model eq openai/gpt-4.1-mini \ + --check route_usage_ids.default eq agent --check route_usage_ids.reviewer eq profile:qa-reviewer \ + --check registry contains profile:qa-reviewer --check registry_holds_route eq true \ + --check after_auto_turn.last_route_name eq default --check after_reviewer_turn.last_route_name eq reviewer \ + --check requests.agent eq 2 --check requests.reviewer eq 1 \ + --check costs.agent eq 0.28 --check costs.profile:qa-reviewer eq 0.14 \ + --check duplicate_route_error eq 'ValueError: Model router route already exists: default' \ + --check missing_profile_error contains FileNotFoundError --check routes_after_errors len-eq 2 \ + --check route_model_after_profile_edit eq openai/gpt-4.1-mini \ + --check multimodal_type eq MultimodalRouter --check multimodal_routes.2 eq qa-reviewer --check multimodal_routes len-eq 3 \ + --check named_base_routes.0 eq main --check named_base_routes.1 eq qa-reviewer + ``` + Adding a route does not change what runs: the `auto` turn still goes to `default` (the wrapped model A, which keeps its `usage_id`); after the pin, R serves the turn and its 0.14 is recorded under `profile:qa-reviewer`. The route is a snapshot of the profile at the time it was added, `route_name` defaults to the profile name, and `base_route_name` is used only when a plain LLM is wrapped. +- **Switching away from a router (`S03.model-router-switch-replaces`).** Two conversations on a `RouterLLM` pinned to `second`: one `switch_profile("qa-plain-b")`, one `switch_llm` to model C, each followed by a turn; then the router's own switch methods. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S03/model_router --arg switch-replaces \ + --check after_switch_profile.type eq LLM --check after_switch_profile.usage_id eq profile:qa-plain-b \ + --check after_switch_profile.status.enabled eq false --check select_after_switch_profile contains 'not a RouterLLM' \ + --check after_switch_llm.type eq LLM --check after_switch_llm.model eq openai/gpt-4.1-nano --check after_switch_llm.status.enabled eq false \ + --check requests.b eq 1 --check requests.c eq 1 --check requests.routes eq 0 \ + --check router_switch_llm.copy eq true --check router_switch_llm.original.mode eq auto \ + --check router_switch_llm.copy_selection.manual_route_name eq second --check router_switch_llm.active_route_name eq second \ + --check router_switch_llm.resumed.mode eq auto \ + --check router_switch_unknown contains "LLM route 'missing' was not found" \ + --check router_with_route_duplicate contains 'already exists: first' + ``` + `switch_profile` and `switch_llm` replace the whole agent LLM, router included: the routes and the pin are gone and the turn runs on the switched model, never on a route. To change routes and keep the router, use the route-selection methods above. +- **A route chosen before the first message (`S03.model-router-before-first-run`).** A conversation on an endpoint that answers nothing adds the saved profile `qa-early-route` as route `early` and pins it before its first message, then runs one turn. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S03/model_router --arg before-first-run \ + --check status eq finished --check requests.base eq 0 --check requests.route eq 1 --check last_route_name eq early \ + --check costs.profile:qa-early-route eq 0.14 + ``` + The turn finishes on the `early` route (the base endpoint gets no request). Expected: its 0.14 is recorded in `conversation_stats` under `profile:qa-early-route`, as it is when the route is added after the first message (`S03.model-router-add-route`). + Known failure (reproduced 2026-11-19, [#5639](https://github.com/OpenHands/software-agent-sdk/issues/5639)): `costs` is `{}`; the route's LLM is registered before the conversation subscribes its stats to the registry on the first message, so the turn's cost never reaches `conversation_stats`. - **Fallback (`S03.fallback-transient`, `S03.fallback-skip`, `S03.fallback-metrics`, `S03.fallback-profile-resolution`).** A rate-limited primary with two fallback profiles (the first also rate-limited; the second saved after the primary was built), then a bad request, a budget denial, quota exhaustion, every model failing, the Responses path and `acompletion`. ```sh export SDK_VERIFY_RUN=$(control-sdk new --field run) @@ -280,5 +348,6 @@ Every model in the scripted bullets is a real `LLM` answering from its own local - Fallback only triggers after the primary's own retries are exhausted (`num_retries=5`, backoff up to 64 s), so forcing a fallback by pointing the primary at a dead endpoint is slow; lower `num_retries` and `retry_*_wait` on the primary. - `OH_PERSISTENCE_DIR` is read into module-level constants at import time; setting it after `import openhands.sdk` has no effect on the default profile directory. - An LLM added to `conversation.llm_registry` before the first `send_message` is not tracked in `conversation_stats` (the stats subscribe to the registry when the agent is initialized, with no replay); add extra LLMs after the first message. +- A `RouterLLM`'s own `model` is a placeholder (its `router_name`, e.g. `base_router` or `multimodal_router`) and its `usage_id` is the router's, not a route's; a route keeps its LLM's `usage_id` (`profile:<name>` for an added profile), so costs and `conversation_stats` are per route, never under the router. `router.status().last_route_name` is only set by a call, and `active_llm` falls back to it in `auto` mode. - `LLMProfileStore.list()` returns file names with the `.json` suffix, while `load`/`delete`/`switch_profile` take the bare name. - `prompt_cache_key` is only sent for models LiteLLM marks as supporting it (sent for `gpt-4.1`, not for `gpt-4o-mini`), so a switch between such models changes whether the key appears at all. diff --git a/.agents/skills/verify-sdk/references/feature-map/S04-local-conversation.md b/.agents/skills/verify-sdk/references/feature-map/S04-local-conversation.md index 6101be3d90c..54982621d57 100644 --- a/.agents/skills/verify-sdk/references/feature-map/S04-local-conversation.md +++ b/.agents/skills/verify-sdk/references/feature-map/S04-local-conversation.md @@ -2,7 +2,7 @@ A Python program calls `Conversation(agent=..., workspace=...)` with a local workspace (a path string or `LocalWorkspace`) and gets a `LocalConversation` that runs the agent in the same process. Around the step loop (family S01) it owns the conversation's lifecycle: a stable `conversation.id`, optional on-disk persistence under `persistence_dir/<id.hex>/` (`base_state.json` plus one `events/event-NNNNN-<event_id>.json` per event) that a second `Conversation(..., persistence_dir=..., conversation_id=same_id)` reloads and continues, `pause()` from another thread, `arun()` on an asyncio loop with `interrupt()` cancelling mid-LLM-call, `update_secrets(...)` that inject values as environment variables into terminal commands and mask them in output, a pluggable visualizer (Rich console output by default, `visualizer=None` for silence), `fork()` into an independent copy or a branch from a chosen event, and `close()`. Helpers next to it read results out of a finished conversation: `get_agent_final_response(events)`, `conversation.conversation_stats` (per-`usage_id` LLM metrics), `generate_title()`, and `run_goal(...)`, which re-prompts the agent until a second judge LLM agrees an objective is complete, and stops with `GoalAsked` when the agent's `ask_user` question awaits the user. -Source: `openhands-sdk/openhands/sdk/conversation/conversation.py`, `openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py`, `openhands-sdk/openhands/sdk/conversation/base.py`, `openhands-sdk/openhands/sdk/conversation/state.py`, `openhands-sdk/openhands/sdk/conversation/persistence_const.py`, `openhands-sdk/openhands/sdk/conversation/cancellation.py`, `openhands-sdk/openhands/sdk/conversation/secret_registry.py`, `openhands-sdk/openhands/sdk/conversation/visualizer/`, `openhands-sdk/openhands/sdk/conversation/goal/`, `openhands-sdk/openhands/sdk/conversation/response_utils.py`, `openhands-sdk/openhands/sdk/conversation/conversation_stats.py`, `openhands-sdk/openhands/sdk/conversation/title_utils.py`, `openhands-sdk/openhands/sdk/conversation/types.py`, `openhands-sdk/openhands/sdk/secret/`, `openhands-sdk/openhands/sdk/utils/async_utils.py` +Source: `openhands-sdk/openhands/sdk/conversation/conversation.py`, `openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py`, `openhands-sdk/openhands/sdk/conversation/base.py`, `openhands-sdk/openhands/sdk/conversation/state.py`, `openhands-sdk/openhands/sdk/conversation/persistence_const.py`, `openhands-sdk/openhands/sdk/conversation/cancellation.py`, `openhands-sdk/openhands/sdk/conversation/secret_registry.py`, `openhands-sdk/openhands/sdk/conversation/visualizer/`, `openhands-sdk/openhands/sdk/conversation/goal/`, `openhands-sdk/openhands/sdk/conversation/resolution.py`, `openhands-sdk/openhands/sdk/conversation/response_utils.py`, `openhands-sdk/openhands/sdk/conversation/conversation_stats.py`, `openhands-sdk/openhands/sdk/conversation/title_utils.py`, `openhands-sdk/openhands/sdk/conversation/types.py`, `openhands-sdk/openhands/sdk/secret/`, `openhands-sdk/openhands/sdk/utils/async_utils.py` Needs: `llm`, `tmux`, `network`, `uvx` @@ -41,6 +41,9 @@ Examples: `examples/01_standalone_sdk/06_interactive_terminal_w_reasoning.py`, ` - `S04.final-response`: `get_agent_final_response(conversation.state.events)` returns the text of whichever came last of the agent's `finish` call and its plain-text `MessageEvent`, else `""`. Unit: `tests/sdk/conversation/test_agent_final_response.py`. Example: none. Real-LLM: `tests/integration/tests/t08_image_file_viewing.py`. - `S04.stats`: `conversation.conversation_stats` keeps `Metrics` per LLM `usage_id`, combines them with `get_combined_metrics()`, restores them on resume, and stats update events carry a snapshot rather than the full history. Unit: `tests/sdk/conversation/test_conversation_stats.py`, `tests/sdk/conversation/test_stats_update_event_snapshot.py`. Example: `examples/01_standalone_sdk/11_async.py` (prints cost). Real-LLM: none. - `S04.title-and-tags`: `generate_title(llm=None, max_length=50)` asks an LLM (the agent's by default) for a title from the first user message, falls back to truncation on error, and strips reasoning text; `tags={...}` keys must be lowercase alphanumeric. Unit: `tests/sdk/conversation/test_generate_title.py`, `tests/sdk/conversation/test_tags.py`. Example: `examples/01_standalone_sdk/48_conversation_fork.py` (`title=`). Real-LLM: none. +- `S04.conversation-sender`: `conversation_sender(conversation_id)` (`openhands.sdk.conversation.types`) is `CONVERSATION_SENDER_PREFIX` (`"conversation:"`) plus the id's hex; `send_message(text, sender=...)` records it on the saved user `MessageEvent`, a plain `send_message` records no sender, and both survive a reload from `persistence_dir`. Unit: **none (untested)** in the SDK; `tests/agent_server/test_event_router.py` covers the server's use of it. Example: none. Real-LLM: none. +- `S04.resolution-verdict`: `verdict_from_judgments(judgments)` (`openhands.sdk.conversation.resolution`) turns the seven yes/no judgments of `RESOLUTION_QUESTIONS` into a `ResolutionVerdict`: `owed` `nothing` (resolved) only when `handled` and `task_done` are both at least 0.85 and every hand-off stays below 0.2; otherwise the strongest hand-off at or above 0.5 (`answer`, `review`, `decide`, `check`), else `unfinished` when the work is not done, else `unclear`; `probability` is `min(handled, task_done, 1 - strongest hand-off)`, `needs_user` is true only for a question, review, choice or check (not a reported problem), and judgments are clamped to 0..1, a non-numeric one counting as 0. Unit: `tests/sdk/conversation/test_resolution.py`. Example: none. Real-LLM: none. +- `S04.resolution-judge`: `resolution_judge(fallback_llm, LLMProfileStore(...), cipher=None)` loads the saved LLM profile `resolution_judge`, first saving a copy of the fallback LLM under that name (with its key, encrypted on disk when a cipher is given) when the store has none; an existing profile wins over a later fallback. The returned `LLMDecisionModel` runs on that profile with a 60 s timeout and no LLM-level retries (`usage_id` `resolution_judge`), and `await judge_resolution(judge, ResolutionState(...))` sends the turn and every question in one chat request and returns the verdict with the profile's model; a reply that is not JSON raises `ValueError` after one request. Unit: `tests/sdk/conversation/test_resolution.py`. Example: none. Real-LLM: none. ## How to get to it (agent POV) @@ -51,6 +54,7 @@ Examples: `examples/01_standalone_sdk/06_interactive_terminal_w_reasoning.py`, ` - `openhands.sdk.conversation.visualizer.ConversationVisualizerBase`, `DefaultConversationVisualizer`: `S04.visualizer-*`. - `conversation.fork(conversation_id=, agent=, title=, tags=, reset_metrics=True, from_event_id=)`, `conversation.navigate_to(event_id)`: `S04.fork`, `S04.fork-branch`; `LocalConversation(..., cipher=Cipher(key))` (`openhands.sdk.utils.cipher`; the `Conversation` factory does not take a cipher): `S04.fork-keeps-secrets`. - `openhands.sdk.conversation.goal.run_goal`, `GoalController`, `judge_goal`: `S04.goal-loop`; `GoalAsked` (a round that ended on an `ask_user` question, `openhands.tools.ask_user`, see S16): `S04.goal-asked`, `S04.goal-asked-branch`. +- `openhands.sdk.conversation.types.conversation_sender`, `CONVERSATION_SENDER_PREFIX`, `conversation.send_message(..., sender=...)`: `S04.conversation-sender`. `openhands.sdk.conversation.resolution` (`verdict_from_judgments`, `resolution_judge`, `judge_resolution`, `ResolutionState`, `ResolutionVerdict`, `LLMDecisionModel`, `RESOLUTION_JUDGE_PROFILE_NAME`): `S04.resolution-verdict`, `S04.resolution-judge`; the profile lives in an `LLMProfileStore` directory (see S03). - `openhands.sdk.conversation.response_utils.get_agent_final_response`: `S04.final-response`. `conversation.conversation_stats` / `conversation.state.stats`: `S04.stats`. `conversation.generate_title()`: `S04.title-and-tags`. `conversation.close()`: `S04.close`. - Env: `LLM_API_KEY` (required), `LLM_MODEL` (default `gpt-5.5`; `anthropic/claude-sonnet-4-5-20250929` in example 50), `LLM_BASE_URL`; example 50 also reads `AUTO_CANCEL_SECONDS` (default 8). @@ -335,6 +339,46 @@ Scripted bullets use the SDK's `TestLLM` (`--script`, or `make_llm(script=...)` --check first_run.agent_cost eq 0.28 --check first_run.secondary_cost eq 0.14 --check first_run.combined_cost eq 0.42 \ --check resumed.restored_agent_cost eq 0.28 --check resumed.agent_cost_after eq 0.42 --check resumed.status eq finished ``` +- **A message from another conversation (`S04.conversation-sender`).** Two persisted conversations; the second gets one plain message and one sent with `sender=conversation_sender(first.id)`, then is closed and reopened from `persistence_dir`. Scripted: nothing runs. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S04/sender \ + --check prefix eq "conversation:" --check value_is_prefix_plus_hex eq true --check hex_parses eq true \ + --check in_memory len-eq 2 --check in_memory.0.text eq QA_S04_FROM_USER --check in_memory.0.sender eq null \ + --check in_memory.1.text eq QA_S04_FROM_CONVERSATION --check in_memory.1.sender matches '^conversation:[0-9a-f]{32}$' \ + --check after_reload len-eq 2 --check after_reload.0.sender eq null \ + --check after_reload.1.sender matches '^conversation:[0-9a-f]{32}$' + ``` + Expected: the helper's value is `conversation:` plus the sender's hex id; only the relayed message carries it, in memory and after the reload. +- **Resolution verdicts (`S04.resolution-verdict`).** Fixed judgment sets through `verdict_from_judgments`, no model. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S04/resolution --arg verdicts \ + --check questions eq '["asks_to_check", "asks_user", "handled", "hands_off_review", "offers_choice", "reports_problem", "task_done"]' \ + --check done.owed eq nothing --check done.resolved eq true --check done.probability eq 0.9 --check done.note eq "0.90 nothing" \ + --check asks.owed eq answer --check asks.needs_user eq true --check asks.probability eq 0.2 \ + --check review.owed eq review --check review.needs_user eq true \ + --check unfinished.owed eq unfinished --check unfinished.needs_user eq false --check unfinished.probability eq 0.4 \ + --check unclear.owed eq unclear --check unclear.resolved eq false --check unclear.probability eq 0.7 \ + --check problem.owed eq check --check problem.needs_user eq false \ + --check out_of_range.owed eq unfinished --check out_of_range.probability eq 0.0 + ``` + Expected: all done with low hand-offs resolves (probability the weakest signal, 0.9); a question, a review, a choice or a check is owed when its judgment holds; unfinished work with no hand-off is `unfinished`; a weak hand-off (0.3) on finished work is `unclear`; a reported problem owes a `check` without needing the user; a non-numeric `task_done` counts as 0. +- **The judge on its saved profile (`S04.resolution-judge`).** An empty profile store and a fallback LLM pointed at a local OpenAI-compatible endpoint (`programs/S16/fake_openai.py`) that answers with fixed JSON (the agent asks the user); then a second fallback, a cipher store, and an endpoint that answers in prose. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S04/resolution --arg judge \ + --check profiles_before len-eq 0 --check profiles_after eq '["resolution_judge.json"]' \ + --check saved.model eq openai/qa-judge-a --check saved.api_key_kept eq true \ + --check judge.usage_id eq resolution_judge --check judge.llm_timeout eq 60 --check judge.llm_num_retries eq 0 \ + --check verdict.owed eq answer --check verdict.needs_user eq true --check verdict.resolved eq false \ + --check verdict.model eq openai/qa-judge-a --check verdict.judgments len-eq 7 \ + --check request.has_state eq true --check request.names_questions eq true \ + --check reused_model eq openai/qa-judge-a \ + --check cipher.plain_key_on_disk eq false --check cipher.loaded_key_matches eq true \ + --check bad_reply_error contains "did not reply with JSON" --check bad_reply_requests eq 1 + ``` + Expected: the first call saves `resolution_judge.json` from the fallback (model and key), the judge runs on it with a 60 s timeout and no LLM retries, and one request carrying the turn and all seven question names yields `owed` `answer` from the profile's model; a later fallback (`qa-judge-b`) does not replace the saved profile; with a cipher the key is not in the file in plain text but loads back; a prose reply raises `ValueError` after one request. - **A real model across a resume (`S04.factory`, `S04.resume`, `S04.final-response`).** Live; each `control-sdk` command reopens the conversation from disk. ```sh export SDK_VERIFY_RUN=$(control-sdk new --llm deepseek-flash --field run) @@ -370,6 +414,7 @@ Scripted bullets use the SDK's `TestLLM` (`--script`, or `make_llm(script=...)` ## Gotchas +- `RemoteConversation.send_message(..., sender=...)` does not record the sender: the server saves the message with `sender: null` ([#7004](https://github.com/OpenHands/software-agent-sdk/issues/7004)); `S04.conversation-sender` holds for local conversations only. Over REST only a conversation sender is recorded (agent-server `F06.send-sender-conversation`). - `run_goal` and `GoalController.on_run_finished` look for a pending `ask_user` question in the whole event log, not the active branch, so a question left on an abandoned branch still stops a goal ([#6907](https://github.com/OpenHands/software-agent-sdk/issues/6907), `S04.goal-asked-branch`). - `examples/01_standalone_sdk/06_interactive_terminal_w_reasoning.py` is in `_EXCLUDED_EXAMPLES` in `tests/examples/test_examples.py`, so the `test-examples` label never runs it; it also prints no `EXAMPLE_COST:` line, which that harness would require. - Examples run under the harness with `uv run pytest tests/examples/test_examples.py --run-examples -k <name>`; without `--run-examples` every case is skipped. diff --git a/.agents/skills/verify-sdk/references/feature-map/S06-context-condensers.md b/.agents/skills/verify-sdk/references/feature-map/S06-context-condensers.md index 211c8467de3..aff352b3d53 100644 --- a/.agents/skills/verify-sdk/references/feature-map/S06-context-condensers.md +++ b/.agents/skills/verify-sdk/references/feature-map/S06-context-condensers.md @@ -160,6 +160,15 @@ The programs under `programs/S06/` record every scripted LLM call (`recorder.py` --check summary_calls_soft eq 0 --check condensations_after_request eq 1 \ --check view_kinds_after_request eq '["SystemPromptEvent", "CondensationSummaryEvent"]' --check hard_offset eq 1 ``` +- **A summary with no text (`S06.summarizing-size`, `S06.soft-vs-hard`).** `LLMSummarizingCondenser(max_size=10, keep_first=2)` over nine scripted terminal steps while the summary LLM answers every call with no text (`""`); then the same with only whitespace (`" "`). + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S06/condense --arg empty-summary \ + --check empty.status eq finished --check empty.summary_calls ge 1 \ + --check empty.empty_summary_condensations eq 0 --check empty.forgotten eq 0 \ + --check blank.status eq finished --check blank.empty_summary_condensations eq 0 --check blank.forgotten eq 0 + ``` + Expected: size pressure is a soft requirement, so with no usable summary the view stays as it was (nothing forgotten, no `Condensation`), as when there is nothing worth forgetting. Known failure (reproduced 2026-11-19, [#6957](https://github.com/OpenHands/software-agent-sdk/issues/6957)): both runs record two condensations whose summary is `""` (or `" "`) and forget 15 events between them; `LLMSummarizingCondenser` takes any successful reply's first text block as the summary. The issue's trigger is an empty reply with `finish_reason="length"`; the scripted LLM answers with `finish_reason="stop"`, which the condenser does not read either. - **Hard context reset (`S06.hard-reset`).** The summary LLM fails on the normal path and on the first reset attempt, then answers; with every attempt failing, `condense()` raises and nothing is forgotten. ```sh export SDK_VERIFY_RUN=$(control-sdk new --field run) diff --git a/.agents/skills/verify-sdk/references/feature-map/S09-mcp.md b/.agents/skills/verify-sdk/references/feature-map/S09-mcp.md index d72d1deaacb..64020958edc 100644 --- a/.agents/skills/verify-sdk/references/feature-map/S09-mcp.md +++ b/.agents/skills/verify-sdk/references/feature-map/S09-mcp.md @@ -18,10 +18,10 @@ Examples: `examples/01_standalone_sdk/07_mcp_integration.py`, `examples/01_stand - `S09.oauth`: `auth=MCPOAuthAuthCredential(strategy="oauth2", authentication=MCPOAuthAuthentication(...))` builds a FastMCP OAuth client with the given scopes, client id/secret or metadata URL and an optional token store or factory; an expired token is refreshed against the token endpoint discovered from server metadata, falling back to `<origin>/token`, and a valid token is sent without discovery. Unit: `tests/sdk/mcp/test_oauth.py`, `tests/sdk/mcp/test_create_mcp_tool.py` (`test_prepare_mcp_config_*oauth*`). Example: `examples/01_standalone_sdk/08_mcp_with_oauth.py`. Real-LLM: none. - `S09.enabled-flag`: `MCPServer(enabled=False)` keeps a server configured but never connects to it; if every server is disabled `create_mcp_tools` raises and the conversation skips MCP entirely. Unit: `tests/sdk/mcp/test_create_mcp_tool.py` (`test_create_mcp_tools_skips_disabled_servers`, `test_create_mcp_tools_all_servers_disabled`), `tests/sdk/conversation/test_local_conversation_mcp.py::test_disabling_every_server_skips_the_mcp_connection`. Example: none. Real-LLM: none. - `S09.listing-timeout`: tool listing that exceeds `timeout` (30 s in a conversation) raises `MCPTimeoutError` naming the servers and closes the client; an unreachable server raises. Unit: `tests/sdk/mcp/test_create_mcp_tool.py` (`test_create_mcp_tools_timeout_error_message`, `test_create_mcp_tools_connection_to_nonexistent_server`). Example: none. Real-LLM: none. -- `S09.startup-failure-tolerated`: when MCP startup fails inside a `LocalConversation`, a warning is logged and the conversation runs with its other tools. Unit: `tests/sdk/conversation/test_local_conversation_mcp.py::test_mcp_startup_failure_does_not_abort_conversation`. Example: none. Real-LLM: none. +- `S09.startup-failure-tolerated`: when MCP startup fails inside a `LocalConversation`, a warning is logged and the conversation runs with its other tools; a single server that advertises a built-in tool's name (`terminal`) gets that tool as `<server>_terminal` while the built-in and the server's other tools stay (known failure #6992: initialization raises `ValueError: Duplicate tool names found: {'terminal'}`). Unit: `tests/sdk/conversation/test_local_conversation_mcp.py::test_mcp_startup_failure_does_not_abort_conversation`. Example: none. Real-LLM: none. - `S09.schema-conversion`: an MCP `inputSchema` becomes the tool's OpenAI and Responses parameters with nested objects, arrays, required lists and descriptions preserved, plus `summary` and (with a security analyzer) `security_risk`; MCP annotations are kept. Unit: `tests/sdk/mcp/test_mcp_nested_schema.py`, `tests/sdk/mcp/test_mcp_security_risk.py`, `tests/sdk/mcp/test_mcp_tool.py`. Example: none. Real-LLM: none. - `S09.argument-validation`: `action_from_arguments` validates the model's arguments against the schema, drops `None` values and the OpenHands `kind` discriminator before the call, keeps a schema property literally named `kind`, and raises on invalid input; `security_risk` and `summary` are removed by the agent before it calls `action_from_arguments`, which rejects them as extra inputs when given directly. Unit: `tests/sdk/mcp/test_mcp_tool_validation.py`, `tests/sdk/mcp/test_mcp_tool_kind_field.py`, `tests/sdk/mcp/test_mcp_action_serialization.py`. Example: none. Real-LLM: none. -- `S09.observation`: a `CallToolResult` becomes an `MCPToolObservation` with text and image content, `isError` mapped to `is_error`, structured JSON rendered as text; an exception or timeout (300 s per call) becomes an error observation. Unit: `tests/sdk/mcp/test_mcp_tool.py`, `tests/sdk/mcp/test_mcp_observation.py`. Example: none. Real-LLM: none. +- `S09.observation`: a `CallToolResult` becomes an `MCPToolObservation` with text and image content, `isError` mapped to `is_error`, structured JSON rendered as text, and the text of an `EmbeddedResource` block and a result that has only `structuredContent` kept (known failure #6993: both are dropped, leaving only `[Tool '<name>' executed.]`); an exception or timeout (300 s per call) becomes an error observation. Unit: `tests/sdk/mcp/test_mcp_tool.py`, `tests/sdk/mcp/test_mcp_observation.py`. Example: none. Real-LLM: none. - `S09.secret-refs-in-calls`: `$VAR`, `${VAR}` and `${VAR:-default}` in a call's arguments are expanded from the conversation's secret registry (never from the host environment), and secret values in the result text are masked. Unit: `tests/sdk/mcp/test_mcp_secret_expansion.py`. Example: none. Real-LLM: none. - `S09.session-reuse`: all calls go through one client connection (one subprocess for a stdio server), so what the server keeps per process persists across calls and tools, and closing the client releases the connection. What the server keeps per MCP session (counters, auth tokens keyed by session id) persists only on a handshake connection (`S09.protocol-negotiation`), where every call carries the same session id; on a `server/discover` connection there is no session and the server sees a different session id on every call. Unit: `tests/sdk/mcp/test_mcp_session_persistence.py`, `tests/sdk/mcp/test_stateful_mcp.py`. Example: none. Real-LLM: none. - `S09.protocol-negotiation`: `MCPClient` connects with FastMCP `mode="auto"`: it probes `server/discover` and uses the sessionless protocol when the server answers it (stdio and streamable HTTP to a current FastMCP server; `client.initialize_result` is `None`), and falls back to the `initialize` handshake when the server does not (`client.initialize_result` is set); SSE always uses the handshake. Unit: `tests/sdk/mcp/test_create_mcp_tool.py::test_create_mcp_tools_http_server` (`mode == "auto"`, `initialize_result is None`), `tests/sdk/mcp/test_stateful_mcp.py` (SSE keeps a session). Example: none. Real-LLM: none. @@ -31,7 +31,7 @@ Examples: `examples/01_standalone_sdk/07_mcp_integration.py`, `examples/01_stand - `S09.config-var-expansion`: `${VAR}` and `${VAR:-default}` placeholders in `Agent.mcp_config` (and Claude Code plugin `.mcp.json`) are expanded from conversation secrets before servers start. Unit: `tests/sdk/skills/test_mcp_config_expansion.py`, `tests/sdk/conversation/test_local_conversation_plugins.py::TestPluginMcpSecretsExpansion`. Example: none. Real-LLM: none. - `S09.tool-serialization`: `MCPToolDefinition` is frozen, serializes polymorphically with `kind`, is written in the MCP wire spelling (`inputSchema`) and loads either spelling, so conversations persisted under older `mcp` majors resume. Unit: `tests/sdk/mcp/test_mcp_tool_serialization.py`, `tests/sdk/mcp/test_mcp_tool_immutability.py`, `tests/sdk/mcp/test_mcp_wire_compat.py`. Example: none. Real-LLM: none. -- `S09.runtime-add-servers`: `LocalConversation.add_mcp_servers({name: MCPServer})` on an initialized agent connects each enabled server before it returns and registers its tools as `<server>_<tool>` (whatever the server count), adds the servers to `agent.mcp_config` and the persisted state, and the next LLM call offers them; before the first message it only records them and they connect with the agent like configured servers (bare names for a single server). A duplicate name and `transport="sse"` raise `ValueError`, a server that cannot start raises its connection error, nothing refused is added, and a disabled server is recorded without connecting. Unit: `tests/sdk/conversation/test_local_conversation_mcp.py` (`test_add_mcp_servers_registers_modern_runtime_tools`, `test_add_mcp_servers_rejects_sse_transport`, `test_add_mcp_servers_accepts_running_conversation`). Example: none. Real-LLM: none. +- `S09.runtime-add-servers`: `LocalConversation.add_mcp_servers({name: MCPServer})` on an initialized agent connects each enabled server before it returns and registers its tools as `<server>_<tool>` (whatever the server count), adds the servers to `agent.mcp_config` and the persisted state, and the next LLM call offers them; before the first message it only records them and they connect with the agent like configured servers (bare names for a single server). A duplicate name and `transport="sse"` raise `ValueError`, a server that cannot start raises `MCPServerStartError` (an `MCPError` from `openhands.sdk.mcp`, with `server_name` and the client's connection `MCPError` as `__cause__`; message `Could not start MCP server <name>: <reason>`) after closing the servers it already started, nothing refused is added, and a disabled server is recorded without connecting. Unit: `tests/sdk/conversation/test_local_conversation_mcp.py` (`test_add_mcp_servers_registers_modern_runtime_tools`, `test_add_mcp_servers_rejects_sse_transport`, `test_add_mcp_servers_accepts_running_conversation`). Example: none. Real-LLM: none. - `S09.runtime-add-resume`: a conversation reopened from its persistence directory reconnects the servers added at runtime and the agent can call their tools by the names it was given (known failure: with one server they come back unprefixed, so `verify_echo` is `Tool ... not found`). Unit: **none (untested)**. Example: none. Real-LLM: none. - `S09.startup-budget`: `LocalConversation(mcp_startup_budget=seconds)` lets agent initialization (the first `send_message`) return after the budget while slower servers keep starting; their tools join the agent when ready, the first step of `run()` waits for them (up to 30 s each), and one that fails late is logged and skipped. Without a budget (the default; the agent server passes 1.0) initialization waits for every server. Unit: `tests/sdk/conversation/test_local_conversation_mcp.py` (`test_startup_budget_starts_the_agent_before_a_slow_mcp_server`, `test_late_mcp_tools_are_closed_when_the_conversation_closed_first`, `test_first_step_waits_for_mcp_servers_that_missed_the_budget`). Example: none. Real-LLM: none. @@ -52,7 +52,7 @@ Preconditions: - `uv sync --dev` has run (provides `fastmcp` and `uvicorn`); `control-sdk` is on `PATH` (`export PATH="$PWD/.agents/skills/verify-sdk/scripts:$PATH"`). `sdk-map run` does both. - Live bullets: `DEEPSEEK_API_KEY`; example 07 also needs `uvx`, `npx` and network. -Every server here is `programs/S09/mcp_server.py`, a small FastMCP server the programs start over stdio, streamable HTTP or SSE on a free local port (tools: `echo`, `add`, `getenv`, `reverse`, `counter`, `session`, `nested`, `kind_echo`, `fail`, `picture`, `structured`, `slow`, `headers`, `die`, `register_extra`, `remove_extra`, `register_late`; options for OAuth endpoints, a slow or failing start, a transient 503, and an HTTP server without `server/discover`). Conversation programs record each scripted LLM call's tool schemas (`recorder.py`). +Every server here is `programs/S09/mcp_server.py`, a small FastMCP server the programs start over stdio, streamable HTTP or SSE on a free local port (tools: `echo`, `add`, `getenv`, `reverse`, `counter`, `session`, `nested`, `kind_echo`, `fail`, `picture`, `structured`, `slow`, `headers`, `die`, `register_extra`, `remove_extra`, `register_late`; with `--payload-tools` also `resource` and `structured_only`, with `--shadow-terminal` `terminal` and `sibling_check`; options for OAuth endpoints, a slow or failing start, a transient 503, and an HTTP server without `server/discover`). Conversation programs record each scripted LLM call's tool schemas (`recorder.py`). - **An MCP server inside a conversation (`S09.stdio-server`, `S09.schema-conversion`, `S09.argument-validation`, `S09.observation`, `S09.secret-refs-in-calls`, `S09.session-reuse`, `S09.tools-list-changed`, `S09.config-var-expansion`, `S09.config-secrets`).** `Agent(mcp_config={"verify": MCPServer(command=python, args=[mcp_server.py, "stdio"], env={"TOKEN": "${MY_TOKEN}", "OTHER": "${UNSET_VAR:-dflt-other}"})})`, conversation secrets `API_KEY` and `MY_TOKEN`, and `HOST_ONLY` set only in the program's environment. The scripted agent calls each tool, passes `summary`/`security_risk` on one call, references secrets as `$API_KEY`, `${API_KEY}` and `${MISSING:-dflt}`, and registers, calls and removes a tool at runtime. ```sh @@ -92,6 +92,15 @@ Every server here is `programs/S09/mcp_server.py`, a small FastMCP server the pr --check only_broken_error missing --check only_broken_status eq finished \ --check only_broken_tools eq '["finish"]' ``` +- **A single server that shadows a built-in tool (`S09.startup-failure-tolerated`).** `programs/S09/conversation.py shadow`: `Agent(tools=[Tool(name="terminal")], mcp_config={"ext": ...})` where `ext` (`--shadow-terminal`) advertises `terminal` and `sibling_check`; the scripted agent runs the built-in terminal, then `sibling_check`. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S09/conversation --arg shadow \ + --check error missing --check status eq finished \ + --check tools contains terminal --check tools contains ext_terminal --check tools contains sibling_check \ + --check texts.0 contains builtin-terminal --check texts.1 contains "sibling ok" + ``` + Expected: the colliding MCP tool is registered as `ext_terminal`, the built-in `terminal` and the bare `sibling_check` stay, and both calls return their output. Known failure (reproduced 2026-11-19, [#6992](https://github.com/OpenHands/software-agent-sdk/issues/6992)): `send_message` raises `ValueError: Duplicate tool names found: {'terminal'}` (single-server tools are never prefixed, so `add_runtime_tools` refuses the second `terminal`); the conversation stays `idle` with only `terminal` and `finish`, and no tool call runs (`error` is set, `texts` is empty). - **Disabled servers (`S09.enabled-flag`).** A conversation whose only server is disabled (its command would write a marker file), and `create_mcp_tools` with everything disabled or a disabled server beside an enabled one. ```sh export SDK_VERIFY_RUN=$(control-sdk new --field run) @@ -203,6 +212,14 @@ Every server here is `programs/S09/mcp_server.py`, a small FastMCP server the pr --check legacy_key_rewritten eq true --check legacy_spelling_loads eq true --check frozen eq true \ --check kind_in_dump eq MCPToolDefinition ``` +- **Embedded resources and structured-only results (`S09.observation`).** `programs/S09/conversation.py payload`: one server with `--payload-tools`; the scripted agent calls `resource` (a single `EmbeddedResource` whose text resource holds `{"payload": "VERIFY_RESOURCE_TEXT"}`) and `structured_only` (no content blocks, `structuredContent` `{"payload": "VERIFY_STRUCTURED"}`). + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S09/conversation --arg payload \ + --check error missing --check status eq finished \ + --check resource_text contains VERIFY_RESOURCE_TEXT --check structured_only_text contains VERIFY_STRUCTURED + ``` + Expected: each `ObservationEvent` carries the tool's payload, the resource's text and the structured content, so the model sees it. Known failure (reproduced 2026-11-19, [#6993](https://github.com/OpenHands/software-agent-sdk/issues/6993)): both observations are only `[Tool 'resource' executed.]` and `[Tool 'structured_only' executed.]`; `MCPToolObservation.from_call_tool_result` keeps only text and image blocks (logging `Unsupported MCP content block type ... Ignoring.`) and never reads `structured_content`. The run itself finishes. - **A real model calls MCP tools (`S09.stdio-server`, `S09.schema-conversion`, `S09.secret-refs-in-calls`, `S09.observation`).** Live, against the local stdio server. ```sh export SDK_VERIFY_RUN=$(control-sdk new --llm deepseek-flash --field run) @@ -223,14 +240,16 @@ Every server here is `programs/S09/mcp_server.py`, a small FastMCP server the pr --check ready_mcp_config eq '["verify"]' --check ready_state_mcp_config eq '["verify"]' \ --check duplicate_error eq "ValueError: MCP server already exists: ['verify']" \ --check sse_error eq "ValueError: Runtime MCP additions do not support SSE transport: remote" \ - --check start_failure_error contains "Connection closed" --check disabled_error eq null \ + --check start_failure_error eq "MCPServerStartError: Could not start MCP server broken: Connection closed" \ + --check start_failure_is_mcp_error eq true --check start_failure_server_name eq broken --check start_failure_cause eq MCPError \ + --check disabled_error eq null \ --check mcp_config_after_refusals eq '["off", "verify"]' \ --check ready_status eq finished --check ready_first_call_tools contains verify_echo --check ready_texts.0 contains "echo: added" \ --check early_mcp_config eq '["solo"]' --check early_status eq finished \ --check early_first_call_tools contains echo --check early_first_call_tools not-contains solo_echo --check early_texts.0 contains "echo: early" ``` - The add on a ready agent connects at once (tools listed right after the call, prefixed although `verify` is the only server); the duplicate, the SSE server and a server that exits at startup raise and leave `mcp_config` with only `verify` and the disabled `off`; the server added before the first message is connected with the agent under bare names. -- **Reopening a conversation with added servers (`S09.runtime-add-resume`).** Known failure (reproduced 2026-11-17, [#6927](https://github.com/OpenHands/software-agent-sdk/issues/6927), fix in [#6929](https://github.com/OpenHands/software-agent-sdk/pull/6929)): the reopened conversation offers the tools as `echo`, ..., so the second `verify_echo` call is an `AgentErrorEvent` `Tool 'verify_echo' not found` and `resumed_second_call_ok` is `false`. + The add on a ready agent connects at once (tools listed right after the call, prefixed although `verify` is the only server); the duplicate and the SSE server raise `ValueError`, a server that exits at startup raises `MCPServerStartError` naming it (the agent server turns both into its 400, F19), and all three leave `mcp_config` with only `verify` and the disabled `off`; the server added before the first message is connected with the agent under bare names. +- **Reopening a conversation with added servers (`S09.runtime-add-resume`).** Known failure (reproduced 2026-11-17 and 2026-11-19, [#6927](https://github.com/OpenHands/software-agent-sdk/issues/6927), closed 2026-11-18 with no fix merged; fix proposed in [#6929](https://github.com/OpenHands/software-agent-sdk/pull/6929), open): the reopened conversation offers the tools as `echo`, ..., so the second `verify_echo` call is an `AgentErrorEvent` `Tool 'verify_echo' not found` and `resumed_second_call_ok` is `false`. ```sh export SDK_VERIFY_RUN=$(control-sdk new --field run) control-sdk program S09/runtime --arg resume \ diff --git a/.agents/skills/verify-sdk/references/feature-map/S10-hooks.md b/.agents/skills/verify-sdk/references/feature-map/S10-hooks.md index 42d99af4c73..037f1b0c151 100644 --- a/.agents/skills/verify-sdk/references/feature-map/S10-hooks.md +++ b/.agents/skills/verify-sdk/references/feature-map/S10-hooks.md @@ -14,7 +14,7 @@ Examples: `examples/01_standalone_sdk/33_hooks/main.py`, `examples/01_standalone - `S10.definition-validation`: `type="command"` needs `command`; `prompt` needs `prompt` and forbids `command` and `async`; `agent` forbids `command` and `async`; `display_command` labels command-less hooks as `<type>-hook:<name>`. Unit: `tests/sdk/hooks/test_config.py` (`test_command_hook_requires_command`, `test_hook_definition_validation_errors`, `test_display_command`, `test_agent_hook_rejects_async`, `test_prompt_hook_rejects_async`). Example: none. Real-LLM: none. - `S10.config-discovery`: `HookConfig.load(path)` reads one file; `HookConfig.load(working_dir=...)` looks for `<dir>/.openhands/hooks.json`, then `~/.openhands/hooks.json`; a missing file gives an empty config. Unit: `tests/sdk/hooks/test_config.py::TestHookConfig` (`test_load_from_json_file`, `test_load_missing_file_returns_empty`, `test_load_discovers_config_in_working_dir`). Example: none. Real-LLM: none. - `S10.matcher`: `*` or an empty matcher matches every tool (and events without a tool), `/regex/` and strings with regex metacharacters are full-matched, anything else must equal the tool name; an invalid regex falls back to an exact match. Unit: `tests/sdk/hooks/test_config.py::TestHookMatcher`, `::TestHookConfig::test_get_hooks_filters_by_tool_name`. Example: `examples/01_standalone_sdk/33_hooks/main.py` (`matcher="terminal"`). Real-LLM: none. -- `S10.command-exec`: a command hook runs through the shell in the workspace with the event JSON on stdin, `OPENHANDS_PROJECT_DIR`, `OPENHANDS_SESSION_ID`, `OPENHANDS_EVENT_TYPE`, `OPENHANDS_TOOL_NAME` in a sanitized environment, captures stdout and stderr, and reports a timeout or a missing command as an error result. Unit: `tests/sdk/hooks/test_executor.py::TestHookExecutor`. Example: `examples/01_standalone_sdk/33_hooks/main.py` (`log_tools.sh`). Real-LLM: none. +- `S10.command-exec`: a command hook runs through the shell in the workspace with the event JSON on stdin, `OPENHANDS_PROJECT_DIR`, `OPENHANDS_SESSION_ID`, `OPENHANDS_EVENT_TYPE`, `OPENHANDS_TOOL_NAME` in a sanitized environment that also carries the conversation's secrets (known failure #6966: it carries none), captures stdout and stderr, and reports a timeout or a missing command as an error result. Unit: `tests/sdk/hooks/test_executor.py::TestHookExecutor`. Example: `examples/01_standalone_sdk/33_hooks/main.py` (`log_tools.sh`). Real-LLM: none. - `S10.exit-codes`: exit 0 is success, exit 2 blocks, any other non-zero exit is a non-blocking failure; JSON stdout `decision: deny` or `continue: false` also blocks, a non-string `decision` is ignored; `execute_all` stops at the first block. Unit: `tests/sdk/hooks/test_executor.py::TestHookExecutor` (`test_execute_blocking_exit_code`, `test_execute_json_output_decision`, `test_non_string_decision_is_ignored_rather_than_fatal`, `test_execute_all_stops_on_block`). Example: `examples/01_standalone_sdk/33_hooks/main.py` (`block_dangerous.sh`). Real-LLM: none. - `S10.async-hooks`: `async: true` command hooks start in their own process group, get stdin, return at once with `async_started`, are killed after their timeout or at `SessionEnd`; an async `PreToolUse` hook logs that it cannot block. Unit: `tests/sdk/hooks/test_executor.py::TestAsyncHookExecution`, `::TestAsyncProcessManager`, `tests/sdk/hooks/test_manager.py::TestAsyncHookManager`. Example: `examples/05_skills_and_plugins/02_loading_plugins/example_plugins/code-quality/hooks/hooks.json` (plugin hook). Real-LLM: none. - `S10.pre-tool-use-block`: a blocking `PreToolUse` hook records the action in `state.blocked_actions`; the agent then emits a `UserRejectObservation` (`rejection_source="hook"`, the hook's reason) instead of running the tool, and other actions in the same batch still run. Unit: `tests/sdk/hooks/test_integration.py::TestHookEventProcessorBlocking`, `tests/sdk/agent/test_action_batch.py` (`test_prepare_with_blocked`, `test_emit_blocked_produces_rejection`), `tests/sdk/hooks/test_manager.py::TestHookManager::test_run_pre_tool_use_blocks_when_hook_denies`. Example: `examples/01_standalone_sdk/33_hooks/main.py` (demo 2). Real-LLM: none. @@ -82,6 +82,13 @@ Preconditions: --check blocked_after_run eq 0 --check action_block_after_pop eq false \ --check user_prompt_hook_blocked.0 eq true --check hook_message_id_set eq true ``` +- **A command hook and the conversation's secrets (`S10.command-exec`).** `programs/S10/hook_secrets.py`: `Conversation(secrets={"VERIFY_HOOK_SECRET": ...}, hook_config=...)` with a `PreToolUse(terminal)` command hook, and a scripted terminal call; both run `test -n "$VERIFY_HOOK_SECRET"` and print `SET` or `UNSET`, never the value. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S10/hook_secrets --check status eq finished --check hook_runs eq 1 \ + --check terminal_text eq TERMINAL_SECRET_SET --check hook_stdout eq HOOK_SECRET_SET + ``` + Expected: the terminal call (the control) and the hook both see the secret. Known failure (reproduced 2026-11-19, [#6966](https://github.com/OpenHands/software-agent-sdk/issues/6966)): the hook prints `HOOK_SECRET_UNSET`; `HookExecutor` builds the hook's environment from `sanitized_env()` and never consults the conversation's `SecretRegistry`, while the terminal call gets the secret (`TERMINAL_SECRET_SET`). - **Config forms, validation, discovery, matchers and exit codes (`S10.config-format`, `S10.definition-validation`, `S10.config-discovery`, `S10.matcher`, `S10.exit-codes`, `S10.command-exec`).** Through the public classes, without a conversation (as example 57 uses `HookManager`): typed, PascalCase and legacy-wrapper configs, `async` alias and merge; the user `hooks.json` under `OH_PERSISTENCE_DIR`; each kind of command-hook answer. ```sh export SDK_VERIFY_RUN=$(control-sdk new --field run) diff --git a/.agents/skills/verify-sdk/references/feature-map/S11-subagents-delegation.md b/.agents/skills/verify-sdk/references/feature-map/S11-subagents-delegation.md index f4530059d6b..3cbe761db83 100644 --- a/.agents/skills/verify-sdk/references/feature-map/S11-subagents-delegation.md +++ b/.agents/skills/verify-sdk/references/feature-map/S11-subagents-delegation.md @@ -30,7 +30,7 @@ Examples: `examples/01_standalone_sdk/25_agent_delegation.py`, `examples/01_stan - `S11.task-hooks`: a definition's `hooks` become the sub-conversation's `hook_config`, on create and on resume. Unit: `tests/tools/task/test_task_manager.py::TestTaskManagerHooks`. Example: none. Real-LLM: none. - `S11.task-observability`: sub-conversations share the parent's prompt cache key and carry delegate trace metadata (`is_delegate`, `task_id`, `subagent_type`, `parent_session_id`, parent span link) and the `delegate` tag. Unit: `tests/tools/task/test_task_manager.py::TestTaskManager::test_sub_agents_inherit_parent_prompt_cache_key`, `::test_marks_conversation_as_delegate_with_linking_metadata`, `::test_delegate_metadata_includes_parent_span_link`. Example: none. Real-LLM: none. - `S11.task-secrets`: secrets the parent conversation holds (`conversation.update_secrets(...)`) resolve in a task sub-agent's terminal, on create and on resume, and stay masked in what comes back. Unit: `tests/tools/task/test_secret_inheritance.py`. Example: none. Real-LLM: none. -- `S11.task-confirmation`: a definition's `permission_mode` sets the sub-conversation's confirmation policy, omission inherits the parent's; when the sub-agent waits for confirmation, `TaskToolSet.create(confirmation_handler=...)` decides, and without a handler the actions are approved. Unit: **none (untested)**. Example: none. Real-LLM: none. +- `S11.task-confirmation`: a definition's `permission_mode` sets the sub-conversation's confirmation policy, omission inherits the parent's; when the sub-agent waits for confirmation, `TaskToolSet.create(confirmation_handler=...)` decides; without a handler the actions are approved and the parent's security analyzer is not passed on (a known failure, [#6999](https://github.com/OpenHands/software-agent-sdk/issues/6999)). Unit: **none (untested)**. Example: none. Real-LLM: none. - `S11.sub-agent-scope`: `SubAgentScope(tools=True, mcp_servers=True)` (the `sub_agent_scope` tool param) hides and refuses sub-agents whose tools or MCP servers the parent lacks, compares built-in tool names canonically, propagates to nested delegation tools, and says so when no sub-agent fits. Unit: `tests/sdk/subagent/test_subagent_scope.py`, `tests/tools/task/test_sub_agent_scope.py`. Example: none. Real-LLM: none. - `S11.delegate-executor`: the older `DelegateExecutor` (`spawn` named sub-agents, then `delegate` a dict of tasks run in parallel threads) closes replaced and spawned sub-agents, rolls back a partial spawn, merges metrics, and inherits the parent's persistence dir. Unit: `tests/tools/delegate/test_delegation.py`. Example: none. Real-LLM: none. - `S11.delegation-visualizer`: `DelegationVisualizer(name=...)` labels messages by sender and agent name and creates per-task sub-visualizers. Unit: `tests/tools/delegate/test_visualizer.py`. Example: all three examples. Real-LLM: none. @@ -73,6 +73,18 @@ Programs give each programmatic sub-agent its own scripted LLM (`programs/S11/s1 --check subagent_dirs eq 3 --check sub_hook_lines len-eq 1 --check sub_hook_lines.0 eq sub-hook ``` The resumed sub-agent's LLM request still holds the first prompt; a non-finished sub-agent comes back as an error observation with the run's error, never an empty success; three sub-conversations stay under `<parent>/subagents/` after close. +- **Resume a task after the parent is reopened (`S11.task-resume`, `S11.task-persistence`).** A persisted parent delegates to `echoer` (`task_00000001`) and closes; a second process reopens the parent by id (a new task tool) and its model calls `task(resume="task_00000001")`. Then, in one session, a parent resumes its `echoer` task naming the type `other` (no tools). + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + CID=$(control-sdk program S11/task_resume_session --arg create --check observations.0.status eq completed --field id) + control-sdk program S11/task_resume_session --arg resume --arg "$CID" \ + --check observations len-eq 2 --check observations.1.task_id eq task_00000001 \ + --check observations.1.status eq completed --check echoer_calls eq 1 --check subagent_dirs len-eq 1 + control-sdk program S11/task_resume_session --arg retype \ + --check observations.1.task_id eq task_00000001 --check observations.1.status eq completed \ + --check echoer_calls eq 2 --check other_calls eq 0 + ``` + Expected: the reopened parent's task tool finds `task_00000001` (its sub-conversation is on disk under `<parent>/subagents/`) and continues it, and a resume runs the type the task was created with. Known failure (reproduced 2026-11-19, [#6990](https://github.com/OpenHands/software-agent-sdk/issues/6990)): the task index lives only in the `TaskManager`'s memory, so after the reopen the resume fails with `Task 'task_00000001' not found. Available tasks: ` (`task_id` `unknown`, `status` `error`) although the sub-conversation directory is there; and `Task` does not record its type, so resuming with `other` builds that agent and fails with `Cannot resume conversation: tools were removed mid-conversation (removed: ['terminal'])`. - **Delegates share the parent's prompt-cache key (`S11.task-observability`).** Same scenario; each `echoer` LLM call records its call context. ```sh export SDK_VERIFY_RUN=$(control-sdk new --field run) @@ -146,7 +158,7 @@ Programs give each programmatic sub-agent its own scripted LLM (`programs/S11/s1 --check builtin_written eq true --check description_lists_ws_reviewer eq true --check script_left eq 0 \ --check temp_task_dirs_before_close eq 1 --check temp_task_dirs_after_close eq 0 ``` -- **Sub-agent confirmation (`S11.task-confirmation`).** The program registers its own task tool set whose `create()` passes a `confirmation_handler`; `careful` has `permission_mode: always_confirm` and the handler rejects it; after `set_confirmation_policy(AlwaysConfirm())` on the parent, `inheritor` (no `permission_mode`) inherits the policy and the handler approves; a plain `TaskToolSet` without a handler approves on its own. +- **Sub-agent confirmation (`S11.task-confirmation`).** The program registers its own task tool set whose `create()` passes a `confirmation_handler`; `careful` has `permission_mode: always_confirm` and the handler rejects it; after `set_confirmation_policy(AlwaysConfirm())` on the parent, `inheritor` (no `permission_mode`) inherits the policy and the handler approves. ```sh export SDK_VERIFY_RUN=$(control-sdk new --field run) control-sdk program S11/task_confirmation \ @@ -155,9 +167,17 @@ Programs give each programmatic sub-agent its own scripted LLM (`programs/S11/s1 --check careful_file eq false --check observations.0.text eq "careful done" \ --check parent_waited eq waiting_for_confirmation \ --check handler_calls.1.task_id eq task_00000002 --check handler_calls.1.approved eq true \ - --check inherited_file eq true --check execution_status eq finished \ - --check auto_file eq true --check plain_observations.0.status eq completed + --check inherited_file eq true --check execution_status eq finished + ``` +- **Sub-agent confirmation with no handler, and the parent's analyzer (`S11.task-confirmation`, `S11.delegate-executor`).** No handler: `careful` (`permission_mode: always_confirm`) runs `touch closed-task.txt` through the plain task tool, and `careful_worker` (the same mode) runs `touch closed-delegate.txt` through a `DelegateExecutor` wrapped in a `delegate` tool. Then a parent with `ConfirmRisky()` and a recording analyzer (LOW for `task`, HIGH otherwise) delegates `touch risky.txt` to `inheritor` (no `permission_mode`) through a task tool whose handler rejects. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S11/task_fail_closed \ + --check task_file eq false --check delegate_file eq false \ + --check risky_status eq finished --check handler_calls len-eq 1 --check handler_calls.0.tools.0 eq terminal \ + --check risky_file eq false --check sub_action_analyzed eq true ``` + Expected: with nothing to ask, a sub-agent action that waits for confirmation is rejected, not run (neither file exists); the sub-conversation gets the parent's security analyzer, so the analyzer rates the sub-agent's `terminal` call. Known failure (reproduced 2026-11-19, [#6999](https://github.com/OpenHands/software-agent-sdk/issues/6999)): `TaskManager._run_until_finished` and `DelegateExecutor._run_until_finished` treat a missing `confirmation_handler` as approval and call `run()` again, so both files are written and the task comes back `completed`; the analyzer is asked only about the parent's `task` call (`analyzed_tools` is `["task"]`, `sub_action_analyzed` false): the sub-agent runs with no analyzer, and the handler is asked only because `ConfirmRisky` confirms an unrated action. - **Sub-agent scope (`S11.sub-agent-scope`).** A parent with only the terminal sets `sub_agent_scope: {tools: true}` on its task tool; `editor` needs `file_editor`, `sheller` needs the terminal and has its own task tool; a parent with no tools of its own is offered nothing. ```sh export SDK_VERIFY_RUN=$(control-sdk new --field run) diff --git a/.agents/skills/verify-sdk/references/feature-map/S14-core-tools.md b/.agents/skills/verify-sdk/references/feature-map/S14-core-tools.md index f06df5a36e0..e65fd1bf073 100644 --- a/.agents/skills/verify-sdk/references/feature-map/S14-core-tools.md +++ b/.agents/skills/verify-sdk/references/feature-map/S14-core-tools.md @@ -143,6 +143,29 @@ Preconditions: --check steps.shell_after_reset.text contains my-shell --check backend_at_end eq SubprocessTerminal ``` Expected: a reset starts a fresh shell of the same kind. Known failure (reproduced 2026-11-17, [#6653](https://github.com/OpenHands/software-agent-sdk/issues/6653)): `TerminalExecutor._reset_single_session` builds the new session with `terminal_type=None`, so an executor forced to `subprocess` comes back from a reset on tmux (`backend_at_end` is `TmuxTerminal`, `$0` is `/bin/bash`), dropping `shell_path`. +- **A timed-out command is flagged (`S14.terminal-timeout`).** With `no_change_timeout_seconds: 2`, a scripted agent runs `sleep 5` with `timeout` 1 (the hard limit) and `sleep 30` with none (no new output for 2 s), each stopped with `C-c`; on tmux, then on the subprocess backend. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + for backend in tmux subprocess; do + control-sdk program S14/terminal_flags --arg "$backend" \ + --check steps.hard.exit_code eq -1 --check steps.hard.is_error eq false \ + --check steps.hard.text contains "timed out after 1.0 seconds" --check steps.hard.timeout eq true \ + --check steps.idle.exit_code eq -1 --check steps.idle.text contains "no new output after 2 seconds" \ + --check steps.idle.timeout eq true + done + ``` + Expected: both observations carry `timeout=True`, so a hook or observer can tell a timed-out command from its fields; text, exit code (-1) and `is_error` (false) as they are. Known failure (reproduced 2026-11-19, [#6991](https://github.com/OpenHands/software-agent-sdk/issues/6991)): on both backends the text says the command timed out but `timeout` is `false`: `TerminalSession._handle_hard_timeout_command` and `_handle_nochange_timeout_command` build the `TerminalObservation` without `timeout=True`. +- **A command that starts with a comment line (`S14.terminal-exec`).** The same program: `sleep 2 && printf '%s-%s\n' synthetic status-output` (only its output, not the command line, holds `synthetic-status-output`), then the same command after a `# Read-only: ...` line; on tmux, then on the subprocess backend. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + for backend in tmux subprocess; do + control-sdk program S14/terminal_flags --arg "$backend" \ + --check steps.control.exit_code eq 0 --check steps.control.text contains synthetic-status-output \ + --check steps.comment.exit_code eq 0 --check steps.comment.seconds ge 1.5 \ + --check steps.comment.text contains synthetic-status-output + done + ``` + Expected: the leading comment changes nothing; the call waits for the command and returns its output. Known failure (reproduced 2026-11-19, [#6972](https://github.com/OpenHands/software-agent-sdk/issues/6972)): on both backends the commented call returns exit 0 within about 0.2 s, its text holding only the echoed command lines and no `synthetic-status-output` (Bash prints a prompt after the comment line, which the session takes for completion); on the subprocess backend the missing output turns up at the start of the next call's observation. - **A one-second observation wait cap and a shell that exits, subprocess backend (`S14.terminal-observation-wait-cap`, `S14.terminal-shell-exit`).** `Tool(name="terminal", params={"terminal_type": "subprocess", "max_observation_wait_seconds": 1})`; a scripted agent starts `sleep 3; echo finished` with `timeout` 600 and polls it with `""` until it finishes, then runs `printf ...; exit 7`, `echo still-here`, a reset and `echo back`. The program also creates the tool with a cap of 0 and of -1. ```sh export SDK_VERIFY_RUN=$(control-sdk new --field run) diff --git a/.agents/skills/verify-sdk/references/feature-map/S19-settings.md b/.agents/skills/verify-sdk/references/feature-map/S19-settings.md index f6dd346a559..ba01344800c 100644 --- a/.agents/skills/verify-sdk/references/feature-map/S19-settings.md +++ b/.agents/skills/verify-sdk/references/feature-map/S19-settings.md @@ -34,6 +34,7 @@ Examples: `examples/01_standalone_sdk/46_agent_settings.py` - `S19.profile-resolve`: since 1.60 a launch has two stages in `openhands.sdk.launch`. `resolve(profile_or_id, LaunchStores(llm_profiles=..., mcp_config=..., skills=...))` runs where the stores live and returns a reference-free `ResolvedLaunch` with validated settings: it loads the LLM profile, filters MCP servers, applies `disabled_skills`, scopes tools and delegation, and keeps persona and verification; every dangling reference (LLM profile, MCP servers, meta-profile) is raised together as `UnresolvedProfileReferences`. `finalize(resolved, LaunchRuntime(...))` builds the agent where it runs; `preview_launch(profile, stores, runtime)` reports the same verdict without raising, with secrets redacted. The legacy `resolve_agent_profile` / `resolve_agent_profile_dry_run` were removed. Unit: `tests/sdk/launch/test_resolve_legacy_parity.py`. Example: none. Real-LLM: none. - `S19.profile-refs`: renaming an LLM profile cascades to every agent profile that cites it, atomically; deleting a referenced LLM profile raises `ProfileReferenced` naming the referrers. Unit: `tests/sdk/profiles/test_profile_refs.py`, `tests/sdk/profiles/test_store_protocol_hoists.py`. Example: none. Real-LLM: none. - `S19.profile-seed`: `build_seed_profile(settings)` derives the default profile from current settings (persona, explicit tools, ACP branch, no disabled skills); `save_profile_preserving_identity` keeps the id and bumps `revision`. Unit: `tests/sdk/profiles/test_store_protocol_hoists.py`. Example: none. Real-LLM: none. +- `S19.profile-sandbox`: both profile variants take an optional `sandbox=ProfileSandboxSettings(spec_id=...)` (exported from `openhands.sdk.profiles`): a sandbox spec id the host that provisions sandboxes reads; it round-trips through `validate_agent_profile` and `AgentProfileStore`, a stored profile without it loads with `sandbox` `None`, an empty `spec_id`, a missing `spec_id`, an extra key or a non-object is refused, and `resolve()` gives exactly the same `AgentSettings` with or without it. Unit: `tests/sdk/profiles/test_agent_profile.py::test_sandbox_defaults_to_none_and_older_payloads_load`, `::test_sandbox_round_trips_on_both_variants`, `::test_sandbox_rejects_empty_or_unknown_fields`, `tests/sdk/launch/test_resolve.py::test_sandbox_metadata_does_not_change_the_resolved_settings`. Example: none. Real-LLM: none. - `S19.credential-binding`: `HttpVersionedCredentialBinding(url, headers).load()` GETs `{value, version}`, `replace(expected_version, value)` PUTs and returns the new version; 404 and 400/422 raise `CredentialNeedsReauthentication`, 409 `CredentialConflict`, 401/403 `CredentialAuthorizationRejected`, 501 `CredentialBindingUnsupported`, invalid URLs `CredentialInvalidResponse`; `reauthorize()` swaps headers for the same URL. Unit: `tests/sdk/test_credential.py`. Example: none. Real-LLM: none. ## How to get to it (agent POV) @@ -42,7 +43,7 @@ Examples: `examples/01_standalone_sdk/46_agent_settings.py` - `settings.create_agent()`, `settings.model_dump(mode="json")`, `OpenHandsAgentSettings.model_validate(payload)`, `OpenHandsAgentSettings.from_persisted(payload)`, `ConversationSettings.from_persisted(payload)`: `S19.settings-roundtrip`, `S19.schema-migrations`. - Constants: `AGENT_SETTINGS_SCHEMA_VERSION = 8`, `CONVERSATION_SETTINGS_SCHEMA_VERSION = 1` (`settings/model.py`), `AGENT_PROFILE_SCHEMA_VERSION = 3` (`profiles/agent_profile.py`). - Files: `tests/sdk/persisted_settings_baselines/v1..v10/*.json`, `.github/scripts/check_persisted_settings_compat.py`, workflow `.github/workflows/persisted-settings-compat.yml`: `S19.persisted-baselines`, `S19.pypi-baseline-gate`. -- `from openhands.sdk.profiles import AgentProfileStore, OpenHandsAgentProfile, ACPAgentProfile, validate_agent_profile, cascade_rename, delete_llm_profile, rename_llm_profile, build_seed_profile, save_profile_preserving_identity`; on disk `~/.openhands/agent-profiles/<name>.json` (moved by `OH_PERSISTENCE_DIR`): `S19.profile-*`. LLM profiles themselves (the `llm_profile_ref` targets) belong to the LLM profile store family (example 37). +- `from openhands.sdk.profiles import AgentProfileStore, OpenHandsAgentProfile, ACPAgentProfile, validate_agent_profile, cascade_rename, delete_llm_profile, rename_llm_profile, build_seed_profile, save_profile_preserving_identity, ProfileSandboxSettings`; on disk `~/.openhands/agent-profiles/<name>.json` (moved by `OH_PERSISTENCE_DIR`): `S19.profile-*`. LLM profiles themselves (the `llm_profile_ref` targets) belong to the LLM profile store family (example 37). - `from openhands.sdk.credential import HttpVersionedCredentialBinding, ResolvedCredential, Credential*`: `S19.credential-binding`; consumed by the ACP agent's file-credential sync. - Env read by example 46: `LLM_API_KEY`, `LLM_MODEL` (default `gpt-5.5`), `LLM_BASE_URL`. @@ -184,6 +185,20 @@ Preconditions: --check seed.id_kept eq true --check seed.revision_bumped eq true ``` Six threads saving the same name through `save_profile_preserving_identity` leave one profile, one id, revision 5. `list()` returns file names (`coder.json`), as documented. +- **Sandbox metadata on a profile (`S19.profile-sandbox`).** A v4 payload without the field, both variants with a spec id, four refused shapes, a save and load through the run's `AgentProfileStore`, and `resolve()` of the same profile with and without the field. No model call. + ```sh + export SDK_VERIFY_RUN=$(control-sdk new --field run) + control-sdk program S19/profile_sandbox \ + --check old_payload_sandbox eq null \ + --check variants.openhands.spec_id eq registry.example/agent-server:qa-s19 --check variants.openhands.round_trip eq true \ + --check variants.acp.type eq ACPAgentProfile --check variants.acp.spec_id eq registry.example/agent-server:qa-s19 \ + --check variants.acp.round_trip eq true \ + --check refused.empty_spec eq string_too_short --check refused.no_spec eq missing \ + --check refused.extra_key eq extra_forbidden --check refused.string eq model_type \ + --check store.spec_id eq registry.example/agent-server:qa-s19 --check store.on_disk_spec eq 1 \ + --check resolve.same_settings eq true --check resolve.settings_mention_spec eq false + ``` + Expected: an older payload loads with no sandbox; the spec id survives validation, a dump-and-load and the store on both variants; the bad shapes are refused with those error types; and the resolved settings are identical with and without the sandbox, which never appears in them. - **Credential binding (`S19.credential-binding`).** A versioned credential source served on localhost (Bearer auth, 409 on a stale version, fixed status paths), loaded and replaced through `HttpVersionedCredentialBinding`. ```sh export SDK_VERIFY_RUN=$(control-sdk new --field run) diff --git a/.agents/skills/verify-sdk/scripts/control_sdk.py b/.agents/skills/verify-sdk/scripts/control_sdk.py index 43993a1aaea..f1ed0d73f39 100644 --- a/.agents/skills/verify-sdk/scripts/control_sdk.py +++ b/.agents/skills/verify-sdk/scripts/control_sdk.py @@ -16,7 +16,8 @@ Models: a conversation or program with a script uses the SDK's scripted `TestLLM` (no key, deterministic). Without one it uses the run's real model (`new --llm deepseek-flash`), with the key read from DEEPSEEK_API_KEY at call -time and never printed or written. +time (or, with $LLM_BASE_URL and $LLM_API_KEY set, deepseek-flash through that +LiteLLM proxy) and never printed or written. Exit codes: 0 ok, 1 action or check failed, 2 usage, 3 blocked (a missing prerequisite such as the model key). @@ -111,6 +112,11 @@ def run_env(run: Path, extra: dict[str, str] | None = None) -> dict[str, str]: for k, v in os.environ.items() if not k.startswith(SCRUBBED_PREFIXES) and k != RUN_ENV } + # The LLM proxy, if any, under names the scrub keeps (sdk_verify.model_spec). + for url_env, key_env in sdk_verify.PROXY_ENVS[1:]: + if os.environ.get(url_env) and os.environ.get(key_env): + env["SDK_VERIFY_LLM_BASE_URL"] = os.environ[url_env] + env["SDK_VERIFY_LLM_API_KEY"] = os.environ[key_env] env.update( { RUN_ENV: str(run), @@ -157,7 +163,7 @@ def cmd_new(args: argparse.Namespace) -> dict[str, Any]: (run / sub).mkdir(parents=True, exist_ok=True) config = {"run": str(run), "llm": args.llm, "created": time.time()} (run / "run.json").write_text(json.dumps(config, indent=2)) - key_env = sdk_verify.MODELS.get(args.llm, {}).get("env") + key_env = (sdk_verify.model_spec(args.llm) or {}).get("env") return { "ok": True, **config, @@ -371,7 +377,7 @@ def last_json(text: str) -> Any: def model_env(run: Path) -> dict[str, str]: """LLM_* variables for example scripts, from the run's real model.""" name = json.loads((run / "run.json").read_text()).get("llm") - spec = sdk_verify.MODELS.get(name or "") + spec = sdk_verify.model_spec(name or "") if spec is None: raise CliError( "--model-env needs a run with a real model", @@ -416,9 +422,10 @@ def run_command( else (exc.stdout or "") ) stderr = "timed out" - secret = env.get("LLM_API_KEY") - if secret: - stdout, stderr = stdout.replace(secret, "***"), stderr.replace(secret, "***") + for secret in {env.get("LLM_API_KEY"), os.environ.get("LLM_API_KEY")}: + if secret: + stdout = stdout.replace(secret, "***") + stderr = stderr.replace(secret, "***") log = run / "state" / "exec.log" with log.open("a") as fh: fh.write(f"$ {' '.join(command)}\n{stdout}\n{stderr}\n[exit {code}]\n") diff --git a/.agents/skills/verify-sdk/scripts/sdk_map.py b/.agents/skills/verify-sdk/scripts/sdk_map.py index 3d6906ad74d..1ef01e9b526 100755 --- a/.agents/skills/verify-sdk/scripts/sdk_map.py +++ b/.agents/skills/verify-sdk/scripts/sdk_map.py @@ -732,7 +732,8 @@ def worse(outcome: str, note: str) -> None: if result["skipped_model"]: skipped = result["skipped_model"] result["notes"].append( - f"{skipped} model-backed block(s) skipped (no DEEPSEEK_API_KEY)" + f"{skipped} model-backed block(s) skipped" + " (no DEEPSEEK_API_KEY or LLM_API_KEY)" ) if ran == 0 and result["outcome"] == "pass": result["outcome"] = "blocked"