diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d622224e..cf6e257e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -128,3 +128,31 @@ jobs: - name: Verify PostgreSQL migrations and restart persistence run: cargo test -p agentic-server-core --test postgres_storage_integration -- --ignored --test-threads=1 + + dynamo-upstream: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_TEST_DEBUG: "0" + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1 + with: + toolchain: 1.98.0 + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + cache-bin: false + + - name: Validate recorded cassettes + run: | + python3 -m pip install 'PyYAML==6.0.3' + python3 scripts/validate-cassettes.py + + - name: Replay Dynamo upstream cassettes + run: cargo test -p agentic-server-core --test dynamo_cassette_test diff --git a/CHANGELOG.md b/CHANGELOG.md index c21db227..0d7485de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to Agentic API are documented here. +## [Unreleased] + +### Added + +- Documented running Agentic API in front of NVIDIA Dynamo and recorded Dynamo cassettes for stateful and + function-call flows. + +### Testing + +- Added Dynamo upstream replay tests, a generic cassette validator (`scripts/validate-cassettes.py`), and a dedicated + CI job for them. + ## [0.5.0] - 2026-08-25 ### Changed diff --git a/README.md b/README.md index b574347d..eda147bb 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,10 @@ vllm serve Qwen/Qwen3-30B-A3B-FP8 \ --reasoning-parser qwen3 --port 5050 ``` +Serving through [NVIDIA Dynamo](https://github.com/ai-dynamo/dynamo) instead of a standalone `vllm serve`? Point the +gateway at the Dynamo frontend the same way; see +[Running Agentic API in front of NVIDIA Dynamo](docs/guides/dynamo-upstream.md). + **2. Start Agentic API**, pointing it at the vLLM server (set the `YOU_*` variables to enable built-in web search): ```bash diff --git a/crates/agentic-server-core/tests/cassettes/README.md b/crates/agentic-server-core/tests/cassettes/README.md index 618335d2..77a1e071 100644 --- a/crates/agentic-server-core/tests/cassettes/README.md +++ b/crates/agentic-server-core/tests/cassettes/README.md @@ -195,6 +195,7 @@ turns: | `record_custom_tool_cassettes.sh` | Matching two-turn custom-tool flows (streaming + non-streaming) | gateway and OpenAI reference | | `record_mcp_cassettes.sh` | Native MCP counter tool discovery and calls (streaming + non-streaming) | gateway and OpenAI reference | | `record_web_search_cassettes.sh` | Matching web-search calls (streaming + non-streaming) | gateway and OpenAI reference | +| `record_dynamo_cassettes.sh` | Stateful two-turn and client-executed function tool call cassettes (streaming + non-streaming) | NVIDIA Dynamo frontend | ### Text-only (OpenAI) @@ -219,6 +220,17 @@ vllm serve Qwen/Qwen3-30B-A3B-FP8 --tool-call-parser hermes --enable-auto-tool-c VLLM_URL=http://0.0.0.0:5050 MODEL=Qwen/Qwen3-30B-A3B-FP8 bash tests/cassettes/record_tool_call_cassettes.sh ``` +### NVIDIA Dynamo (vLLM worker behind the Dynamo frontend) + +Dynamo's `/v1/responses` rejects `previous_response_id` with `501`, so the recorder's own turn chaining cannot be +used. The script records turn 1 from a prompt, builds turn 2's input from turn 1's recorded assistant message (the +hydrated item history the gateway sends upstream), records it, and merges both into one cassette. See +[docs/guides/dynamo-upstream.md](../../../../docs/guides/dynamo-upstream.md) for the Dynamo launch commands. + +```bash +DYNAMO_URL=http://127.0.0.1:8000 MODEL=openai/gpt-oss-20b bash tests/cassettes/record_dynamo_cassettes.sh +``` + ### Web search (gateway and OpenAI) The default records both providers. Use `WEB_SEARCH_RECORD_SET=gateway` or diff --git a/crates/agentic-server-core/tests/cassettes/dynamo/dynamo-stateful-openai-gpt-oss-20b-nonstreaming.yaml b/crates/agentic-server-core/tests/cassettes/dynamo/dynamo-stateful-openai-gpt-oss-20b-nonstreaming.yaml new file mode 100644 index 00000000..4fbb12a2 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/dynamo/dynamo-stateful-openai-gpt-oss-20b-nonstreaming.yaml @@ -0,0 +1,160 @@ +turns: +- filename: t1 + request: + body: + input: 'Remember the word APPLE. Just say: OK' + max_output_tokens: 2048 + model: openai/gpt-oss-20b + store: true + stream: false + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + background: false + billing: null + completed_at: 1787829072 + conversation: null + created_at: 1787829072 + error: null + frequency_penalty: 0.0 + id: resp_e823dacfde14451baad50ab4915bd39b + incomplete_details: null + instructions: null + max_output_tokens: 2048 + max_tool_calls: null + metadata: {} + model: openai/gpt-oss-20b + object: response + output: + - content: + - annotations: [] + logprobs: [] + text: OK + type: output_text + id: msg_aeb098ffaa8c4c0092ed38e3586e8e24 + role: assistant + status: completed + type: message + parallel_tool_calls: true + presence_penalty: 0.0 + previous_response_id: null + prompt: null + prompt_cache_key: null + prompt_cache_retention: null + reasoning: null + safety_identifier: null + service_tier: auto + status: completed + store: true + temperature: 1.0 + text: + format: + type: text + tool_choice: auto + tools: [] + top_logprobs: 0 + top_p: 1.0 + truncation: disabled + usage: + input_tokens: 77 + input_tokens_details: + cached_tokens: 0 + output_tokens: 35 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 112 + headers: + content-type: application/json + status_code: 200 +- filename: t2 + request: + body: + input: + - content: 'Remember the word APPLE. Just say: OK' + role: user + type: message + - content: + - text: OK + type: output_text + id: msg_aeb098ffaa8c4c0092ed38e3586e8e24 + role: assistant + status: completed + type: message + - content: What word did I ask you to remember? Reply with just the word. + role: user + type: message + max_output_tokens: 2048 + model: openai/gpt-oss-20b + store: true + stream: false + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + background: false + billing: null + completed_at: 1787829074 + conversation: null + created_at: 1787829074 + error: null + frequency_penalty: 0.0 + id: resp_f24a1d1897c74063bfcbbd6fc91b5667 + incomplete_details: null + instructions: null + max_output_tokens: 2048 + max_tool_calls: null + metadata: {} + model: openai/gpt-oss-20b + object: response + output: + - content: + - annotations: [] + logprobs: [] + text: APPLE + type: output_text + id: msg_a52548907c8844198586ba7ce6eef89e + role: assistant + status: completed + type: message + parallel_tool_calls: true + presence_penalty: 0.0 + previous_response_id: null + prompt: null + prompt_cache_key: null + prompt_cache_retention: null + reasoning: null + safety_identifier: null + service_tier: auto + status: completed + store: true + temperature: 1.0 + text: + format: + type: text + tool_choice: auto + tools: [] + top_logprobs: 0 + top_p: 1.0 + truncation: disabled + usage: + input_tokens: 103 + input_tokens_details: + cached_tokens: 64 + output_tokens: 35 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 138 + headers: + content-type: application/json + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/dynamo/dynamo-stateful-openai-gpt-oss-20b-streaming.yaml b/crates/agentic-server-core/tests/cassettes/dynamo/dynamo-stateful-openai-gpt-oss-20b-streaming.yaml new file mode 100644 index 00000000..16769eeb --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/dynamo/dynamo-stateful-openai-gpt-oss-20b-streaming.yaml @@ -0,0 +1,228 @@ +turns: +- filename: t1 + request: + body: + input: 'Remember the word APPLE. Just say: OK' + max_output_tokens: 2048 + model: openai/gpt-oss-20b + store: true + stream: true + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","sequence_number":0,"response":{"background":false,"billing":null,"conversation":null,"created_at":1787829079,"completed_at":null,"error":null,"id":"resp_96ef0035cc8b4fa7b99ceccc8199b2cd","incomplete_details":null,"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":{},"model":"openai/gpt-oss-20b","object":"response","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":null,"safety_identifier":null,"service_tier":"auto","status":"in_progress","temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"presence_penalty":0.0,"frequency_penalty":0.0,"store":true}} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","sequence_number":1,"response":{"background":false,"billing":null,"conversation":null,"created_at":1787829079,"completed_at":null,"error":null,"id":"resp_96ef0035cc8b4fa7b99ceccc8199b2cd","incomplete_details":null,"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":{},"model":"openai/gpt-oss-20b","object":"response","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":null,"safety_identifier":null,"service_tier":"auto","status":"in_progress","temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"presence_penalty":0.0,"frequency_penalty":0.0,"store":true}} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"message","content":[],"id":"msg_6ff56ed5582c493ba4b6bd8d57b514cb","role":"assistant","status":"in_progress"}} + + ' + - ' + + ' + - 'event: response.content_part.added + + ' + - 'data: {"type":"response.content_part.added","sequence_number":3,"item_id":"msg_6ff56ed5582c493ba4b6bd8d57b514cb","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""}} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":4,"item_id":"msg_6ff56ed5582c493ba4b6bd8d57b514cb","output_index":0,"content_index":0,"delta":"OK","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.done + + ' + - 'data: {"type":"response.output_text.done","sequence_number":5,"item_id":"msg_6ff56ed5582c493ba4b6bd8d57b514cb","output_index":0,"content_index":0,"text":"OK","logprobs":[]} + + ' + - ' + + ' + - 'event: response.content_part.done + + ' + - 'data: {"type":"response.content_part.done","sequence_number":6,"item_id":"msg_6ff56ed5582c493ba4b6bd8d57b514cb","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"OK"}} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","sequence_number":7,"output_index":0,"item":{"type":"message","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"OK"}],"id":"msg_6ff56ed5582c493ba4b6bd8d57b514cb","role":"assistant","status":"completed"}} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","sequence_number":8,"response":{"background":false,"billing":null,"conversation":null,"created_at":1787829079,"completed_at":1787829080,"error":null,"id":"resp_96ef0035cc8b4fa7b99ceccc8199b2cd","incomplete_details":null,"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":{},"model":"openai/gpt-oss-20b","object":"response","output":[{"type":"message","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"OK"}],"id":"msg_6ff56ed5582c493ba4b6bd8d57b514cb","role":"assistant","status":"completed"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":null,"safety_identifier":null,"service_tier":"auto","status":"completed","temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":77,"input_tokens_details":{"cached_tokens":64},"output_tokens":38,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":115},"presence_penalty":0.0,"frequency_penalty":0.0,"store":true}} + + ' + - ' + + ' + - 'data: [DONE] + + ' + - ' + + ' + status_code: 200 +- filename: t2 + request: + body: + input: + - content: 'Remember the word APPLE. Just say: OK' + role: user + type: message + - content: + - text: OK + type: output_text + id: msg_6ff56ed5582c493ba4b6bd8d57b514cb + role: assistant + status: completed + type: message + - content: What word did I ask you to remember? Reply with just the word. + role: user + type: message + max_output_tokens: 2048 + model: openai/gpt-oss-20b + store: true + stream: true + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","sequence_number":0,"response":{"background":false,"billing":null,"conversation":null,"created_at":1787829081,"completed_at":null,"error":null,"id":"resp_fe040e9a687c47fb9a3bd0284a866422","incomplete_details":null,"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":{},"model":"openai/gpt-oss-20b","object":"response","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":null,"safety_identifier":null,"service_tier":"auto","status":"in_progress","temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"presence_penalty":0.0,"frequency_penalty":0.0,"store":true}} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","sequence_number":1,"response":{"background":false,"billing":null,"conversation":null,"created_at":1787829081,"completed_at":null,"error":null,"id":"resp_fe040e9a687c47fb9a3bd0284a866422","incomplete_details":null,"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":{},"model":"openai/gpt-oss-20b","object":"response","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":null,"safety_identifier":null,"service_tier":"auto","status":"in_progress","temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"presence_penalty":0.0,"frequency_penalty":0.0,"store":true}} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"message","content":[],"id":"msg_c8ca06b8cb7541c2baad58c63ff81ac1","role":"assistant","status":"in_progress"}} + + ' + - ' + + ' + - 'event: response.content_part.added + + ' + - 'data: {"type":"response.content_part.added","sequence_number":3,"item_id":"msg_c8ca06b8cb7541c2baad58c63ff81ac1","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""}} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":4,"item_id":"msg_c8ca06b8cb7541c2baad58c63ff81ac1","output_index":0,"content_index":0,"delta":"APPLE","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.done + + ' + - 'data: {"type":"response.output_text.done","sequence_number":5,"item_id":"msg_c8ca06b8cb7541c2baad58c63ff81ac1","output_index":0,"content_index":0,"text":"APPLE","logprobs":[]} + + ' + - ' + + ' + - 'event: response.content_part.done + + ' + - 'data: {"type":"response.content_part.done","sequence_number":6,"item_id":"msg_c8ca06b8cb7541c2baad58c63ff81ac1","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"APPLE"}} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","sequence_number":7,"output_index":0,"item":{"type":"message","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"APPLE"}],"id":"msg_c8ca06b8cb7541c2baad58c63ff81ac1","role":"assistant","status":"completed"}} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","sequence_number":8,"response":{"background":false,"billing":null,"conversation":null,"created_at":1787829081,"completed_at":1787829084,"error":null,"id":"resp_fe040e9a687c47fb9a3bd0284a866422","incomplete_details":null,"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":{},"model":"openai/gpt-oss-20b","object":"response","output":[{"type":"message","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"APPLE"}],"id":"msg_c8ca06b8cb7541c2baad58c63ff81ac1","role":"assistant","status":"completed"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":null,"safety_identifier":null,"service_tier":"auto","status":"completed","temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":103,"input_tokens_details":{"cached_tokens":96},"output_tokens":96,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":199},"presence_penalty":0.0,"frequency_penalty":0.0,"store":true}} + + ' + - ' + + ' + - 'data: [DONE] + + ' + - ' + + ' + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/dynamo/dynamo-tool-call-auto-openai-gpt-oss-20b-nonstreaming.yaml b/crates/agentic-server-core/tests/cassettes/dynamo/dynamo-tool-call-auto-openai-gpt-oss-20b-nonstreaming.yaml new file mode 100644 index 00000000..c0df4971 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/dynamo/dynamo-tool-call-auto-openai-gpt-oss-20b-nonstreaming.yaml @@ -0,0 +1,363 @@ +turns: +- filename: t1 + request: + body: + input: What is the current NVIDIA stock price? Use the tool. + max_output_tokens: 2048 + model: openai/gpt-oss-20b + store: true + stream: false + tool_choice: auto + tools: + - description: Get current temperature and conditions for a city + name: get_weather + parameters: + additionalProperties: false + properties: + location: + description: City name + type: string + unit: + enum: + - celsius + - fahrenheit + type: string + required: + - location + type: object + strict: true + type: function + - description: Get the current date and time in a given IANA timezone + name: get_time + parameters: + additionalProperties: false + properties: + timezone: + description: IANA timezone, e.g. Europe/Paris + type: string + required: + - timezone + type: object + strict: true + type: function + - description: Get the latest stock price and daily change for a ticker symbol + name: get_stock_price + parameters: + additionalProperties: false + properties: + currency: + description: Currency to return the price in, e.g. USD + type: string + ticker: + description: Stock ticker symbol, e.g. AAPL + type: string + required: + - ticker + type: object + strict: true + type: function + - description: Search the web and return the top results for a query + name: search_web + parameters: + additionalProperties: false + properties: + num_results: + default: 3 + description: Number of results to return (1-10) + type: integer + query: + description: Search query string + type: string + required: + - query + type: object + strict: true + type: function + - description: Translate text from one language to another + name: translate_text + parameters: + additionalProperties: false + properties: + source_language: + description: Source language code; omit for auto-detect + type: string + target_language: + description: Target language code, e.g. fr, de, ja + type: string + text: + description: Text to translate + type: string + required: + - text + - target_language + type: object + strict: true + type: function + - description: Evaluate a mathematical expression and return the numeric result + name: calculate + parameters: + additionalProperties: false + properties: + expression: + description: Math expression to evaluate, e.g. (12 * 8) / 3 + sqrt(16) + type: string + required: + - expression + type: object + strict: true + type: function + - description: Send an email to one or more recipients + name: send_email + parameters: + additionalProperties: false + properties: + body: + description: Plain-text email body + type: string + cc: + description: CC recipients (optional) + items: + type: string + type: array + subject: + description: Email subject line + type: string + to: + description: Recipient email addresses + items: + type: string + type: array + required: + - to + - subject + - body + type: object + strict: true + type: function + - description: Read the contents of a file at the given path + name: read_file + parameters: + additionalProperties: false + properties: + encoding: + description: File encoding + enum: + - utf-8 + - latin-1 + - ascii + type: string + path: + description: Absolute or relative file path + type: string + required: + - path + type: object + strict: true + type: function + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + background: false + billing: null + completed_at: 1787829076 + conversation: null + created_at: 1787829076 + error: null + frequency_penalty: 0.0 + id: resp_18180aae3e7849a098b11367649c1e96 + incomplete_details: null + instructions: null + max_output_tokens: 2048 + max_tool_calls: null + metadata: {} + model: openai/gpt-oss-20b + object: response + output: + - arguments: '{"ticker":"NVDA"}' + call_id: call-4be0f84c-b6d7-4321-89db-29c4bc7d46dc + id: fc_7469dd39282341ce9406eb3efba386d7 + name: get_stock_price + status: completed + type: function_call + parallel_tool_calls: true + presence_penalty: 0.0 + previous_response_id: null + prompt: null + prompt_cache_key: null + prompt_cache_retention: null + reasoning: null + safety_identifier: null + service_tier: auto + status: completed + store: true + temperature: 1.0 + text: + format: + type: text + tool_choice: auto + tools: + - description: Get current temperature and conditions for a city + name: get_weather + parameters: + additionalProperties: false + properties: + location: + description: City name + type: string + unit: + enum: + - celsius + - fahrenheit + type: string + required: + - location + type: object + strict: true + type: function + - description: Get the current date and time in a given IANA timezone + name: get_time + parameters: + additionalProperties: false + properties: + timezone: + description: IANA timezone, e.g. Europe/Paris + type: string + required: + - timezone + type: object + strict: true + type: function + - description: Get the latest stock price and daily change for a ticker symbol + name: get_stock_price + parameters: + additionalProperties: false + properties: + currency: + description: Currency to return the price in, e.g. USD + type: string + ticker: + description: Stock ticker symbol, e.g. AAPL + type: string + required: + - ticker + type: object + strict: true + type: function + - description: Search the web and return the top results for a query + name: search_web + parameters: + additionalProperties: false + properties: + num_results: + default: 3 + description: Number of results to return (1-10) + type: integer + query: + description: Search query string + type: string + required: + - query + type: object + strict: true + type: function + - description: Translate text from one language to another + name: translate_text + parameters: + additionalProperties: false + properties: + source_language: + description: Source language code; omit for auto-detect + type: string + target_language: + description: Target language code, e.g. fr, de, ja + type: string + text: + description: Text to translate + type: string + required: + - text + - target_language + type: object + strict: true + type: function + - description: Evaluate a mathematical expression and return the numeric result + name: calculate + parameters: + additionalProperties: false + properties: + expression: + description: Math expression to evaluate, e.g. (12 * 8) / 3 + sqrt(16) + type: string + required: + - expression + type: object + strict: true + type: function + - description: Send an email to one or more recipients + name: send_email + parameters: + additionalProperties: false + properties: + body: + description: Plain-text email body + type: string + cc: + description: CC recipients (optional) + items: + type: string + type: array + subject: + description: Email subject line + type: string + to: + description: Recipient email addresses + items: + type: string + type: array + required: + - to + - subject + - body + type: object + strict: true + type: function + - description: Read the contents of a file at the given path + name: read_file + parameters: + additionalProperties: false + properties: + encoding: + description: File encoding + enum: + - utf-8 + - latin-1 + - ascii + type: string + path: + description: Absolute or relative file path + type: string + required: + - path + type: object + strict: true + type: function + top_logprobs: 0 + top_p: 1.0 + truncation: disabled + usage: + input_tokens: 507 + input_tokens_details: + cached_tokens: 48 + output_tokens: 54 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 561 + headers: + content-type: application/json + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/dynamo/dynamo-tool-call-auto-openai-gpt-oss-20b-streaming.yaml b/crates/agentic-server-core/tests/cassettes/dynamo/dynamo-tool-call-auto-openai-gpt-oss-20b-streaming.yaml new file mode 100644 index 00000000..e4422ae9 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/dynamo/dynamo-tool-call-auto-openai-gpt-oss-20b-streaming.yaml @@ -0,0 +1,306 @@ +turns: +- filename: t1 + request: + body: + input: What is the current NVIDIA stock price? Use the tool. + max_output_tokens: 2048 + model: openai/gpt-oss-20b + store: true + stream: true + tool_choice: auto + tools: + - description: Get current temperature and conditions for a city + name: get_weather + parameters: + additionalProperties: false + properties: + location: + description: City name + type: string + unit: + enum: + - celsius + - fahrenheit + type: string + required: + - location + type: object + strict: true + type: function + - description: Get the current date and time in a given IANA timezone + name: get_time + parameters: + additionalProperties: false + properties: + timezone: + description: IANA timezone, e.g. Europe/Paris + type: string + required: + - timezone + type: object + strict: true + type: function + - description: Get the latest stock price and daily change for a ticker symbol + name: get_stock_price + parameters: + additionalProperties: false + properties: + currency: + description: Currency to return the price in, e.g. USD + type: string + ticker: + description: Stock ticker symbol, e.g. AAPL + type: string + required: + - ticker + type: object + strict: true + type: function + - description: Search the web and return the top results for a query + name: search_web + parameters: + additionalProperties: false + properties: + num_results: + default: 3 + description: Number of results to return (1-10) + type: integer + query: + description: Search query string + type: string + required: + - query + type: object + strict: true + type: function + - description: Translate text from one language to another + name: translate_text + parameters: + additionalProperties: false + properties: + source_language: + description: Source language code; omit for auto-detect + type: string + target_language: + description: Target language code, e.g. fr, de, ja + type: string + text: + description: Text to translate + type: string + required: + - text + - target_language + type: object + strict: true + type: function + - description: Evaluate a mathematical expression and return the numeric result + name: calculate + parameters: + additionalProperties: false + properties: + expression: + description: Math expression to evaluate, e.g. (12 * 8) / 3 + sqrt(16) + type: string + required: + - expression + type: object + strict: true + type: function + - description: Send an email to one or more recipients + name: send_email + parameters: + additionalProperties: false + properties: + body: + description: Plain-text email body + type: string + cc: + description: CC recipients (optional) + items: + type: string + type: array + subject: + description: Email subject line + type: string + to: + description: Recipient email addresses + items: + type: string + type: array + required: + - to + - subject + - body + type: object + strict: true + type: function + - description: Read the contents of a file at the given path + name: read_file + parameters: + additionalProperties: false + properties: + encoding: + description: File encoding + enum: + - utf-8 + - latin-1 + - ascii + type: string + path: + description: Absolute or relative file path + type: string + required: + - path + type: object + strict: true + type: function + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","sequence_number":0,"response":{"background":false,"billing":null,"conversation":null,"created_at":1787829085,"completed_at":null,"error":null,"id":"resp_acc92bcea2614732ba1ba2c064ae5db8","incomplete_details":null,"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":{},"model":"openai/gpt-oss-20b","object":"response","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":null,"safety_identifier":null,"service_tier":"auto","status":"in_progress","temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[{"type":"function","name":"get_weather","parameters":{"type":"object","properties":{"location":{"type":"string","description":"City + name"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"],"additionalProperties":false},"strict":true,"description":"Get + current temperature and conditions for a city"},{"type":"function","name":"get_time","parameters":{"type":"object","properties":{"timezone":{"type":"string","description":"IANA + timezone, e.g. Europe/Paris"}},"required":["timezone"],"additionalProperties":false},"strict":true,"description":"Get + the current date and time in a given IANA timezone"},{"type":"function","name":"get_stock_price","parameters":{"type":"object","properties":{"ticker":{"type":"string","description":"Stock + ticker symbol, e.g. AAPL"},"currency":{"type":"string","description":"Currency + to return the price in, e.g. USD"}},"required":["ticker"],"additionalProperties":false},"strict":true,"description":"Get + the latest stock price and daily change for a ticker symbol"},{"type":"function","name":"search_web","parameters":{"type":"object","properties":{"query":{"type":"string","description":"Search + query string"},"num_results":{"type":"integer","description":"Number of results + to return (1-10)","default":3}},"required":["query"],"additionalProperties":false},"strict":true,"description":"Search + the web and return the top results for a query"},{"type":"function","name":"translate_text","parameters":{"type":"object","properties":{"text":{"type":"string","description":"Text + to translate"},"target_language":{"type":"string","description":"Target language + code, e.g. fr, de, ja"},"source_language":{"type":"string","description":"Source + language code; omit for auto-detect"}},"required":["text","target_language"],"additionalProperties":false},"strict":true,"description":"Translate + text from one language to another"},{"type":"function","name":"calculate","parameters":{"type":"object","properties":{"expression":{"type":"string","description":"Math + expression to evaluate, e.g. (12 * 8) / 3 + sqrt(16)"}},"required":["expression"],"additionalProperties":false},"strict":true,"description":"Evaluate + a mathematical expression and return the numeric result"},{"type":"function","name":"send_email","parameters":{"type":"object","properties":{"to":{"type":"array","items":{"type":"string"},"description":"Recipient + email addresses"},"subject":{"type":"string","description":"Email subject line"},"body":{"type":"string","description":"Plain-text + email body"},"cc":{"type":"array","items":{"type":"string"},"description":"CC + recipients (optional)"}},"required":["to","subject","body"],"additionalProperties":false},"strict":true,"description":"Send + an email to one or more recipients"},{"type":"function","name":"read_file","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Absolute + or relative file path"},"encoding":{"type":"string","enum":["utf-8","latin-1","ascii"],"description":"File + encoding"}},"required":["path"],"additionalProperties":false},"strict":true,"description":"Read + the contents of a file at the given path"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"presence_penalty":0.0,"frequency_penalty":0.0,"store":true}} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","sequence_number":1,"response":{"background":false,"billing":null,"conversation":null,"created_at":1787829085,"completed_at":null,"error":null,"id":"resp_acc92bcea2614732ba1ba2c064ae5db8","incomplete_details":null,"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":{},"model":"openai/gpt-oss-20b","object":"response","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":null,"safety_identifier":null,"service_tier":"auto","status":"in_progress","temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[{"type":"function","name":"get_weather","parameters":{"type":"object","properties":{"location":{"type":"string","description":"City + name"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"],"additionalProperties":false},"strict":true,"description":"Get + current temperature and conditions for a city"},{"type":"function","name":"get_time","parameters":{"type":"object","properties":{"timezone":{"type":"string","description":"IANA + timezone, e.g. Europe/Paris"}},"required":["timezone"],"additionalProperties":false},"strict":true,"description":"Get + the current date and time in a given IANA timezone"},{"type":"function","name":"get_stock_price","parameters":{"type":"object","properties":{"ticker":{"type":"string","description":"Stock + ticker symbol, e.g. AAPL"},"currency":{"type":"string","description":"Currency + to return the price in, e.g. USD"}},"required":["ticker"],"additionalProperties":false},"strict":true,"description":"Get + the latest stock price and daily change for a ticker symbol"},{"type":"function","name":"search_web","parameters":{"type":"object","properties":{"query":{"type":"string","description":"Search + query string"},"num_results":{"type":"integer","description":"Number of results + to return (1-10)","default":3}},"required":["query"],"additionalProperties":false},"strict":true,"description":"Search + the web and return the top results for a query"},{"type":"function","name":"translate_text","parameters":{"type":"object","properties":{"text":{"type":"string","description":"Text + to translate"},"target_language":{"type":"string","description":"Target language + code, e.g. fr, de, ja"},"source_language":{"type":"string","description":"Source + language code; omit for auto-detect"}},"required":["text","target_language"],"additionalProperties":false},"strict":true,"description":"Translate + text from one language to another"},{"type":"function","name":"calculate","parameters":{"type":"object","properties":{"expression":{"type":"string","description":"Math + expression to evaluate, e.g. (12 * 8) / 3 + sqrt(16)"}},"required":["expression"],"additionalProperties":false},"strict":true,"description":"Evaluate + a mathematical expression and return the numeric result"},{"type":"function","name":"send_email","parameters":{"type":"object","properties":{"to":{"type":"array","items":{"type":"string"},"description":"Recipient + email addresses"},"subject":{"type":"string","description":"Email subject line"},"body":{"type":"string","description":"Plain-text + email body"},"cc":{"type":"array","items":{"type":"string"},"description":"CC + recipients (optional)"}},"required":["to","subject","body"],"additionalProperties":false},"strict":true,"description":"Send + an email to one or more recipients"},{"type":"function","name":"read_file","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Absolute + or relative file path"},"encoding":{"type":"string","enum":["utf-8","latin-1","ascii"],"description":"File + encoding"}},"required":["path"],"additionalProperties":false},"strict":true,"description":"Read + the contents of a file at the given path"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"presence_penalty":0.0,"frequency_penalty":0.0,"store":true}} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"function_call","arguments":"","call_id":"call-1d299854-be41-4117-bce5-43c2ebf09282","name":"get_stock_price","id":"fc_a10c7c3220ba4090b01a38b49620c55c","status":"in_progress"}} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","sequence_number":3,"item_id":"fc_a10c7c3220ba4090b01a38b49620c55c","output_index":0,"delta":"{\"ticker\":\"NVDA\",\"currency\":\"USD\"}"} + + ' + - ' + + ' + - 'event: response.function_call_arguments.done + + ' + - 'data: {"type":"response.function_call_arguments.done","name":"get_stock_price","sequence_number":4,"item_id":"fc_a10c7c3220ba4090b01a38b49620c55c","output_index":0,"arguments":"{\"ticker\":\"NVDA\",\"currency\":\"USD\"}"} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","sequence_number":5,"output_index":0,"item":{"type":"function_call","arguments":"{\"ticker\":\"NVDA\",\"currency\":\"USD\"}","call_id":"call-1d299854-be41-4117-bce5-43c2ebf09282","name":"get_stock_price","id":"fc_a10c7c3220ba4090b01a38b49620c55c","status":"completed"}} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","sequence_number":6,"response":{"background":false,"billing":null,"conversation":null,"created_at":1787829085,"completed_at":1787829086,"error":null,"id":"resp_acc92bcea2614732ba1ba2c064ae5db8","incomplete_details":null,"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":{},"model":"openai/gpt-oss-20b","object":"response","output":[{"type":"function_call","arguments":"{\"ticker\":\"NVDA\",\"currency\":\"USD\"}","call_id":"call-1d299854-be41-4117-bce5-43c2ebf09282","name":"get_stock_price","id":"fc_a10c7c3220ba4090b01a38b49620c55c","status":"completed"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":null,"safety_identifier":null,"service_tier":"auto","status":"completed","temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[{"type":"function","name":"get_weather","parameters":{"type":"object","properties":{"location":{"type":"string","description":"City + name"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"],"additionalProperties":false},"strict":true,"description":"Get + current temperature and conditions for a city"},{"type":"function","name":"get_time","parameters":{"type":"object","properties":{"timezone":{"type":"string","description":"IANA + timezone, e.g. Europe/Paris"}},"required":["timezone"],"additionalProperties":false},"strict":true,"description":"Get + the current date and time in a given IANA timezone"},{"type":"function","name":"get_stock_price","parameters":{"type":"object","properties":{"ticker":{"type":"string","description":"Stock + ticker symbol, e.g. AAPL"},"currency":{"type":"string","description":"Currency + to return the price in, e.g. USD"}},"required":["ticker"],"additionalProperties":false},"strict":true,"description":"Get + the latest stock price and daily change for a ticker symbol"},{"type":"function","name":"search_web","parameters":{"type":"object","properties":{"query":{"type":"string","description":"Search + query string"},"num_results":{"type":"integer","description":"Number of results + to return (1-10)","default":3}},"required":["query"],"additionalProperties":false},"strict":true,"description":"Search + the web and return the top results for a query"},{"type":"function","name":"translate_text","parameters":{"type":"object","properties":{"text":{"type":"string","description":"Text + to translate"},"target_language":{"type":"string","description":"Target language + code, e.g. fr, de, ja"},"source_language":{"type":"string","description":"Source + language code; omit for auto-detect"}},"required":["text","target_language"],"additionalProperties":false},"strict":true,"description":"Translate + text from one language to another"},{"type":"function","name":"calculate","parameters":{"type":"object","properties":{"expression":{"type":"string","description":"Math + expression to evaluate, e.g. (12 * 8) / 3 + sqrt(16)"}},"required":["expression"],"additionalProperties":false},"strict":true,"description":"Evaluate + a mathematical expression and return the numeric result"},{"type":"function","name":"send_email","parameters":{"type":"object","properties":{"to":{"type":"array","items":{"type":"string"},"description":"Recipient + email addresses"},"subject":{"type":"string","description":"Email subject line"},"body":{"type":"string","description":"Plain-text + email body"},"cc":{"type":"array","items":{"type":"string"},"description":"CC + recipients (optional)"}},"required":["to","subject","body"],"additionalProperties":false},"strict":true,"description":"Send + an email to one or more recipients"},{"type":"function","name":"read_file","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Absolute + or relative file path"},"encoding":{"type":"string","enum":["utf-8","latin-1","ascii"],"description":"File + encoding"}},"required":["path"],"additionalProperties":false},"strict":true,"description":"Read + the contents of a file at the given path"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":507,"input_tokens_details":{"cached_tokens":496},"output_tokens":59,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":566},"presence_penalty":0.0,"frequency_penalty":0.0,"store":true}} + + ' + - ' + + ' + - 'data: [DONE] + + ' + - ' + + ' + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/record_dynamo_cassettes.sh b/crates/agentic-server-core/tests/cassettes/record_dynamo_cassettes.sh new file mode 100755 index 00000000..92d1bedc --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/record_dynamo_cassettes.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# record_dynamo_cassettes.sh +# +# Records cassettes from an NVIDIA Dynamo frontend (python -m dynamo.frontend) serving a vLLM worker. +# Dynamo exposes the same /v1/responses surface as vLLM, so --vllm points at the Dynamo HTTP port. +# +# Dynamo's /v1/responses is stateless: it rejects `previous_response_id` with 501. The gateway therefore +# rehydrates the conversation and sends the full item history upstream. The second stateful turn is recorded +# from that hydrated item history, built from turn 1's recorded assistant message, so the cassette mirrors +# real gateway traffic including the item id. +# +# Prerequisites (see docs/guides/dynamo-upstream.md): +# python -m dynamo.frontend --http-port 8000 --discovery-backend file +# python -m dynamo.vllm --model openai/gpt-oss-20b --discovery-backend file \ +# --kv-events-config '{"enable_kv_cache_events": false}' \ +# --dyn-reasoning-parser gpt_oss --dyn-tool-call-parser harmony +# (recorded with ai-dynamo==1.4.1; pin the version, --prerelease=allow selects dev builds) +# +# Usage: +# DYNAMO_URL=http://127.0.0.1:8000 MODEL=openai/gpt-oss-20b bash tests/cassettes/record_dynamo_cassettes.sh + +set -euo pipefail + +SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BASE_DIR="$SCRIPTS_DIR/dynamo" +TOOLS_FILE="$SCRIPTS_DIR/tool_calls/tools.json" +DYNAMO_URL="${DYNAMO_URL:-http://127.0.0.1:8000}" +MODEL="${MODEL:-openai/gpt-oss-20b}" +MODEL_SLUG="$(echo "$MODEL" | tr '/: ' '---')" +PYTHON="${PYTHON:-python}" + +green() { printf '\033[32m%s\033[0m\n' "$*"; } +bold() { printf '\033[1m%s\033[0m\n' "$*"; } + +mkdir -p "$BASE_DIR" + +bold "Dynamo URL: $DYNAMO_URL" +bold "Model: $MODEL" +echo + +# record NAME STREAM_FLAG PROMPT [extra recorder args...] +record() { + local name="$1" stream_flag="$2" prompt="$3"; shift 3 + local suffix; [[ -n "$stream_flag" ]] && suffix=nonstreaming || suffix=streaming + bold "── $name ($suffix) ──" + record_into "$BASE_DIR/${name}-${MODEL_SLUG}-${suffix}.yaml" "$stream_flag" "$prompt" "$@" + green "✓ $name ($suffix) done." +} + +# append_turn NAME STREAM_FLAG PROMPT [extra recorder args...] +# The recorder truncates its output file, so extra turns are recorded separately and merged. +append_turn() { + local name="$1" stream_flag="$2" prompt="$3"; shift 3 + local suffix; [[ -n "$stream_flag" ]] && suffix=nonstreaming || suffix=streaming + local target="$BASE_DIR/${name}-${MODEL_SLUG}-${suffix}.yaml" + local extra="${target%.yaml}.next-turn.yaml" + bold "── $name ($suffix), next turn ──" + record_into "$extra" "$stream_flag" "$prompt" "$@" + $PYTHON - "$target" "$extra" <<'PY' +import sys, yaml +target, extra = sys.argv[1], sys.argv[2] +merged = yaml.safe_load(open(target)) +for turn in yaml.safe_load(open(extra))["turns"]: + turn["filename"] = f"t{len(merged['turns']) + 1}" + merged["turns"].append(turn) +yaml.safe_dump(merged, open(target, "w"), sort_keys=False, allow_unicode=True, width=10**9) +PY + rm -f "$extra" + green "✓ $name ($suffix) turn appended." +} + +record_into() { + local output="$1" stream_flag="$2" prompt="$3"; shift 3 + # shellcheck disable=SC2086 + printf '%s\n' "$prompt" | $PYTHON "$SCRIPTS_DIR/record_cassette.py" \ + --mode responses \ + --turns 1 \ + --model "$MODEL" \ + --vllm "$DYNAMO_URL" \ + --max-output-tokens 2048 \ + $stream_flag \ + "$@" \ + --output "$output" +} + +# hydrated_turn2_input CASSETTE OUT_JSON +# Writes the item history the gateway sends for turn 2: the user prompt, the assistant message exactly as +# recorded in turn 1 (same id and text), and the follow-up user prompt. +hydrated_turn2_input() { + $PYTHON - "$1" "$2" "$TURN1_PROMPT" "$TURN2_PROMPT" <<'PY' +import json, sys, yaml +cassette, out, turn1, turn2 = sys.argv[1:5] +response = yaml.safe_load(open(cassette))["turns"][0]["response"] +if response.get("body"): + completed = response["body"] +else: + completed = next( + json.loads(line[len("data: "):])["response"] + for raw in response["sse"] + for line in raw.splitlines() + if line.startswith("data: ") and json.loads(line[len("data: "):]).get("type") == "response.completed" + ) +assistant = next(item for item in completed["output"] if item["type"] == "message") +history = [ + {"type": "message", "role": "user", "content": turn1}, + { + "type": "message", + "id": assistant["id"], + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": part["text"]} for part in assistant["content"]], + }, + {"type": "message", "role": "user", "content": turn2}, +] +json.dump(history, open(out, "w"), indent=2) +PY +} + +TURN1_PROMPT="Remember the word APPLE. Just say: OK" +TURN2_PROMPT="What word did I ask you to remember? Reply with just the word." + +for stream_flag in --no-stream ""; do + [[ -n "$stream_flag" ]] && suffix=nonstreaming || suffix=streaming + record dynamo-stateful "$stream_flag" "$TURN1_PROMPT" + turn2_input="$(mktemp --suffix=.json)" + hydrated_turn2_input "$BASE_DIR/dynamo-stateful-${MODEL_SLUG}-${suffix}.yaml" "$turn2_input" + append_turn dynamo-stateful "$stream_flag" "" --input-file "$turn2_input" + rm -f "$turn2_input" + record dynamo-tool-call-auto "$stream_flag" "What is the current NVIDIA stock price? Use the tool." \ + --tools "$TOOLS_FILE" --tool-choice auto +done + +echo +green "All Dynamo cassettes recorded -> $BASE_DIR" diff --git a/crates/agentic-server-core/tests/dynamo_cassette_test.rs b/crates/agentic-server-core/tests/dynamo_cassette_test.rs new file mode 100644 index 00000000..7a4e94c8 --- /dev/null +++ b/crates/agentic-server-core/tests/dynamo_cassette_test.rs @@ -0,0 +1,190 @@ +//! Cassette replay tests for NVIDIA Dynamo as the inference upstream. +//! +//! The recordings in `tests/cassettes/dynamo` were captured from a Dynamo +//! frontend (`python -m dynamo.frontend`) fronting a `dynamo.vllm` worker +//! serving `openai/gpt-oss-20b`. Dynamo speaks the same `/v1/responses` wire +//! format as vLLM but is stateless: it rejects `previous_response_id` with +//! `501 Not Implemented`. These tests pin the gateway behavior that makes +//! Dynamo usable as an upstream anyway — the gateway owns the conversation +//! state and sends the fully rehydrated item history on every turn. The +//! function-call cassettes cover a client-executed function tool. +//! +//! Re-record with `tests/cassettes/record_dynamo_cassettes.sh`. + +mod support; + +use agentic_core::executor::execute; +use agentic_core::types::io::OutputItem; +use agentic_core::types::request_response::ResponsePayload; +use agentic_core::types::tools::ResponsesTool; +use serde_json::Value; +use std::sync::Arc; +use support::{ + TestFixture, Turn, collect_stream, expected_text, load_cassette, make_request, output_text, request_input_texts, + unwrap_blocking, +}; + +const DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/cassettes/dynamo"); +const MODEL_SLUG: &str = "openai-gpt-oss-20b"; + +const TURN1_PROMPT: &str = "Remember the word APPLE. Just say: OK"; +const TURN2_PROMPT: &str = "What word did I ask you to remember? Reply with just the word."; +const TOOL_PROMPT: &str = "What is the current NVIDIA stock price? Use the tool."; + +fn cassette_path(name: &str, streaming: bool) -> String { + let suffix = if streaming { "streaming" } else { "nonstreaming" }; + format!("{DIR}/{name}-{MODEL_SLUG}-{suffix}.yaml") +} + +fn turn2_prompt_from(turn: &Turn) -> String { + request_input_texts(&serde_json::json!({ "input": turn.request.body.input })) + .pop() + .expect("turn 2 recording ends with the user prompt") +} + +fn function_calls(payload: &ResponsePayload) -> Vec<(String, String)> { + payload + .output + .iter() + .filter_map(|item| match item { + OutputItem::FunctionCall(fc) => Some((fc.name.clone(), fc.arguments.clone())), + _ => None, + }) + .collect() +} + +fn first_message_id(payload: &ResponsePayload) -> &str { + payload + .output + .iter() + .find_map(|item| match item { + OutputItem::Message(msg) => Some(msg.id.as_str()), + _ => None, + }) + .expect("turn 1 output contains an assistant message") +} + +/// The upstream request for the second turn must carry the rehydrated item +/// history instead of `previous_response_id`, because Dynamo rejects the latter. +/// The history is compared structurally with the recorded turn-2 request; only +/// the assistant message id differs per run, so it is taken from turn 1's payload. +fn assert_upstream_requests_are_stateless(requests: &[Value], t2: &Turn, p1: &ResponsePayload) { + assert_eq!(requests.len(), 2, "one upstream call per turn"); + for request in requests { + assert!( + request.get("previous_response_id").is_none(), + "Dynamo returns 501 for previous_response_id, even when null; upstream request was {request}" + ); + } + assert_eq!(request_input_texts(&requests[0]), vec![TURN1_PROMPT]); + + let mut expected_history = t2.request.body.input.clone(); + let recorded_assistant = &mut expected_history[1]; + assert_eq!( + recorded_assistant["role"], "assistant", + "recorded turn 2 replays the assistant item" + ); + recorded_assistant["id"] = Value::String(first_message_id(p1).to_owned()); + assert_eq!( + requests[1]["input"], expected_history, + "turn 2 must replay the full item history to the stateless upstream" + ); +} + +async fn run_stateful_two_turn(streaming: bool) { + let cassette = load_cassette(&cassette_path("dynamo-stateful", streaming)); + let (t1, t2) = (&cassette.turns[0], &cassette.turns[1]); + assert_eq!(turn2_prompt_from(t2), TURN2_PROMPT); + let fixture = TestFixture::new(&[t1, t2]).await; + + let first = execute( + make_request(TURN1_PROMPT, true, streaming, None, None), + Arc::clone(&fixture.exec_ctx), + ) + .await + .expect("t1"); + let p1 = if streaming { + collect_stream(first).await + } else { + unwrap_blocking(first) + }; + let second = execute( + make_request(TURN2_PROMPT, true, streaming, Some(p1.id.clone()), None), + Arc::clone(&fixture.exec_ctx), + ) + .await + .expect("t2"); + let p2 = if streaming { + collect_stream(second).await + } else { + unwrap_blocking(second) + }; + + assert_eq!(p1.status, "completed"); + assert_eq!(output_text(&p1), expected_text(t1)); + assert_eq!(output_text(&p1), "OK"); + assert_ne!(p2.id, p1.id); + assert_eq!(p2.status, "completed"); + assert_eq!(p2.previous_response_id.as_deref(), Some(p1.id.as_str())); + assert_eq!(output_text(&p2), expected_text(t2)); + assert_eq!(output_text(&p2), "APPLE"); + + assert_upstream_requests_are_stateless(&fixture.request_bodies().await, t2, &p1); +} + +#[tokio::test] +async fn dynamo_stateful_two_turn_nonstreaming() { + run_stateful_two_turn(false).await; +} + +#[tokio::test] +async fn dynamo_stateful_two_turn_streaming() { + run_stateful_two_turn(true).await; +} + +async fn run_function_tool_call(streaming: bool) { + let cassette = load_cassette(&cassette_path("dynamo-tool-call-auto", streaming)); + let t1 = &cassette.turns[0]; + let tools: Vec = + serde_json::from_value(Value::Array(t1.request.body.tools.clone())).expect("recorded tools parse"); + let fixture = TestFixture::new(&[t1]).await; + + let mut request = make_request(TOOL_PROMPT, true, streaming, None, None); + request.tools = Some(tools); + let result = execute(request, Arc::clone(&fixture.exec_ctx)).await.expect("t1"); + let payload = if streaming { + collect_stream(result).await + } else { + unwrap_blocking(result) + }; + + assert_eq!(payload.status, "completed"); + let calls = function_calls(&payload); + assert_eq!( + calls.len(), + 1, + "Dynamo's harmony parser yields one function_call: {calls:?}" + ); + let (name, arguments) = &calls[0]; + assert_eq!(name, "get_stock_price"); + let arguments: Value = serde_json::from_str(arguments).expect("arguments are JSON"); + assert_eq!(arguments["ticker"], "NVDA"); + + let requests = fixture.request_bodies().await; + assert_eq!(requests.len(), 1, "client-executed function tools take one model call"); + let upstream_tools = requests[0]["tools"].as_array().expect("tools forwarded upstream"); + assert!( + upstream_tools.iter().any(|tool| tool["name"] == "get_stock_price"), + "function declarations must reach Dynamo unchanged" + ); +} + +#[tokio::test] +async fn dynamo_function_tool_call_nonstreaming() { + run_function_tool_call(false).await; +} + +#[tokio::test] +async fn dynamo_function_tool_call_streaming() { + run_function_tool_call(true).await; +} diff --git a/docs/guides/dynamo-upstream.md b/docs/guides/dynamo-upstream.md new file mode 100644 index 00000000..a6eade5e --- /dev/null +++ b/docs/guides/dynamo-upstream.md @@ -0,0 +1,172 @@ +# Running Agentic API in front of NVIDIA Dynamo + +[NVIDIA Dynamo](https://github.com/ai-dynamo/dynamo) is a distributed inference serving framework. Its frontend +exposes an OpenAI-compatible HTTP API, including `/v1/responses`, and routes requests to backend workers. Dynamo ships +its own workers (vLLM, SGLang, TensorRT-LLM); the vLLM worker (`python -m dynamo.vllm`) embeds the vLLM engine, so a +Dynamo deployment is a complete inference stack on its own. You do not run `vllm serve` alongside it. + +This guide records how to put Agentic API in front of a Dynamo deployment and what differs from pointing the gateway +at a standalone `vllm serve`. + +The short version: nothing in Agentic API needs to change. Start the gateway with `--llm-api-base` pointing at the +Dynamo frontend and it works, including stateful `previous_response_id` chaining and client-executed function tools. + +## What Dynamo does and does not provide + +| Capability | Dynamo frontend | Agentic API adds | +|---|---|---| +| `POST /v1/responses`, `/v1/chat/completions`, `/v1/models`, `/health` | ✅ | — | +| Reasoning / tool-call parsing | ✅ via `--dyn-reasoning-parser` / `--dyn-tool-call-parser` on the worker | — | +| `previous_response_id` | ❌ returns `501 Not Implemented` (`Validation: previous_response_id is not supported.`) | ✅ Stores every response and rehydrates the full item history on each turn, so the upstream call is stateless | +| Gateway-executed built-in tools (web search, MCP), background execution, WebSocket transport | ❌ | ✅ | + +Because Dynamo rejects `previous_response_id`, the gateway never forwards it. The second turn of a conversation reaches +Dynamo as one `input` array containing the earlier user message, the stored assistant message, and the new user +message. The replay tests in `crates/agentic-server-core/tests/dynamo_cassette_test.rs` assert exactly that shape. + +## 1. Install Dynamo + +Dynamo publishes wheels on PyPI. The `[vllm]` extra pulls in the vLLM version Dynamo's worker is built against, so use +a dedicated virtual environment, and pin the Dynamo release: + +```bash +mkdir -p ~/dev/dynamo && cd ~/dev/dynamo +uv venv --python 3.12 .venv +VIRTUAL_ENV=$PWD/.venv uv pip install "ai-dynamo[vllm]==1.4.1" +``` + +Pin the version. Dynamo's README suggests `uv pip install --prerelease=allow "ai-dynamo[vllm]"`, but that flag lets uv +resolve *any* dependency to a pre-release and, with an unpinned `ai-dynamo`, installs the latest `1.5.0.devYYYYMMDD` +build rather than a release. Check what you got with `python -c 'import importlib.metadata as m; print(m.version("ai-dynamo"))'`. + +This guide was verified with `ai-dynamo==1.4.1` (which installs `vllm==0.26.0` and `torch` cu130) on an aarch64 host +with a single GB10 GPU. See Dynamo's [release artifacts](https://docs.nvidia.com/dynamo/resources/release-artifacts) +and [support matrix](https://docs.nvidia.com/dynamo/resources/support-matrix) for the wheel/CUDA combinations of other +releases. No etcd or NATS is needed for a single-host setup when the components use file-based discovery. + +## 2. Start the Dynamo frontend and a worker + +Run each in its own terminal (or tmux window). The frontend is the HTTP entry point (default port 8000, round-robin +routing across whatever workers register); the worker loads the model. Models are resolved from the Hugging Face +cache, so anything already downloaded for vLLM is reused. + +```bash +# Frontend: OpenAI-compatible HTTP on :8000 +.venv/bin/python -m dynamo.frontend --http-port 8000 --discovery-backend file + +# Worker: vLLM engine managed by Dynamo +.venv/bin/python -m dynamo.vllm \ + --model openai/gpt-oss-20b \ + --discovery-backend file \ + --kv-events-config '{"enable_kv_cache_events": false}' \ + --dyn-reasoning-parser gpt_oss \ + --dyn-tool-call-parser harmony \ + --max-model-len 32768 +``` + +`openai/gpt-oss-20b` needs roughly 16 GB of GPU memory for weights plus KV cache, so it fits a single 24 GB GPU with +vLLM's default `--gpu-memory-utilization 0.9`. Lower that only when the GPU is shared (see below). + +Flags worth knowing: + +| Flag | Why | +|---|---| +| `--discovery-backend file` | Lets the frontend and worker find each other via `/tmp/dynamo_store_kv` instead of etcd. Pass it to both. | +| `--kv-events-config '{"enable_kv_cache_events": false}'` | Required for the vLLM worker without NATS. | +| `--dyn-reasoning-parser` / `--dyn-tool-call-parser` | The Dynamo *frontend* parses model output, not vLLM. vLLM's `--reasoning-parser` is ignored and `--tool-call-parser` / `--enable-auto-tool-choice` are rejected as unknown arguments. Without the `--dyn-*` flags, gpt-oss "analysis" text leaks into `content` and tool calls come back as plain text. `gpt_oss` + `harmony` is the verified pair for gpt-oss models; `python -m dynamo.vllm --help` lists the parsers for other model families. | +| `--gpu-memory-utilization` | Standard vLLM engine flag (the worker accepts vLLM engine arguments). It is a fraction of *total* device memory and must fit in the memory currently free, or the engine fails at startup. On a dedicated GPU keep the default; on a shared GPU size it to what is actually free (the recordings for this guide used `0.15` on a 121 GB unified-memory host that was also running another model). | + +Confirm the worker registered and parsing works: + +```bash +curl -s localhost:8000/v1/models | jq '.data[].id' +curl -s localhost:8000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "Say hello in five words."}], + "max_tokens": 500 +}' | jq '.choices[0].message | {content, reasoning_content}' +``` + +`content` should hold the answer and `reasoning_content` the chain of thought. If the answer starts with `analysis`, the +worker is missing `--dyn-reasoning-parser`. + +## 3. Start Agentic API against the Dynamo frontend + +```bash +cargo build -p agentic-server --bins +./target/debug/agentic-server --llm-api-base http://127.0.0.1:8000 +``` + +Agentic API's startup probe uses Dynamo's `/health`, so no `--skip-llm-ready-check` is needed. Be aware of what that +probe means: Dynamo returns `200 {"status":"healthy", ...}` as soon as the frontend's HTTP service is up, even with no +worker registered (the `instances` list is simply empty). It does **not** mean a model is loaded. Wait for the +per-model readiness endpoint before sending traffic: + +```bash +curl -s localhost:8000/v1/models/openai%2Fgpt-oss-20b/ready # 404 "Model not found" until the worker registers, + # then {"model": "...", "ready": true, ...} +``` + +The harness CLI works the same way: `./target/debug/agentic run codex --upstream http://127.0.0.1:8000`. + +## 4. Verify a stateful conversation and a tool call + +```bash +R1=$(curl -s localhost:9000/v1/responses -H 'Content-Type: application/json' -d '{ + "model": "openai/gpt-oss-20b", + "input": "Remember the word APPLE. Just say: OK", + "max_output_tokens": 2048 +}') +echo "$R1" | jq -r '.output[] | select(.type=="message") | .content[0].text' # OK + +curl -s localhost:9000/v1/responses -H 'Content-Type: application/json' -d "{ + \"model\": \"openai/gpt-oss-20b\", + \"input\": \"What word did I ask you to remember? Reply with just the word.\", + \"previous_response_id\": \"$(echo "$R1" | jq -r .id)\", + \"max_output_tokens\": 2048 +}" | jq -r '.output[] | select(.type=="message") | .content[0].text' # APPLE + +curl -s localhost:9000/v1/responses -H 'Content-Type: application/json' -d '{ + "model": "openai/gpt-oss-20b", + "input": "What is the current NVIDIA stock price? Use the tool.", + "max_output_tokens": 2048, + "tools": [{"type": "function", "name": "get_stock_price", + "description": "Get the latest stock price for a ticker symbol", + "parameters": {"type": "object", "properties": {"ticker": {"type": "string"}}, "required": ["ticker"]}}] +}' | jq '.output[] | select(.type=="function_call") | {name, arguments}' +``` + +Expected: `OK`, then `APPLE`, then a `get_stock_price` call with `{"ticker":"NVDA", ...}`. + +Sending the second request straight to Dynamo (port 8000) instead of the gateway fails with `501`; that difference is +the value the gateway adds. The third request exercises a client-executed function tool: Dynamo returns the function +call and the application runs it. + +## Recorded cassettes and CI + +The interactions above are recorded in `crates/agentic-server-core/tests/cassettes/dynamo/` and replayed by +`tests/dynamo_cassette_test.rs` on every `cargo test`, so CI covers the Dynamo upstream without a GPU. The +`dynamo-upstream` CI job also runs `scripts/validate-cassettes.py`, a structural check over every recorded cassette. To re-record +against a live Dynamo (for example after a Dynamo release changes the response shape): + +```bash +cd crates/agentic-server-core +DYNAMO_URL=http://127.0.0.1:8000 MODEL=openai/gpt-oss-20b \ + bash tests/cassettes/record_dynamo_cassettes.sh +``` + +The script records the second stateful turn from the hydrated item history the gateway would send (built from turn +1's recorded assistant message), because the recorder's own `previous_response_id` chaining cannot be used against a +stateless upstream. + +## Troubleshooting + +| Symptom | Cause / fix | +|---|---| +| `unrecognized arguments: --tool-call-parser` | Use `--dyn-tool-call-parser` on the worker. | +| Answer text begins with `analysis…assistantfinal…` | Add `--dyn-reasoning-parser gpt_oss` (or the parser for your model). | +| `Free memory on device … is less than desired GPU memory utilization` | Lower `--gpu-memory-utilization`; it is a fraction of total memory. | +| `CUDA error: out of memory` right after restarting a worker | A previous `dynamo.vllm` process is still alive and holding memory; `pkill -f "python -m dynamo.vllm"` before relaunching. Closing its terminal or tmux window does not kill it. | +| `501 Validation: previous_response_id is not supported.` | You are calling Dynamo directly. Send the request to the gateway. | +| Gateway logs `LLM ready` but requests fail with no model / `model not found` | `/health` is green before the worker registers. Check `/v1/models` or `/v1/models/{model}/ready` and the worker log. | +| Installed version is `1.5.0.dev…` | `--prerelease=allow` with an unpinned `ai-dynamo` picked a dev build. Reinstall with `"ai-dynamo[vllm]==1.4.1"`. | diff --git a/mkdocs.yaml b/mkdocs.yaml index 6bb344da..393a194e 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -98,6 +98,7 @@ nav: - Guides: - Responses Compaction: guides/responses-compaction.md - Harness CLI Testing: guides/harness-cli-testing.md + - NVIDIA Dynamo Upstream: guides/dynamo-upstream.md - Developing: - Getting Started: developing/getting-started.md - Architecture: diff --git a/scripts/validate-cassettes.py b/scripts/validate-cassettes.py new file mode 100755 index 00000000..a056223f --- /dev/null +++ b/scripts/validate-cassettes.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Structurally validate every recorded cassette under a directory. + +Cassettes are YAML files with a ``turns`` list (see +crates/agentic-server-core/tests/cassettes/README.md). This checks the parts +every replay test relies on, independent of which upstream recorded them: + +- ``turns`` is a non-empty list; each turn names a request ``method`` and + ``path`` and carries a JSON-object ``body``. +- Each response has an integer ``status_code`` and exactly one of ``body`` + (non-streaming) or ``sse`` (streaming). +- Every ``data:`` line in an ``sse`` recording is valid JSON, and a 2xx + streaming recording terminates: a ``response.completed``-style event, a + ``message_stop`` event (Messages), or a ``data: [DONE]`` marker. + +Usage: scripts/validate-cassettes.py [CASSETTE_DIR ...] +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import yaml + +DEFAULT_DIRS = [Path("crates/agentic-server-core/tests/cassettes")] +TERMINAL_EVENTS = {"response.completed", "response.incomplete", "response.failed", "message_stop"} + + +def sse_events(sse: list[str], where: str) -> tuple[list[dict], bool]: + """Return the decoded ``data:`` events and whether a ``[DONE]`` marker was seen.""" + events = [] + done = False + for raw in sse: + for line in raw.splitlines(): + if not line.startswith("data: "): + continue + payload = line[len("data: ") :] + if payload.strip() == "[DONE]": + done = True + continue + try: + events.append(json.loads(payload)) + except json.JSONDecodeError as error: + raise ValueError(f"{where}: SSE data line is not JSON ({error}): {payload[:120]}") from error + return events, done + + +def validate_turn(turn: dict, where: str) -> None: + request = turn.get("request") + if not isinstance(request, dict): + raise ValueError(f"{where}: missing request") + if not isinstance(request.get("path"), str) or not request["path"].startswith("/"): + raise ValueError(f"{where}: request.path must be an absolute path") + if not isinstance(request.get("method"), str): + raise ValueError(f"{where}: request.method missing") + if not isinstance(request.get("body"), dict): + raise ValueError(f"{where}: request.body must be a JSON object") + + response = turn.get("response") + if not isinstance(response, dict): + raise ValueError(f"{where}: missing response") + status = response.get("status_code") + if not isinstance(status, int): + raise ValueError(f"{where}: response.status_code must be an integer") + has_body = response.get("body") is not None + has_sse = response.get("sse") is not None + if has_body == has_sse: + raise ValueError(f"{where}: response must have exactly one of body or sse") + if has_sse: + if not isinstance(response["sse"], list): + raise ValueError(f"{where}: response.sse must be a list of raw SSE strings") + events, done = sse_events(response["sse"], where) + if 200 <= status < 300: + if not events: + raise ValueError(f"{where}: streaming response has no events") + if not done and events[-1].get("type") not in TERMINAL_EVENTS: + raise ValueError(f"{where}: streaming response does not end with a terminal event or [DONE]") + + +def validate_cassette(path: Path) -> int: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + turns = data.get("turns") if isinstance(data, dict) else None + if not isinstance(turns, list) or not turns: + raise ValueError(f"{path}: expected a non-empty turns list") + for index, turn in enumerate(turns, start=1): + validate_turn(turn, f"{path} turn {index}") + return len(turns) + + +def main(argv: list[str]) -> int: + roots = [Path(arg) for arg in argv] or DEFAULT_DIRS + files = sorted(file for root in roots for file in root.rglob("*.yaml") if "turns:" in file.read_text("utf-8")) + if not files: + print(f"no cassettes found under {', '.join(map(str, roots))}", file=sys.stderr) + return 1 + failures = 0 + total_turns = 0 + for file in files: + try: + total_turns += validate_cassette(file) + except (ValueError, yaml.YAMLError) as error: + failures += 1 + print(f"FAIL {error}", file=sys.stderr) + print(f"validated {len(files) - failures}/{len(files)} cassettes, {total_turns} turns") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))