From 9d98dbd38d93ca0ec4ecf41e49f228e8229d0132 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 25 Jun 2026 10:46:08 -0700 Subject: [PATCH 1/9] Add skill to replace hardcoded foundry project endpoint and model --- .../create_dynamic_workflow_executor.py | 10 +++- .../skills/foundry-config-setup/SKILL.md | 53 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index 6ebe25a8d4e..97b8524ec1e 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -3,10 +3,12 @@ import logging from collections import deque from dataclasses import dataclass +from pathlib import Path from agent_framework import ( Executor, Message, + SkillsProvider, Workflow, WorkflowBuilder, WorkflowContext, @@ -31,6 +33,9 @@ logger = logging.getLogger(__name__) +# Directory containing file-based skills used by the validation agents. +SKILLS_DIR = Path(__file__).parent / "skills" + class AgentResponseFormat(BaseModel): status: str @@ -58,7 +63,9 @@ class BatchCompletion: "Analyze the sample code and execute it as it is. Based on the execution result, determine " "if it runs successfully, fails, or is missing_setup. Use `missing_setup` if the sample reports " "missing required environment variables. The environment you're given should contain the necessary " - "variables. Don't create new environment variables nor modify the sample code.\n" + "variables. Don't create new environment variables nor modify the sample code, unless an available " + "skill instructs you to do so for the setup issue you detected. When a skill applies to the problem, " + "follow its guidance to resolve the setup and then re-run the sample.\n" "Feel free to install any required dependencies if needed.\n" "The sample can be interactive. If it is interactive, respond to the sample when prompted " "based on your analysis of the code. You do not need to consult human on what to respond.\n" @@ -289,6 +296,7 @@ async def create( id=agent_id, name=agent_id, instructions=AgentInstruction, + context_providers=[SkillsProvider.from_paths(skill_paths=str(SKILLS_DIR))], default_options={ "on_permission_request": prompt_permission, "timeout": 120, diff --git a/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md b/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md new file mode 100644 index 00000000000..b8e3a8d85b0 --- /dev/null +++ b/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md @@ -0,0 +1,53 @@ +--- +name: foundry-config-setup +description: Resolve missing setup caused by a hardcoded Foundry project endpoint or model in a sample. Use when a sample fails because it uses a placeholder/hardcoded project_endpoint (for example "https://your-project.services.ai.azure.com") or a hardcoded model instead of reading them from the environment. +license: MIT +compatibility: Works with any model that supports tool use. +metadata: + author: agent-framework-samples + version: "1.0" +--- + +## Usage + +Some samples (notably those under `01-get-started`) hardcode the Foundry +project endpoint and model directly in the `FoundryChatClient` constructor +using placeholder values, for example: + +```python +client = FoundryChatClient( + project_endpoint="https://your-project.services.ai.azure.com", + model="gpt-4o", + credential=AzureCliCredential(), +) +``` + +These placeholder values are not real and the sample cannot run as written. +The validation environment provides the real values through environment +variables, so the sample must read them from the environment instead. + +When you detect a hardcoded/placeholder `project_endpoint` (or `model`) that +is causing the sample to fail with missing or invalid setup: + +1. Replace the hardcoded `project_endpoint` value with a read from the + `FOUNDRY_PROJECT_ENDPOINT` environment variable. +2. Replace the hardcoded `model` value with a read from the `FOUNDRY_MODEL` + environment variable. +3. Ensure `import os` is present at the top of the file. + +The corrected constructor should look like: + +```python +import os + +client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ["FOUNDRY_MODEL"], + credential=AzureCliCredential(), +) +``` + +These samples are intentionally written with hardcoded placeholders, so this +is expected setup—not a defect in the sample. After applying the change, +re-run the sample and report the result as a `success` if it now runs. Do not +include a suggested `fix`. From ae13f79542c2ae8ac64683a53f24471f9c725704 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 25 Jun 2026 11:54:43 -0700 Subject: [PATCH 2/9] Include more samples and fix migration samples part 1 --- .../workflows/python-sample-validation.yml | 69 +++++++++---------- .../single_agent/01_basic_agent.py | 4 +- .../01_basic_chat_completion.py | 2 + .../02_chat_completion_with_tool.py | 2 + .../03_chat_completion_thread_and_stream.py | 2 + .../skills/hosting-sample-runner/SKILL.md | 50 ++++++++++++++ 6 files changed, 93 insertions(+), 36 deletions(-) create mode 100644 python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index bd76eb12d23..320f9c38b11 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -23,8 +23,8 @@ jobs: environment: integration env: # Required configuration for get-started samples - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} defaults: run: working-directory: python @@ -61,8 +61,8 @@ jobs: environment: integration env: # Foundry configuration - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} @@ -70,12 +70,12 @@ jobs: AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME || vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }} # OpenAI configuration - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # GitHub MCP GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # Observability ENABLE_INSTRUMENTATION: "true" defaults: @@ -122,10 +122,10 @@ jobs: runs-on: ubuntu-latest environment: integration env: - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} defaults: run: working-directory: python @@ -329,12 +329,11 @@ jobs: validate-02-agents-foundry: name: Validate 02-agents/providers/foundry - if: false # Temporarily disabled - provider folder also contains the local Foundry sample runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME || '' }} FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION || '' }} defaults: @@ -445,8 +444,8 @@ jobs: runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} defaults: run: working-directory: python @@ -479,12 +478,11 @@ jobs: validate-04-hosting: name: Validate 04-hosting - if: false # Temporarily disabled because of sample complexity runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # A2A configuration A2A_AGENT_HOST: http://localhost:5001/ defaults: @@ -518,8 +516,8 @@ jobs: runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} @@ -560,16 +558,16 @@ jobs: runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # OpenAI configuration - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} - OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} + OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} defaults: run: working-directory: python @@ -610,20 +608,21 @@ jobs: runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration for AF AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration for SK AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }} # OpenAI key - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} - OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} + OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # OpenAI configuration for SK - OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }} + OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }} # Copilot Studio COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }} COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }} diff --git a/python/samples/autogen-migration/single_agent/01_basic_agent.py b/python/samples/autogen-migration/single_agent/01_basic_agent.py index bfa0c915eae..a9bcb3da359 100644 --- a/python/samples/autogen-migration/single_agent/01_basic_agent.py +++ b/python/samples/autogen-migration/single_agent/01_basic_agent.py @@ -1,12 +1,14 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", # "autogen-agentchat", # "autogen-ext[openai]", +# "python-dotenv", # ] # /// # Run with any PEP 723 compatible runner, e.g.: -# uv run samples/autogen-migration/single_agent/01_basic_assistant_agent.py +# uv run samples/autogen-migration/single_agent/01_basic_agent.py # Copyright (c) Microsoft. All rights reserved. diff --git a/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py b/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py index ebd10122dc3..d4493b22a91 100644 --- a/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py +++ b/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py b/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py index d5b12035184..21397b62c9a 100644 --- a/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py +++ b/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py b/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py index f656220f204..b6fe966fe67 100644 --- a/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py +++ b/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md b/python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md new file mode 100644 index 00000000000..58412c6450a --- /dev/null +++ b/python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md @@ -0,0 +1,50 @@ +--- +name: hosting-sample-runner +description: Decide how to validate a hosting sample (for example anything under 04-hosting that starts a server, function host, or deployed agent). Use when a sample hosts an agent or workflow and is run by starting a local server/host and calling it, or by deploying to a cloud provider. +license: MIT +compatibility: Works with any model that supports tool use. +metadata: + author: agent-framework-samples + version: "1.0" +--- + +## Usage + +Hosting samples are different from ordinary scripts: instead of running to +completion, they typically start a server, function host, or hosted agent and +are exercised by a separate client call. Each hosting sample includes a +`README.md` that documents how to set up and run it. + +When validating a hosting sample: + +1. Read the sample's `README.md` (and any sibling READMEs in parent + directories) to understand how the sample is meant to be run. +2. Decide whether the sample can be run **locally** — that is, fully exercised + on this machine without deploying to a cloud provider (for example Azure + Functions deployment, an Azure Container App, or a Foundry hosted-agent + publish step). A sample is locally runnable when its README describes a + local launch path, such as starting a local server (e.g. Hypercorn, + `uv run python app.py`, the Functions Core Tools `func start`, or a durable + task worker) and then calling it from a local client/HTTP request. + +### If the sample can be run locally + +Follow the README's local setup and run instructions: + +1. Install any required dependencies it lists. +2. Start the host process in the background (it will not exit on its own). +3. Exercise it as the README describes — run the companion client script, + send the documented HTTP request, or otherwise drive a single end-to-end + interaction. +4. If the interaction succeeds, stop the host process and mark the sample as + `success`. +5. If the host fails to start or the interaction errors, treat it as a + `failure` and investigate the error. + +### If the sample cannot be run locally + +If the README only documents a cloud deployment path (for example deploying +to Azure Functions, publishing a Foundry hosted agent, or otherwise requiring +provisioned cloud infrastructure to exercise the sample), do not attempt to +deploy it. Mark the sample as `missing_setup` and note in the output that it +requires cloud deployment that cannot be performed locally. From 0ffeb715cb767df0d1206e4bac4b309cb4820ddd Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 25 Jun 2026 13:32:38 -0700 Subject: [PATCH 3/9] Fix migration samples --- .github/workflows/python-sample-validation.yml | 2 +- .../copilot_studio/01_basic_copilot_studio_agent.py | 11 +++++++++++ .../copilot_studio/02_copilot_studio_streaming.py | 11 +++++++++++ .../openai_responses/01_basic_responses_agent.py | 2 ++ .../openai_responses/02_responses_agent_with_tool.py | 2 ++ .../03_responses_agent_structured_output.py | 2 ++ .../orchestrations/concurrent_basic.py | 3 +++ .../orchestrations/group_chat.py | 3 +++ .../orchestrations/handoff.py | 3 +++ .../orchestrations/magentic.py | 3 +++ .../orchestrations/sequential.py | 3 +++ .../processes/fan_out_fan_in_process.py | 2 ++ .../processes/nested_process.py | 2 ++ 13 files changed, 48 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 320f9c38b11..116c1595b7d 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -9,7 +9,7 @@ env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: claude-opus-4.6 + GITHUB_COPILOT_MODEL: claude-opus-4.8 COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} permissions: diff --git a/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py b/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py index a477181b264..240a03440dd 100644 --- a/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py +++ b/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py @@ -1,11 +1,22 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-copilotstudio", +# "python-dotenv", # "semantic-kernel", # ] # /// # Run with any PEP 723 compatible runner, e.g.: # uv run samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py +# +# NOTE: The metadata above resolves the Agent Framework half only. +# The Semantic Kernel half (run_semantic_kernel) requires the older +# dot-namespace Microsoft Agents SDK (microsoft.agents.copilotstudio.client and +# microsoft.agents.core, from microsoft-agents-copilotstudio-client<0.3), while +# Agent Framework requires the newer underscore-namespace SDK +# (microsoft_agents.copilotstudio.client, from +# microsoft-agents-copilotstudio-client>=0.3.1). These two generations cannot be +# installed in the same environment, so run each half in its own isolated env. # Copyright (c) Microsoft. All rights reserved. """Call a Copilot Studio agent with SK and Agent Framework.""" diff --git a/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py b/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py index 97ef158c532..991b4b5bf1c 100644 --- a/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py +++ b/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py @@ -1,11 +1,22 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-copilotstudio", +# "python-dotenv", # "semantic-kernel", # ] # /// # Run with any PEP 723 compatible runner, e.g.: # uv run samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py +# +# NOTE: The metadata above resolves the Agent Framework half only. +# The Semantic Kernel half (run_semantic_kernel) requires the older +# dot-namespace Microsoft Agents SDK (microsoft.agents.copilotstudio.client and +# microsoft.agents.core, from microsoft-agents-copilotstudio-client<0.3), while +# Agent Framework requires the newer underscore-namespace SDK +# (microsoft_agents.copilotstudio.client, from +# microsoft-agents-copilotstudio-client>=0.3.1). These two generations cannot be +# installed in the same environment, so run each half in its own isolated env. # Copyright (c) Microsoft. All rights reserved. """Stream responses from Copilot Studio agents in SK and AF.""" diff --git a/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py b/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py index d74487d1e8f..b360da34e8f 100644 --- a/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py +++ b/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py b/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py index 01b783aff90..09b68fb7d00 100644 --- a/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py +++ b/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py b/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py index cbfbf470a08..04fad5754ce 100644 --- a/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py +++ b/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py b/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py index 11140aa8752..1a9470b7135 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py +++ b/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py @@ -1,6 +1,9 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/group_chat.py b/python/samples/semantic-kernel-migration/orchestrations/group_chat.py index 89613072d8c..8c64bcb4c13 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/group_chat.py +++ b/python/samples/semantic-kernel-migration/orchestrations/group_chat.py @@ -1,6 +1,9 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/handoff.py b/python/samples/semantic-kernel-migration/orchestrations/handoff.py index 5313c2943f8..08f77400611 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/handoff.py +++ b/python/samples/semantic-kernel-migration/orchestrations/handoff.py @@ -1,6 +1,9 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/magentic.py b/python/samples/semantic-kernel-migration/orchestrations/magentic.py index 4ce62492e21..5e03f0a3037 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/magentic.py +++ b/python/samples/semantic-kernel-migration/orchestrations/magentic.py @@ -1,6 +1,9 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/sequential.py b/python/samples/semantic-kernel-migration/orchestrations/sequential.py index 22a2be6f23e..c40792561d2 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/sequential.py +++ b/python/samples/semantic-kernel-migration/orchestrations/sequential.py @@ -1,6 +1,9 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py b/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py index 37e210e80b8..4463de36c40 100644 --- a/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py +++ b/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-core", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/processes/nested_process.py b/python/samples/semantic-kernel-migration/processes/nested_process.py index ee8d889229a..61a9aef7ee6 100644 --- a/python/samples/semantic-kernel-migration/processes/nested_process.py +++ b/python/samples/semantic-kernel-migration/processes/nested_process.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-core", +# "python-dotenv", # "semantic-kernel", # ] # /// From 02611676d625a4a823f1d78035775f58270dbd95 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 7 Jul 2026 14:06:19 -0700 Subject: [PATCH 4/9] Replace Foundry hosted agent validation skill --- .../workflows/python-sample-validation.yml | 4 + .../create_dynamic_workflow_executor.py | 8 +- python/scripts/sample_validation/models.py | 2 - ...un_dynamic_validation_workflow_executor.py | 7 +- .../skills/foundry-config-setup/SKILL.md | 3 +- .../foundry-hosted-agent-validation/SKILL.md | 309 ++++++++++++++ .../scripts/validate_hosted_agent.sh | 389 ++++++++++++++++++ .../skills/hosting-sample-runner/SKILL.md | 50 --- 8 files changed, 708 insertions(+), 64 deletions(-) create mode 100644 python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md create mode 100755 python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh delete mode 100644 python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 116c1595b7d..531d96800a8 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -483,6 +483,10 @@ jobs: env: FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + # Foundry hosted agent configuration + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.FOUNDRY_MODEL }} + FOUNDRY_PROJECT_ID: ${{ vars.FOUNDRY_PROJECT_ID }} + AZURE_CONTAINER_REGISTRY_ENDPOINT: ${{ vars.AZURE_CONTAINER_REGISTRY_ENDPOINT }} # A2A configuration A2A_AGENT_HOST: http://localhost:5001/ defaults: diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index 97b8524ec1e..3a2a4e3a318 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -68,14 +68,13 @@ class BatchCompletion: "follow its guidance to resolve the setup and then re-run the sample.\n" "Feel free to install any required dependencies if needed.\n" "The sample can be interactive. If it is interactive, respond to the sample when prompted " - "based on your analysis of the code. You do not need to consult human on what to respond.\n" - "If the sample fails, investigate the error and suggest a fix.\n" + "based on your analysis of the code. You do not need to consult human on what to respond.\n" \ + "Fail fast and do not attempt to fix the sample unless instructed by a skill.\n" "Return ONLY valid JSON with this schema:\n" "{\n" ' "status": "success|failure|missing_setup",\n' ' "output": "short summary of the result and what you did if the sample was interactive",\n' ' "error": "error details or empty string",\n' - ' "fix": "suggested code fix if the sample failed, otherwise empty string"\n' "}\n\n" ) @@ -151,7 +150,6 @@ async def handle_task( status=status_from_text(result_payload.status), output=result_payload.output, error=result_payload.error, - fix=result_payload.fix, ) break except Exception as ex: @@ -174,7 +172,6 @@ async def handle_task( status=RunStatus.FAILURE, output="", error=f"Original error: {ex}. Restart error: {restart_ex}", - fix="", ) break @@ -184,7 +181,6 @@ async def handle_task( status=RunStatus.FAILURE, output="", error=str(ex), - fix="", ) break diff --git a/python/scripts/sample_validation/models.py b/python/scripts/sample_validation/models.py index ff45b5909ba..e41de324eba 100644 --- a/python/scripts/sample_validation/models.py +++ b/python/scripts/sample_validation/models.py @@ -72,7 +72,6 @@ class RunResult: status: RunStatus output: str error: str - fix: str @dataclass @@ -154,7 +153,6 @@ def to_dict(self) -> dict[str, object]: "status": r.status.value, "output": r.output, "error": r.error, - "fix": r.fix, } for r in self.results ], diff --git a/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py b/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py index c7244cff2a2..a895e5ddc83 100644 --- a/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py +++ b/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py @@ -50,10 +50,10 @@ async def run( CoordinatorStart(samples=creation.samples), stream=True ): if event.type == "output" and isinstance(event.data, ExecutionResult): - result = event.data # type: ignore - elif event.type == WORKER_COMPLETED and isinstance( + result = event.data + elif event.type == WORKER_COMPLETED and isinstance( # type: ignore event.data, SampleInfo - ): # type: ignore + ): remaining_sample_counts -= 1 print( f"Completed validation for sample: {event.data.relative_path:<80} | " @@ -69,7 +69,6 @@ async def run( status=RunStatus.FAILURE, output="", error="Nested workflow did not return an ExecutionResult.", - fix="", ) for sample in creation.samples ] diff --git a/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md b/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md index b8e3a8d85b0..eb4e75746e0 100644 --- a/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md +++ b/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md @@ -49,5 +49,4 @@ client = FoundryChatClient( These samples are intentionally written with hardcoded placeholders, so this is expected setup—not a defect in the sample. After applying the change, -re-run the sample and report the result as a `success` if it now runs. Do not -include a suggested `fix`. +re-run the sample and report the result as a `success` if it now runs. diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md new file mode 100644 index 00000000000..ed70c965e9b --- /dev/null +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md @@ -0,0 +1,309 @@ +--- +name: foundry-hosted-agent-validation +description: > + Step-by-step process for validating a Python Foundry hosted agent sample + (under python/samples/04-hosting/foundry-hosted-agents/) end to end — running + it locally (native runtime and `azd ai agent run`) and after deploying it to + an Azure AI Foundry project with `azd`. Use this when asked to validate, smoke + test, or verify a hosted agent sample works locally and/or deployed, or when + deploying one of these samples to Foundry. +license: MIT +compatibility: Works with any model that supports tool use. +metadata: + author: agent-framework-samples + version: "1.0" +--- + +# Validating a Foundry Hosted Agent Sample + +A hosted agent sample is "validated" when it passes **three** independent +checks, plus cleanup: + +1. **Local, native runtime** — run the sample's own entry point + (`python main.py`) and invoke it over HTTP. +2. **Local, via `azd ai agent run`** — the `azd` local dev loop. +3. **Deployed** — `azd deploy` to Foundry, then invoke the hosted agent. + +Each check must succeed for **single-turn** and **multi-turn** (session / +`previous_response_id`) conversation. Always end with **cleanup** (delete the +deployed agent, remove the temp `azd` project, restore the sample dir). + +> Read the sample's own `README.md` and the parent +> `.../foundry-hosted-agents/README.md` first — they define the run/deploy +> commands and any sample-specific payload. This skill captures the process and +> the non-obvious gotchas the READMEs don't. + +--- + +## Automated script + +[`scripts/validate_hosted_agent.sh`](scripts/validate_hosted_agent.sh) runs all +three phases (and cleanup) non-interactively — use it for a full pass, and read +the phases below to interpret failures or validate a non-`responses` sample by +hand. Run `--help` for the full dependency list and options. + +```bash +# Full validation of the responses/01_basic sample (default sample dir): +python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh \ + --project-endpoint "https://.services.ai.azure.com/api/projects/" \ + --model "" \ + --acr-endpoint "" + +# Local-only (skip deploy), or point at another sample: +python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh --skip-deploy \ + --sample-dir python/samples/04-hosting/foundry-hosted-agents/responses/02_tools \ + --project-endpoint "..." --model "..." +``` + +Inputs may also come from env vars (`FOUNDRY_PROJECT_ENDPOINT`, +`AZURE_AI_MODEL_DEPLOYMENT_NAME`, `FOUNDRY_PROJECT_ID`, +`AZURE_CONTAINER_REGISTRY_ENDPOINT`). Phase flags: `--skip-native`, +`--skip-azd-local`, `--skip-deploy`; `--no-cleanup`/`--keep-agent` to inspect +afterward. The script encodes every gotcha below (model template fix, +pre-existing-agent removal, ACR reuse, port/temp cleanup). + +--- + +## Inputs you need before starting + +Gather these (ask the user if not provided): + +- **Foundry project endpoint**, e.g. + `https://.services.ai.azure.com/api/projects/`. +- **Foundry project resource id** (for non-interactive `azd ai agent init`): + `/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts//projects/`. + Find it with `az cognitiveservices account list` + the project name. +- **A real, deployed model name** in that project (e.g. `gpt-4.1-mini`). This is + often **different** from the model id in `agent.manifest.yaml` — the actual + deployment name wins. +- **An existing ACR to reuse** for deployment (login server, e.g. + `myacr.azurecr.io`). Reusing one avoids `azd provision` creating resources. +- Whether a like-named agent **already exists** in the project (remove it first + for a clean validation — see below). + +## Tooling / auth + +- `az` (logged in: `az login`) and `azd` (logged in: `azd auth login`). +- `azd` **agents extension**: `azd extension list` should show + `azure.ai.agents`; install with `azd extension install azure.ai.agents`. +- `uv` for the native-Python local run. **`python` need not be on PATH** — `uv` + and `azd ai agent run` provision their own interpreter. +- Docker is **not** required when you reuse an ACR (`remoteBuild: true` builds + in ACR Tasks). + +--- + +## Phase 0 — Understand the sample + +A responses/invocations sample folder typically contains: +`main.py` (entry point + `ResponsesHostServer`/`InvocationsHostServer`), +`agent.manifest.yaml` (used by `azd ai agent init`), `agent.yaml` (the deployed +agent definition), `requirements.txt`, `Dockerfile`, `.env.example`. + +Note the **protocol** (`responses` or `invocations`) from `agent.yaml` / +manifest — it changes the invoke command (`--protocol invocations`) and the HTTP +path (`/responses` vs the invocations route). + +--- + +## Phase 1 — Local validation, native runtime (Python) + +Run from the sample directory. + +```bash +uv venv .venv --python 3.12 # 3.12 matches the sample Dockerfile +uv pip install --python .venv/... -r requirements.txt +``` + +Create `.env` from `.env.example` with the **real** values: + +``` +FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" +AZURE_AI_MODEL_DEPLOYMENT_NAME="" +``` + +Start the server (`python main.py`) — it listens on `http://localhost:8088`. +`main.py` uses `DefaultAzureCredential`, so `az login` must be current. + +Invoke (single turn), capture the returned `response_id`, then reuse it for a +follow-up turn to confirm memory: + +```bash +curl -X POST http://localhost:8088/responses -H "Content-Type: application/json" \ + -d '{"input": "My name is Tao. Remember it."}' +# take response_id from the JSON, then: +curl -X POST http://localhost:8088/responses -H "Content-Type: application/json" \ + -d '{"input": "What is my name?", "previous_response_id": ""}' +``` + +PowerShell: use `Invoke-WebRequest -Uri http://localhost:8088/responses -Method POST -ContentType application/json -Body '...'`. + +**Pass:** HTTP 200, non-empty `output[].content[].text`, and the second turn +recalls the name. Stop the server afterward. + +--- + +## Phase 2 — Local validation via `azd ai agent run` + +### Init the azd project (once) + +Run in an **empty temp directory outside the repo** (short path avoids Windows +path-length issues, e.g. `C:\afval\`). Point `-m` at the **local** +manifest so it validates the working-tree sample: + +```bash +azd ai agent init -m /agent.manifest.yaml \ + --project-id "" \ + --model-deployment "" \ + --agent-name "" \ + --no-prompt --force +``` + +`init` downloads the template into a **subfolder named after the agent**, so the +azd project root is `//`. `cd` there for all later `azd` +commands. + +> **Before init, remove any `.venv` you created in the sample dir** — `init` +> copies the entire manifest directory into `src/`. (`.venv` is excluded from +> deploy packaging by `.agentignore`/`.dockerignore`, so it is harmless but +> bloats/slows the copy.) + +### Fix the model deployment name (critical — see Gotcha 1) + +```bash +azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME "" +``` + +### Run and invoke locally + +```bash +azd ai agent run --no-inspector # auto-creates a uv venv + installs deps; listens on :8088 +azd ai agent invoke --local --new-session "My name is Tao. Remember it." +azd ai agent invoke --local "What is my name?" # same session is reused automatically +``` + +**Pass:** both invokes return text; the second recalls the name (same +`Session:` id). Stop the `run` process afterward. + +--- + +## Removing a pre-existing agent (do this before deploying) + +`init` prints a warning if the agent name already exists in the project. To +delete it, note that **`azd ai agent delete`/`show` resolve the deployed agent +name from an azd env var, not from the positional argument.** The var is +`AGENT_{SERVICEKEY}_NAME`, where `SERVICEKEY` = the `azure.yaml` service name +uppercased with `-`/spaces → `_`. + +Example for service `agent-framework-agent-basic-responses`: + +```bash +azd env set AGENT_AGENT_FRAMEWORK_AGENT_BASIC_RESPONSES_NAME agent-framework-agent-basic-responses +azd ai agent delete --force --no-prompt --output json +# -> {"object":"agent.deleted","name":"...","deleted":true} +``` + +(After a successful `azd deploy`, this var is set automatically, so later +`show`/`delete`/`invoke` work without setting it.) + +--- + +## Phase 3 — Deploy and validate + +### Reuse an existing ACR (avoid provisioning) + +For an existing project + model, **do not run `azd provision`/`azd up`** — the +generated `azure.yaml` has a `deployments` block for the manifest's model +(often an auto-selected `GlobalProvisionedManaged` PTU SKU) that provision would +try to create (costly / quota failures). Instead reuse an ACR: + +```bash +azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT # e.g. myacr.azurecr.io +azd deploy +``` + +`azd deploy` fails with _"could not determine container registry endpoint"_ if +this is unset and no ACR is provisioned. + +### Verify the deployed model env var, then invoke + +```bash +azd ai agent show --output json # check definition.environment_variables.AZURE_AI_MODEL_DEPLOYMENT_NAME +azd ai agent invoke --new-session "My name is Tao. Remember it." +azd ai agent invoke "What is my name?" +``` + +Use `--output raw` on invoke to see raw SSE events and any failure, e.g.: + +``` +event: response.failed +... "code": "DeploymentNotFound" ... 404 ... +``` + +`DeploymentNotFound` means the deployed `AZURE_AI_MODEL_DEPLOYMENT_NAME` points +at a model that isn't deployed → fix per Gotcha 1 and redeploy (creates a new +version). + +**Pass:** agent reaches `status: active`, invoke returns text (not empty, no +`response.failed`), and multi-turn recalls the name. + +--- + +## Cleanup (always) + +- Delete the deployed agent: `azd ai agent delete --force --no-prompt`. +- Delete the temp `azd` project directory. +- Remove `.env`/`.venv` you created in the sample dir; confirm the sample dir is + pristine (`git status --porcelain ` is empty — `.env`/`.venv` are + gitignored). +- Stop any leftover local server still holding port 8088: + `Get-NetTCPConnection -LocalPort 8088 -State Listen` → `Stop-Process -Id ` + (Linux/macOS: `lsof -ti:8088 | xargs kill`). Stopping the shell may leave the + child interpreter running. + +--- + +## Gotchas (the parts that waste the most time) + +1. **The deployed model name comes from `agent.yaml`, not the azd env.** + `azd ai agent init` ignores `--model-deployment` in `--no-prompt` mode and + writes the **manifest's** model id (e.g. `gpt-4.1-mini`) as a **literal** into + both the azd env `AZURE_AI_MODEL_DEPLOYMENT_NAME` and the generated + `src//agent.yaml` env var. Local runs read the azd env (so + `azd env set` fixes them), but **deployment injects `agent.yaml`'s value**. + Fix by setting the generated `agent.yaml` env var to the template + `value: ${AZURE_AI_MODEL_DEPLOYMENT_NAME}` (what the repo sample already uses; + `init` flattens it) **and** `azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME `, + then redeploy. A literal `value: ` also works. + +2. **`azd ai agent delete/show` need the `AGENT_{SERVICEKEY}_NAME` env var** — a + bare positional agent name is treated as the _service_ name and the deployed + agent name is looked up from that env var (see "Removing a pre-existing + agent"). + +3. **`azd provision`/`azd up` will try to create the manifest's model + deployment** (from `azure.yaml`'s `deployments` block). Prefer `azd deploy` + with a reused ACR when the project + model already exist. + +4. **`python` on PATH is not required.** `uv venv` and `azd ai agent run` + provision their own interpreter and install `requirements.txt`. + +5. **`init` copies the whole manifest directory into `src/`.** Remove a local + `.venv` from the sample dir first to keep the copy clean/fast. + +6. **Port 8088 can stay bound after stopping the shell** — kill the interpreter + by PID (see Cleanup). + +--- + +## Success checklist + +- [ ] Native local run: 200 + non-empty text + multi-turn recall. +- [ ] `azd ai agent run` local: text returned + session reused across invokes. +- [ ] Pre-existing agent removed (if any). +- [ ] `azd deploy` succeeds; agent `status: active`. +- [ ] `azd ai agent show` confirms `AZURE_AI_MODEL_DEPLOYMENT_NAME` = the real + deployed model. +- [ ] Deployed invoke: text returned (no `response.failed`) + multi-turn recall. +- [ ] Cleanup done: agent deleted, temp project removed, sample dir pristine, + port 8088 free. diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh new file mode 100755 index 00000000000..23e385ffc9a --- /dev/null +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh @@ -0,0 +1,389 @@ +#!/usr/bin/env bash +# +# validate_hosted_agent.sh — automate validation of a Foundry hosted agent sample. +# +# Runs the three validation checks from the `foundry-hosted-agent-validation` +# skill and cleans up afterward: +# 1. Native local run (uv venv + `python main.py` + curl; responses protocol) +# 2. Local via azd (`azd ai agent init` -> `azd ai agent run` -> invoke) +# 3. Deployed (`azd deploy` to Foundry -> `azd ai agent invoke`) +# Each check exercises a single-turn and a multi-turn (memory) exchange. +# +# --------------------------------------------------------------------------- +# DEPENDENCIES (must be installed and on PATH in the runner / pipeline) +# --------------------------------------------------------------------------- +# bash >= 3.2 (no bash-4-only features are used) +# az Azure CLI, logged in (`az login`) — used to derive the project +# id when --project-id is omitted, and by DefaultAzureCredential. +# azd Azure Developer CLI, logged in (`azd auth login`), WITH the +# agents extension installed: +# azd extension install azure.ai.agents +# uv https://docs.astral.sh/uv/ — creates the native venv & installs +# requirements. (Note: a `python` interpreter on PATH is NOT +# required; uv and `azd ai agent run` provision their own.) +# curl native HTTP invocation of the local server. +# jq JSON parsing of responses / `azd ai agent show` output. +# coreutils awk, sed, grep, tr, printf, mktemp, sleep (standard on Linux/macOS). +# +# Optional but recommended: +# lsof OR fuser reliable freeing of the local port during cleanup. Without +# them, a child server process may linger on the port. +# Docker NOT needed when reusing an ACR (remote build runs in ACR +# Tasks). Only needed if you switch to a local docker build. +# +# --------------------------------------------------------------------------- +# REQUIRED INPUTS (flags or environment variables) +# --------------------------------------------------------------------------- +# --project-endpoint | FOUNDRY_PROJECT_ENDPOINT (native + all phases) +# --model | AZURE_AI_MODEL_DEPLOYMENT_NAME (a REAL deployed model) +# --project-id | FOUNDRY_PROJECT_ID (azd-local + deploy) +# --acr-endpoint | AZURE_CONTAINER_REGISTRY_ENDPOINT (deploy only) +# +# If --project-id is omitted it is derived from the endpoint via `az`. +# +# --------------------------------------------------------------------------- +# USAGE +# --------------------------------------------------------------------------- +# validate_hosted_agent.sh [options] +# +# --sample-dir DIR Sample folder (default: the responses/01_basic sample +# resolved relative to this script's repo). +# --project-endpoint URL Foundry project endpoint. +# --project-id ID Foundry project ARM resource id. +# --model NAME Real model deployment name in the project. +# --acr-endpoint HOST Existing ACR login server (e.g. myacr.azurecr.io). +# --agent-name NAME Override agent/service name (default: read agent.yaml). +# --port N Local port (default: 8088). +# --skip-native Skip phase 1 (native local run). +# --skip-azd-local Skip phase 2 (azd ai agent run). +# --skip-deploy Skip phase 3 (deploy to Foundry). +# --keep-agent Do NOT delete the deployed agent during cleanup. +# --no-cleanup Keep temp azd project, sample .env/.venv, and agent. +# -h | --help Show this help. +# +# EXIT CODES: 0 = all enabled checks passed; non-zero = a check or setup failed. +# +set -euo pipefail + +# --------------------------- pretty logging -------------------------------- +_c() { [ -t 1 ] && printf '%s' "$1" || printf ''; } +log() { printf '%s[validate]%s %s\n' "$(_c $'\033[1;34m')" "$(_c $'\033[0m')" "$*"; } +ok() { printf '%s[ pass ]%s %s\n' "$(_c $'\033[1;32m')" "$(_c $'\033[0m')" "$*"; } +warn() { printf '%s[ warn ]%s %s\n' "$(_c $'\033[1;33m')" "$(_c $'\033[0m')" "$*" >&2; } +die() { printf '%s[ FAIL ]%s %s\n' "$(_c $'\033[1;31m')" "$(_c $'\033[0m')" "$*" >&2; exit 1; } + +# --------------------------- defaults / args ------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# scripts -> foundry-hosted-agent-validation -> skills -> sample_validation -> scripts -> python +PYTHON_ROOT="$(cd "$SCRIPT_DIR/../../../../.." && pwd)" +DEFAULT_SAMPLE="$PYTHON_ROOT/samples/04-hosting/foundry-hosted-agents/responses/01_basic" + +SAMPLE_DIR="${DEFAULT_SAMPLE}" +PROJECT_ENDPOINT="${FOUNDRY_PROJECT_ENDPOINT:-}" +PROJECT_ID="${FOUNDRY_PROJECT_ID:-}" +MODEL="${AZURE_AI_MODEL_DEPLOYMENT_NAME:-}" +ACR_ENDPOINT="${AZURE_CONTAINER_REGISTRY_ENDPOINT:-}" +AGENT_NAME="" +PORT=8088 +DO_NATIVE=1; DO_AZD_LOCAL=1; DO_DEPLOY=1 +KEEP_AGENT=0; DO_CLEANUP=1 + +usage() { sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//; s/^#//' | sed '$d'; } + +while [ $# -gt 0 ]; do + case "$1" in + --sample-dir) SAMPLE_DIR="$2"; shift 2;; + --project-endpoint) PROJECT_ENDPOINT="$2"; shift 2;; + --project-id) PROJECT_ID="$2"; shift 2;; + --model) MODEL="$2"; shift 2;; + --acr-endpoint) ACR_ENDPOINT="$2"; shift 2;; + --agent-name) AGENT_NAME="$2"; shift 2;; + --port) PORT="$2"; shift 2;; + --skip-native) DO_NATIVE=0; shift;; + --skip-azd-local) DO_AZD_LOCAL=0; shift;; + --skip-deploy) DO_DEPLOY=0; shift;; + --keep-agent) KEEP_AGENT=1; shift;; + --no-cleanup) DO_CLEANUP=0; KEEP_AGENT=1; shift;; + -h|--help) usage; exit 0;; + *) die "unknown argument: $1 (use --help)";; + esac +done + +# --------------------------- state / cleanup ------------------------------- +WORK_DIR=""; PROJECT_ROOT=""; NATIVE_PID=""; AZD_RUN_PID="" +DEPLOYED=0; CREATED_ENV=0; CREATED_VENV=0 +PASS_COUNT=0; FAIL_COUNT=0; SUMMARY="" + +record() { # $1=pass|fail $2=text + if [ "$1" = pass ]; then PASS_COUNT=$((PASS_COUNT+1)); SUMMARY="${SUMMARY} [pass] $2"$'\n'; + else FAIL_COUNT=$((FAIL_COUNT+1)); SUMMARY="${SUMMARY} [FAIL] $2"$'\n'; fi +} + +free_port() { # $1=port + local port="$1" pids + if command -v lsof >/dev/null 2>&1; then + pids="$(lsof -ti "tcp:${port}" 2>/dev/null || true)" + [ -n "$pids" ] && kill -9 $pids 2>/dev/null || true + elif command -v fuser >/dev/null 2>&1; then + fuser -k "${port}/tcp" 2>/dev/null || true + fi +} + +stop_bg() { # $1=pid $2=port + local pid="${1:-}" port="${2:-}" + if [ -n "$pid" ]; then kill "$pid" 2>/dev/null || true; sleep 1; kill -9 "$pid" 2>/dev/null || true; fi + [ -n "$port" ] && free_port "$port" +} + +cleanup() { + local ec=$? + log "cleanup..." + stop_bg "$NATIVE_PID" "$PORT" + stop_bg "$AZD_RUN_PID" "$PORT" + if [ "$DO_CLEANUP" = 1 ]; then + if [ "$DEPLOYED" = 1 ] && [ "$KEEP_AGENT" = 0 ] && [ -n "$PROJECT_ROOT" ] && [ -d "$PROJECT_ROOT" ]; then + log "deleting deployed agent '$AGENT_NAME'" + ( cd "$PROJECT_ROOT" && azd ai agent delete "$AGENT_NAME" --force --no-prompt >/dev/null 2>&1 ) || \ + warn "could not delete deployed agent (delete it manually)" + fi + [ "$CREATED_ENV" = 1 ] && rm -f "$SAMPLE_DIR/.env" 2>/dev/null || true + [ "$CREATED_VENV" = 1 ] && rm -rf "$SAMPLE_DIR/.venv" 2>/dev/null || true + [ -n "$WORK_DIR" ] && rm -rf "$WORK_DIR" 2>/dev/null || true + else + warn "cleanup skipped (--no-cleanup). Temp project: ${WORK_DIR:-n/a}" + fi + if [ "$ec" -ne 0 ]; then printf '\n'; die "aborted (exit $ec)"; fi +} +trap cleanup EXIT INT TERM + +# --------------------------- preflight ------------------------------------- +require_cmd() { command -v "$1" >/dev/null 2>&1 || die "missing dependency '$1' — $2"; } + +log "preflight: checking dependencies" +require_cmd az "install Azure CLI: https://aka.ms/azcli" +require_cmd azd "install Azure Developer CLI: https://aka.ms/azd" +require_cmd uv "install uv: https://docs.astral.sh/uv/" +require_cmd curl "install curl" +require_cmd jq "install jq: https://jqlang.github.io/jq/" +require_cmd awk "install coreutils (awk)" +require_cmd mktemp "install coreutils (mktemp)" + +az account show >/dev/null 2>&1 || die "not logged in to Azure CLI — run: az login" +azd auth login --check-status >/dev/null 2>&1 || die "not logged in to azd — run: azd auth login" +azd extension list 2>/dev/null | grep -qi 'azure.ai.agents' || \ + die "azd agents extension missing — run: azd extension install azure.ai.agents" +command -v lsof >/dev/null 2>&1 || command -v fuser >/dev/null 2>&1 || \ + warn "neither lsof nor fuser found; port ${PORT} may not be freed cleanly on exit" + +[ -d "$SAMPLE_DIR" ] || die "sample dir not found: $SAMPLE_DIR" +[ -f "$SAMPLE_DIR/main.py" ] || die "no main.py in sample dir: $SAMPLE_DIR" +[ -f "$SAMPLE_DIR/agent.yaml" ] || die "no agent.yaml in sample dir: $SAMPLE_DIR" +[ -n "$MODEL" ] || die "model deployment name required (--model or AZURE_AI_MODEL_DEPLOYMENT_NAME)" + +# derive agent name + protocol from agent.yaml +if [ -z "$AGENT_NAME" ]; then + AGENT_NAME="$(grep -E '^name:' "$SAMPLE_DIR/agent.yaml" | head -1 | sed -E 's/^name:[[:space:]]*//' | tr -d '"'\''\r')" +fi +[ -n "$AGENT_NAME" ] || die "could not determine agent name (pass --agent-name)" +PROTOCOL="$(grep -oE 'protocol:[[:space:]]*[a-zA-Z]+' "$SAMPLE_DIR/agent.yaml" | head -1 | sed -E 's/protocol:[[:space:]]*//')" +[ -n "$PROTOCOL" ] || PROTOCOL=responses + +WORK_DIR="$(mktemp -d 2>/dev/null || mktemp -d -t afval)" +log "sample=$SAMPLE_DIR" +log "agent=$AGENT_NAME protocol=$PROTOCOL model=$MODEL port=$PORT" +log "workdir=$WORK_DIR" + +# --------------------------- helpers --------------------------------------- +wait_http() { # $1=url $2=timeout_s + local url="$1" timeout="${2:-90}" i=0 code + while :; do + code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 "$url" 2>/dev/null || echo 000)" + [ "$code" != "000" ] && return 0 + i=$((i+1)); [ "$i" -ge "$timeout" ] && return 1; sleep 1 + done +} + +# assert an `azd ai agent invoke --output raw` SSE stream succeeded. +# $1 = raw output ; $2 = optional expected substring (case-insensitive) +check_sse() { + local raw="$1" expect="${2:-}" + printf '%s' "$raw" | grep -qi 'DeploymentNotFound' && { warn "DeploymentNotFound"; return 2; } + printf '%s' "$raw" | grep -q 'response.failed' && { warn "response.failed"; return 2; } + printf '%s' "$raw" | grep -q 'response.completed' || { warn "no response.completed"; return 3; } + if [ -n "$expect" ]; then printf '%s' "$raw" | grep -qi "$expect" || { warn "missing '$expect'"; return 4; }; fi + return 0 +} + +resolve_project_id() { + [ -n "$PROJECT_ID" ] && return 0 + [ -n "$PROJECT_ENDPOINT" ] || die "need --project-id or --project-endpoint to resolve the project" + # endpoint: https://.services.ai.azure.com/api/projects/ + local acct proj + acct="$(printf '%s' "$PROJECT_ENDPOINT" | sed -E 's#https?://([^.]+)\..*#\1#')" + proj="$(printf '%s' "$PROJECT_ENDPOINT" | sed -E 's#.*/projects/([^/?]+).*#\1#')" + [ -n "$acct" ] && [ -n "$proj" ] || die "could not parse account/project from endpoint" + local acct_id + acct_id="$(az cognitiveservices account list --query "[?name=='${acct}'].id | [0]" -o tsv 2>/dev/null || true)" + [ -n "$acct_id" ] || die "could not find Foundry account '$acct' via az (check subscription/login)" + PROJECT_ID="${acct_id}/projects/${proj}" + log "resolved project-id=$PROJECT_ID" +} + +# service-name -> AGENT_{KEY}_NAME env var used by `azd ai agent delete/show` +name_key() { # $1=service name + local k; k="$(printf '%s' "$1" | tr 'a-z' 'A-Z' | tr '-' '_' | tr ' ' '_')" + printf 'AGENT_%s_NAME' "$k" +} + +INITED=0 +ensure_azd_project() { + [ "$INITED" = 1 ] && return 0 + resolve_project_id + # Remove a local .venv from the sample dir so `init` doesn't copy it into src/. + [ -d "$SAMPLE_DIR/.venv" ] && [ "$CREATED_VENV" = 1 ] && { rm -rf "$SAMPLE_DIR/.venv"; CREATED_VENV=0; } + + log "azd ai agent init (this can take a few minutes)" + ( cd "$WORK_DIR" && azd ai agent init \ + -m "$SAMPLE_DIR/agent.manifest.yaml" \ + --project-id "$PROJECT_ID" \ + --model-deployment "$MODEL" \ + --agent-name "$AGENT_NAME" \ + --no-prompt --force ) >"$WORK_DIR/init.log" 2>&1 \ + || { tail -n 40 "$WORK_DIR/init.log" >&2; die "azd ai agent init failed"; } + + PROJECT_ROOT="$WORK_DIR/$AGENT_NAME" + [ -f "$PROJECT_ROOT/azure.yaml" ] || die "azd project not found at $PROJECT_ROOT after init" + + # GOTCHA fix: init hardcodes the manifest model into the generated agent.yaml. + # Restore the template so `azd deploy` substitutes the azd env value. + local ay="$PROJECT_ROOT/src/$AGENT_NAME/agent.yaml" + if [ -f "$ay" ]; then + awk ' + seen && /value:/ { sub(/value:.*/, "value: ${AZURE_AI_MODEL_DEPLOYMENT_NAME}"); seen=0 } + /- name: AZURE_AI_MODEL_DEPLOYMENT_NAME/ { seen=1 } + { print } + ' "$ay" > "$ay.tmp" && mv "$ay.tmp" "$ay" + fi + + ( cd "$PROJECT_ROOT" && azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME "$MODEL" >/dev/null ) + INITED=1 +} + +# ========================= PHASE 1: native local =========================== +if [ "$DO_NATIVE" = 1 ]; then + [ "$PROTOCOL" = responses ] || { warn "phase 1 (native curl) supports 'responses'; sample is '$PROTOCOL' — skipping"; DO_NATIVE=0; } +fi +if [ "$DO_NATIVE" = 1 ]; then + log "=== Phase 1: native local run ===" + [ -n "$PROJECT_ENDPOINT" ] || die "native run needs --project-endpoint (FOUNDRY_PROJECT_ENDPOINT)" + + printf 'FOUNDRY_PROJECT_ENDPOINT="%s"\nAZURE_AI_MODEL_DEPLOYMENT_NAME="%s"\n' \ + "$PROJECT_ENDPOINT" "$MODEL" > "$SAMPLE_DIR/.env"; CREATED_ENV=1 + + log "creating venv + installing requirements (uv)" + uv venv "$SAMPLE_DIR/.venv" --python 3.12 >"$WORK_DIR/venv.log" 2>&1 || { cat "$WORK_DIR/venv.log" >&2; die "uv venv failed"; } + CREATED_VENV=1 + if [ -x "$SAMPLE_DIR/.venv/bin/python" ]; then VENV_PY="$SAMPLE_DIR/.venv/bin/python" + elif [ -x "$SAMPLE_DIR/.venv/Scripts/python.exe" ]; then VENV_PY="$SAMPLE_DIR/.venv/Scripts/python.exe" + else die "venv python not found under $SAMPLE_DIR/.venv"; fi + uv pip install --python "$VENV_PY" -r "$SAMPLE_DIR/requirements.txt" >"$WORK_DIR/pip.log" 2>&1 \ + || { tail -n 30 "$WORK_DIR/pip.log" >&2; die "uv pip install failed"; } + + log "starting server: python main.py (:$PORT)" + ( cd "$SAMPLE_DIR" && exec "$VENV_PY" main.py ) >"$WORK_DIR/native-server.log" 2>&1 & + NATIVE_PID=$! + wait_http "http://localhost:$PORT/responses" 90 || { tail -n 40 "$WORK_DIR/native-server.log" >&2; die "native server did not start on :$PORT"; } + + log "invoke: turn 1 (set name)" + b1="$(curl -sS -X POST "http://localhost:$PORT/responses" -H 'Content-Type: application/json' \ + -d '{"input":"My name is Tao. Please remember it."}')" + [ "$(printf '%s' "$b1" | jq -r '.status // empty')" = completed ] || die "native turn1 not completed: $b1" + rid="$(printf '%s' "$b1" | jq -r '.response_id // empty')" + [ -n "$rid" ] || die "native turn1 missing response_id" + + log "invoke: turn 2 (recall via previous_response_id)" + b2="$(curl -sS -X POST "http://localhost:$PORT/responses" -H 'Content-Type: application/json' \ + -d "$(jq -nc --arg id "$rid" '{input:"What is my name?", previous_response_id:$id}')")" + t2="$(printf '%s' "$b2" | jq -r '.output[0].content[0].text // empty')" + printf '%s' "$t2" | grep -qi 'Tao' || die "native multi-turn recall failed (got: '$t2')" + ok "native local: single + multi-turn (recall: '$t2')"; record pass "native local (python main.py)" + + stop_bg "$NATIVE_PID" "$PORT"; NATIVE_PID="" +else + log "skipping Phase 1 (native local)" +fi + +# ========================= PHASE 2: azd local ============================== +if [ "$DO_AZD_LOCAL" = 1 ]; then + log "=== Phase 2: local via 'azd ai agent run' ===" + ensure_azd_project + + log "starting: azd ai agent run --no-inspector (:$PORT)" + ( cd "$PROJECT_ROOT" && exec azd ai agent run --no-inspector --port "$PORT" ) >"$WORK_DIR/azd-run.log" 2>&1 & + AZD_RUN_PID=$! + wait_http "http://localhost:$PORT/responses" 240 || { tail -n 60 "$WORK_DIR/azd-run.log" >&2; die "azd ai agent run did not start on :$PORT"; } + + log "invoke --local: turn 1 (set name)" + r1="$( ( cd "$PROJECT_ROOT" && azd ai agent invoke --local --protocol "$PROTOCOL" --output raw --new-session \ + "My name is Tao. Please remember it." ) 2>&1 || true )" + check_sse "$r1" || die "azd local turn1 failed" + log "invoke --local: turn 2 (recall, same session)" + r2="$( ( cd "$PROJECT_ROOT" && azd ai agent invoke --local --protocol "$PROTOCOL" --output raw \ + "What is my name?" ) 2>&1 || true )" + check_sse "$r2" "Tao" || die "azd local multi-turn recall failed" + ok "azd local: single + multi-turn"; record pass "azd local (azd ai agent run)" + + stop_bg "$AZD_RUN_PID" "$PORT"; AZD_RUN_PID="" +else + log "skipping Phase 2 (azd local)" +fi + +# ========================= PHASE 3: deploy ================================= +if [ "$DO_DEPLOY" = 1 ]; then + log "=== Phase 3: deploy to Foundry ===" + [ -n "$ACR_ENDPOINT" ] || die "deploy needs --acr-endpoint (AZURE_CONTAINER_REGISTRY_ENDPOINT) to reuse an ACR" + ensure_azd_project + + ( cd "$PROJECT_ROOT" && azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT "$ACR_ENDPOINT" >/dev/null ) + + # Ensure a clean deploy: best-effort delete a pre-existing agent of this name. + log "removing any pre-existing agent named '$AGENT_NAME'" + NK="$(name_key "$AGENT_NAME")" + ( cd "$PROJECT_ROOT" && azd env set "$NK" "$AGENT_NAME" >/dev/null ) + ( cd "$PROJECT_ROOT" && azd ai agent delete "$AGENT_NAME" --force --no-prompt >/dev/null 2>&1 ) \ + && log "deleted pre-existing agent" || log "no pre-existing agent to delete" + + log "azd deploy (remote build via ACR; can take a few minutes)" + ( cd "$PROJECT_ROOT" && azd deploy ) >"$WORK_DIR/deploy.log" 2>&1 \ + || { tail -n 60 "$WORK_DIR/deploy.log" >&2; die "azd deploy failed"; } + DEPLOYED=1 + ok "azd deploy succeeded" + + log "verify deployed model env var" + shown="$( cd "$PROJECT_ROOT" && azd ai agent show "$AGENT_NAME" --output json 2>/dev/null || true )" + dep_model="$(printf '%s' "$shown" | jq -r '.definition.environment_variables.AZURE_AI_MODEL_DEPLOYMENT_NAME // empty')" + [ "$dep_model" = "$MODEL" ] || die "deployed model env var is '$dep_model', expected '$MODEL'" + status="$(printf '%s' "$shown" | jq -r '.status // empty')" + [ "$status" = active ] || warn "deployed agent status is '$status' (expected active)" + log "deployed model=$dep_model status=$status" + + log "invoke deployed: turn 1 (set name)" + d1="$( ( cd "$PROJECT_ROOT" && azd ai agent invoke "$AGENT_NAME" --protocol "$PROTOCOL" --output raw --new-session \ + "My name is Tao. Please remember it." ) 2>&1 || true )" + check_sse "$d1" || die "deployed turn1 failed" + log "invoke deployed: turn 2 (recall, same session)" + d2="$( ( cd "$PROJECT_ROOT" && azd ai agent invoke "$AGENT_NAME" --protocol "$PROTOCOL" --output raw \ + "What is my name?" ) 2>&1 || true )" + check_sse "$d2" "Tao" || die "deployed multi-turn recall failed" + ok "deployed: single + multi-turn"; record pass "deployed (azd deploy + invoke)" +else + log "skipping Phase 3 (deploy)" +fi + +# ============================= summary ===================================== +printf '\n===================== validation summary =====================\n' +printf '%s' "$SUMMARY" +printf '==============================================================\n' +printf 'passed: %d failed: %d\n' "$PASS_COUNT" "$FAIL_COUNT" +[ "$FAIL_COUNT" -eq 0 ] || exit 1 +log "all enabled checks passed" diff --git a/python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md b/python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md deleted file mode 100644 index 58412c6450a..00000000000 --- a/python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: hosting-sample-runner -description: Decide how to validate a hosting sample (for example anything under 04-hosting that starts a server, function host, or deployed agent). Use when a sample hosts an agent or workflow and is run by starting a local server/host and calling it, or by deploying to a cloud provider. -license: MIT -compatibility: Works with any model that supports tool use. -metadata: - author: agent-framework-samples - version: "1.0" ---- - -## Usage - -Hosting samples are different from ordinary scripts: instead of running to -completion, they typically start a server, function host, or hosted agent and -are exercised by a separate client call. Each hosting sample includes a -`README.md` that documents how to set up and run it. - -When validating a hosting sample: - -1. Read the sample's `README.md` (and any sibling READMEs in parent - directories) to understand how the sample is meant to be run. -2. Decide whether the sample can be run **locally** — that is, fully exercised - on this machine without deploying to a cloud provider (for example Azure - Functions deployment, an Azure Container App, or a Foundry hosted-agent - publish step). A sample is locally runnable when its README describes a - local launch path, such as starting a local server (e.g. Hypercorn, - `uv run python app.py`, the Functions Core Tools `func start`, or a durable - task worker) and then calling it from a local client/HTTP request. - -### If the sample can be run locally - -Follow the README's local setup and run instructions: - -1. Install any required dependencies it lists. -2. Start the host process in the background (it will not exit on its own). -3. Exercise it as the README describes — run the companion client script, - send the documented HTTP request, or otherwise drive a single end-to-end - interaction. -4. If the interaction succeeds, stop the host process and mark the sample as - `success`. -5. If the host fails to start or the interaction errors, treat it as a - `failure` and investigate the error. - -### If the sample cannot be run locally - -If the README only documents a cloud deployment path (for example deploying -to Azure Functions, publishing a Foundry hosted agent, or otherwise requiring -provisioned cloud infrastructure to exercise the sample), do not attempt to -deploy it. Mark the sample as `missing_setup` and note in the output that it -requires cloud deployment that cannot be performed locally. From 2958f19df34a9553110fca938e40a96010fc64bb Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 7 Jul 2026 14:09:10 -0700 Subject: [PATCH 5/9] Fix hosted agent file sample --- .../responses/06_files/main.py | 130 +++++------------- 1 file changed, 32 insertions(+), 98 deletions(-) diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/06_files/main.py b/python/samples/04-hosting/foundry-hosted-agents/responses/06_files/main.py index 43a1d241735..c47addd4a2e 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/06_files/main.py +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/06_files/main.py @@ -2,70 +2,17 @@ import asyncio import os -from collections.abc import Callable -from urllib.parse import urlsplit -import httpx -from agent_framework import Agent, MCPStreamableHTTPTool, tool +from agent_framework import Agent, tool from agent_framework.foundry import FoundryChatClient -from agent_framework_foundry_hosting import ResponsesHostServer -from azure.identity import DefaultAzureCredential, get_bearer_token_provider +from agent_framework_foundry_hosting import FoundryToolbox, ResponsesHostServer +from azure.identity import DefaultAzureCredential from dotenv import load_dotenv # Load environment variables from .env file load_dotenv() -def resolve_toolbox_endpoint() -> str: - """Resolve the toolbox MCP endpoint URL. - - Prefers the explicit ``TOOLBOX_ENDPOINT`` env var (set in ``agent.yaml`` or - ``agent.manifest.yaml`` and via ``azd env set TOOLBOX_ENDPOINT`` after the toolbox - is created); falls back to constructing the URL from ``FOUNDRY_PROJECT_ENDPOINT`` - and ``TOOLBOX_NAME``. - """ - if (endpoint := os.environ.get("TOOLBOX_ENDPOINT")) is not None: - if not endpoint: - raise ValueError("TOOLBOX_ENDPOINT is set but empty") - return endpoint - try: - project_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"].rstrip("/") - toolbox_name = os.environ["TOOLBOX_NAME"] - except KeyError as e: - raise ValueError( - "Either set TOOLBOX_ENDPOINT, or set both FOUNDRY_PROJECT_ENDPOINT " - "and TOOLBOX_NAME to build the toolbox MCP endpoint." - ) from e - return f"{project_endpoint}/toolboxes/{toolbox_name}/mcp?api-version=v1" - - -def _toolbox_name_from_endpoint(endpoint: str) -> str: - """Extract the toolbox name from a toolbox MCP endpoint URL. - - Handles both the versioned (``.../toolboxes//versions//mcp``) and - unversioned (``.../toolboxes//mcp``) endpoint shapes that Foundry - produces. Falls back to ``"toolbox"`` when the path has no ``toolboxes`` - segment. - """ - segments = urlsplit(endpoint).path.split("/") - if "toolboxes" in segments: - idx = segments.index("toolboxes") - if idx + 1 < len(segments) and segments[idx + 1]: - return segments[idx + 1] - return "toolbox" - - -class ToolboxAuth(httpx.Auth): - """Injects a fresh bearer token on every request.""" - - def __init__(self, token_provider: Callable[[], str]): - self._get_token = token_provider - - def auth_flow(self, request: httpx.Request): - request.headers["Authorization"] = f"Bearer {self._get_token()}" - yield request - - @tool(description="Get the current working directory.", approval_mode="never_require") def get_cwd() -> str: """Get the current working directory.""" @@ -97,48 +44,35 @@ def read_file(file_path: str) -> str: async def main(): credential = DefaultAzureCredential() - # Create the toolbox - token_provider = get_bearer_token_provider(credential, "https://ai.azure.com/.default") - - # Resolve the endpoint once and derive a friendly tool name from it. When - # ``TOOLBOX_NAME`` isn't set, extract the toolbox name from the URL path so - # the tool's local name matches the upstream toolbox. - toolbox_endpoint = resolve_toolbox_endpoint() - toolbox_name = os.environ.get("TOOLBOX_NAME") or _toolbox_name_from_endpoint(toolbox_endpoint) - - async with httpx.AsyncClient( - auth=ToolboxAuth(token_provider), - timeout=120.0, - ) as http_client: - toolbox = MCPStreamableHTTPTool( - name=toolbox_name, - url=toolbox_endpoint, - http_client=http_client, - load_prompts=False, - ) - - # Create the chat client - client = FoundryChatClient( - project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], - credential=credential, - ) - - agent = Agent( - client=client, - instructions=( - "You are a friendly assistant. Keep your answers brief. " - "Make sure all mathematical calculations are performed using the code interpreter " - "instead of mental arithmetic." - ), - tools=[get_cwd, list_files, read_file, toolbox], - # History will be managed by the hosting infrastructure, thus there - # is no need to store history by the service. Learn more at: - # https://developers.openai.com/api/reference/resources/responses/methods/create - default_options={"store": False}, - ) - server = ResponsesHostServer(agent) - await server.run_async() + # FoundryToolbox resolves the toolbox endpoint from the environment + # (TOOLBOX_ENDPOINT, or FOUNDRY_PROJECT_ENDPOINT + TOOLBOX_NAME), authenticates + # every request with the credential, and transparently forwards the platform + # per-request call-id to the toolbox. The hosting server enters the agent, which + # connects the toolbox on first use and closes it at shutdown. + toolbox = FoundryToolbox(credential) + + # Create the chat client + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + credential=credential, + ) + + agent = Agent( + client=client, + instructions=( + "You are a friendly assistant. Keep your answers brief. " + "Make sure all mathematical calculations are performed using the code interpreter " + "instead of mental arithmetic." + ), + tools=[get_cwd, list_files, read_file, toolbox], + # History will be managed by the hosting infrastructure, thus there + # is no need to store history by the service. Learn more at: + # https://developers.openai.com/api/reference/resources/responses/methods/create + default_options={"store": False}, + ) + server = ResponsesHostServer(agent) + await server.run_async() if __name__ == "__main__": From 6b3b89b9d1a81c7ef7375f8e39341c1477cd88c8 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 7 Jul 2026 14:50:23 -0700 Subject: [PATCH 6/9] Fix agent result format --- .../sample_validation/create_dynamic_workflow_executor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index 3a2a4e3a318..ce245b8e7df 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -41,7 +41,6 @@ class AgentResponseFormat(BaseModel): status: str output: str error: str - fix: str @dataclass From d63c00895bc2709ab57278271008a64312747d42 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 7 Jul 2026 16:58:32 -0700 Subject: [PATCH 7/9] Reorganize jobs --- .../workflows/python-sample-validation.yml | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 531d96800a8..83ec7aeac77 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -9,7 +9,7 @@ env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: claude-opus-4.8 + GITHUB_COPILOT_MODEL: auto COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} permissions: @@ -199,6 +199,7 @@ jobs: validate-02-agents-anthropic: name: Validate 02-agents/providers/anthropic + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -236,6 +237,7 @@ jobs: validate-02-agents-github-copilot: name: Validate 02-agents/providers/github_copilot + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration defaults: @@ -476,8 +478,8 @@ jobs: name: validation-report-03-workflows path: python/samples/sample_validation/reports/ - validate-04-hosting: - name: Validate 04-hosting + validate-04-hosting-foundry-hosted-agents: + name: Validate 04-hosting (foundry-hosted-agents) runs-on: ubuntu-latest environment: integration env: @@ -487,6 +489,40 @@ jobs: AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.FOUNDRY_MODEL }} FOUNDRY_PROJECT_ID: ${{ vars.FOUNDRY_PROJECT_ID }} AZURE_CONTAINER_REGISTRY_ENDPOINT: ${{ vars.AZURE_CONTAINER_REGISTRY_ENDPOINT }} + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents + + - name: Upload validation report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: validation-report-04-hosting-foundry-hosted-agents + path: python/samples/sample_validation/reports/ + + validate-04-hosting-other: + name: Validate 04-hosting (other) + if: false # Temporarily disabled - to free up Copilot quota for other jobs + runs-on: ubuntu-latest + environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # A2A configuration A2A_AGENT_HOST: http://localhost:5001/ defaults: @@ -505,13 +541,13 @@ jobs: - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 04-hosting --save-report --report-name 04-hosting + cd scripts && uv run python -m sample_validation --subdir 04-hosting --exclude foundry-hosted-agents --save-report --report-name 04-hosting-other - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() with: - name: validation-report-04-hosting + name: validation-report-04-hosting-other path: python/samples/sample_validation/reports/ validate-05-end-to-end: @@ -688,7 +724,8 @@ jobs: - validate-02-agents-copilotstudio - validate-02-agents-custom - validate-03-workflows - - validate-04-hosting + - validate-04-hosting-foundry-hosted-agents + - validate-04-hosting-other - validate-05-end-to-end - validate-autogen-migration - validate-semantic-kernel-migration From a6fbc3c23b56e2d4be8ccd93db84c2236025c601 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 9 Jul 2026 09:52:43 -0700 Subject: [PATCH 8/9] Update discovery heuristic for apps --- .../workflows/python-sample-validation.yml | 3 ++- python/scripts/sample_validation/discovery.py | 21 ++++++++++++------- python/scripts/sample_validation/models.py | 2 -- .../foundry-hosted-agent-validation/SKILL.md | 6 ++++++ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 83ec7aeac77..82d9f5f96eb 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -505,8 +505,9 @@ jobs: os: ${{ runner.os }} - name: Run sample validation + # Maximum parallel workers is set to 1 because all samples use the same port run: | - cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents + cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents --max-parallel-workers 1 - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/python/scripts/sample_validation/discovery.py b/python/scripts/sample_validation/discovery.py index c5424dd6ee9..1b613c1f5b5 100644 --- a/python/scripts/sample_validation/discovery.py +++ b/python/scripts/sample_validation/discovery.py @@ -58,7 +58,7 @@ def discover_samples( exclude: list[str] | None = None, ) -> list[SampleInfo]: """ - Find all Python sample files in the samples directory. + Find all samples in the samples directory. Args: samples_dir: Root samples directory @@ -80,7 +80,7 @@ def discover_samples( # Resolve excluded paths to absolute for reliable comparison exclude_paths = {(search_dir / exc).resolve() for exc in (exclude or [])} - python_files: list[Path] = [] + samples: list[Path] = [] # Walk through all subdirectories and find .py files for root, dirs, files in os.walk(search_dir): @@ -93,25 +93,30 @@ def discover_samples( and (Path(root) / d).resolve() not in exclude_paths ] + # If the whole directory is a sample, add the directory itself and skip its subdirectories + if any(file in ("main.py", "app.py") for file in files): + samples.append(Path(root)) + continue + for file in files: # Skip files that start with _ and include only scripts with a main entrypoint guard if file.endswith(".py") and not file.startswith("_"): file_path = Path(root) / file if _has_main_entrypoint_guard(file_path): - python_files.append(file_path) + samples.append(file_path) # Sort files for consistent execution order - python_files = sorted(python_files) + samples = sorted(samples) # Convert to SampleInfo objects - samples: list[SampleInfo] = [] - for path in python_files: + samples_info: list[SampleInfo] = [] + for path in samples: try: - samples.append(SampleInfo.from_path(path, samples_dir)) + samples_info.append(SampleInfo.from_path(path, samples_dir)) except Exception as e: print(f"Warning: Could not read {path}: {e}") - return samples + return samples_info class DiscoverSamplesExecutor(Executor): diff --git a/python/scripts/sample_validation/models.py b/python/scripts/sample_validation/models.py index e41de324eba..f5e6c40066d 100644 --- a/python/scripts/sample_validation/models.py +++ b/python/scripts/sample_validation/models.py @@ -28,7 +28,6 @@ class SampleInfo: path: Path relative_path: str - code: str @classmethod def from_path(cls, path: Path, samples_dir: Path) -> "SampleInfo": @@ -36,7 +35,6 @@ def from_path(cls, path: Path, samples_dir: Path) -> "SampleInfo": return cls( path=path, relative_path=str(path.relative_to(samples_dir)), - code=path.read_text(encoding="utf-8"), ) diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md index ed70c965e9b..8609120166e 100644 --- a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md @@ -100,6 +100,12 @@ A responses/invocations sample folder typically contains: `agent.manifest.yaml` (used by `azd ai agent init`), `agent.yaml` (the deployed agent definition), `requirements.txt`, `Dockerfile`, `.env.example`. +**The sample is the whole directory whose entry point is `main.py` — not every +`.py` file in it.** Other Python files in (or alongside) a sample folder are +**helper/companion scripts**, not standalone samples. Do **not** treat a helper +script as an individual sample — validate the sample via its `main.py` host, and +run a helper only when the sample's `README.md` calls for it as a setup. + Note the **protocol** (`responses` or `invocations`) from `agent.yaml` / manifest — it changes the invoke command (`--protocol invocations`) and the HTTP path (`/responses` vs the invocations route). From 5f9406f761f7eedda919dec70fbb5f02261cad72 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 9 Jul 2026 16:42:54 -0700 Subject: [PATCH 9/9] Split agents into even more jobs --- .../workflows/python-sample-validation.yml | 83 ++++++++++++++++++- .../foundry-hosted-agent-validation/SKILL.md | 5 +- 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 82d9f5f96eb..711f06ee13b 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -108,7 +108,7 @@ jobs: - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers --save-report --report-name 02-agents + cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers harness tools --save-report --report-name 02-agents - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -117,6 +117,85 @@ jobs: name: validation-report-02-agents path: python/samples/sample_validation/reports/ + validate-02-agents-harness: + name: Validate 02-agents/harness + runs-on: ubuntu-latest + environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + # Optional: enables the Foundry memory path in harness samples + FOUNDRY_EMBEDDING_MODEL: ${{ vars.FOUNDRY_EMBEDDING_MODEL || '' }} + FOUNDRY_MEMORY_STORE: ${{ vars.FOUNDRY_MEMORY_STORE || '' }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Create .env for samples + run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + echo "FOUNDRY_EMBEDDING_MODEL=$FOUNDRY_EMBEDDING_MODEL" >> .env + echo "FOUNDRY_MEMORY_STORE=$FOUNDRY_MEMORY_STORE" >> .env + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/harness --save-report --report-name 02-agents-harness + + - name: Upload validation report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: validation-report-02-agents-harness + path: python/samples/sample_validation/reports/ + + validate-02-agents-tools: + name: Validate 02-agents/tools + runs-on: ubuntu-latest + environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Create .env for samples + run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/tools --save-report --report-name 02-agents-tools + + - name: Upload validation report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: validation-report-02-agents-tools + path: python/samples/sample_validation/reports/ + validate-02-agents-openai: name: Validate 02-agents/providers/openai runs-on: ubuntu-latest @@ -715,6 +794,8 @@ jobs: needs: - validate-01-get-started - validate-02-agents + - validate-02-agents-harness + - validate-02-agents-tools - validate-02-agents-openai - validate-02-agents-azure - validate-02-agents-anthropic diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md index 8609120166e..e57c05cf384 100644 --- a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md @@ -4,9 +4,8 @@ description: > Step-by-step process for validating a Python Foundry hosted agent sample (under python/samples/04-hosting/foundry-hosted-agents/) end to end — running it locally (native runtime and `azd ai agent run`) and after deploying it to - an Azure AI Foundry project with `azd`. Use this when asked to validate, smoke - test, or verify a hosted agent sample works locally and/or deployed, or when - deploying one of these samples to Foundry. + an Azure AI Foundry project with `azd`. Use this when asked to validate a hosted + agent sample. license: MIT compatibility: Works with any model that supports tool use. metadata: