diff --git a/.github/workflows/loongsuite_lint_0.yml b/.github/workflows/loongsuite_lint_0.yml index 77fbf1f89..fb3276413 100644 --- a/.github/workflows/loongsuite_lint_0.yml +++ b/.github/workflows/loongsuite_lint_0.yml @@ -84,7 +84,7 @@ jobs: shell: bash env: LOONGSUITE_ALL_JOBS: >- - [{"name": "lint-loongsuite-instrumentation-agentscope", "package": "loongsuite-instrumentation-agentscope", "tox_env": "lint-loongsuite-instrumentation-agentscope", "ui_name": "loongsuite-instrumentation-agentscope"}, {"name": "lint-loongsuite-instrumentation-dashscope", "package": "loongsuite-instrumentation-dashscope", "tox_env": "lint-loongsuite-instrumentation-dashscope", "ui_name": "loongsuite-instrumentation-dashscope"}, {"name": "lint-loongsuite-instrumentation-claude-agent-sdk", "package": "loongsuite-instrumentation-claude-agent-sdk", "tox_env": "lint-loongsuite-instrumentation-claude-agent-sdk", "ui_name": "loongsuite-instrumentation-claude-agent-sdk"}, {"name": "lint-loongsuite-instrumentation-google-adk", "package": "loongsuite-instrumentation-google-adk", "tox_env": "lint-loongsuite-instrumentation-google-adk", "ui_name": "loongsuite-instrumentation-google-adk"}, {"name": "lint-loongsuite-instrumentation-google-genai", "package": "loongsuite-instrumentation-google-genai", "tox_env": "lint-loongsuite-instrumentation-google-genai", "ui_name": "loongsuite-instrumentation-google-genai"}, {"name": "lint-loongsuite-instrumentation-agno", "package": "loongsuite-instrumentation-agno", "tox_env": "lint-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno"}, {"name": "lint-loongsuite-instrumentation-langchain", "package": "loongsuite-instrumentation-langchain", "tox_env": "lint-loongsuite-instrumentation-langchain", "ui_name": "loongsuite-instrumentation-langchain"}, {"name": "lint-loongsuite-instrumentation-langgraph", "package": "loongsuite-instrumentation-langgraph", "tox_env": "lint-loongsuite-instrumentation-langgraph", "ui_name": "loongsuite-instrumentation-langgraph"}, {"name": "lint-loongsuite-instrumentation-autogen", "package": "loongsuite-instrumentation-autogen", "tox_env": "lint-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen"}, {"name": "lint-loongsuite-instrumentation-deepagents", "package": "loongsuite-instrumentation-deepagents", "tox_env": "lint-loongsuite-instrumentation-deepagents", "ui_name": "loongsuite-instrumentation-deepagents"}, {"name": "lint-loongsuite-instrumentation-microsoft-agent-framework", "package": "loongsuite-instrumentation-microsoft-agent-framework", "tox_env": "lint-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework"}, {"name": "lint-loongsuite-instrumentation-strands", "package": "loongsuite-instrumentation-strands", "tox_env": "lint-loongsuite-instrumentation-strands", "ui_name": "loongsuite-instrumentation-strands"}, {"name": "lint-loongsuite-instrumentation-qwen-agent", "package": "loongsuite-instrumentation-qwen-agent", "tox_env": "lint-loongsuite-instrumentation-qwen-agent", "ui_name": "loongsuite-instrumentation-qwen-agent"}, {"name": "lint-loongsuite-instrumentation-hermes-agent", "package": "loongsuite-instrumentation-hermes-agent", "tox_env": "lint-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent"}, {"name": "lint-loongsuite-instrumentation-mem0", "package": "loongsuite-instrumentation-mem0", "tox_env": "lint-loongsuite-instrumentation-mem0", "ui_name": "loongsuite-instrumentation-mem0"}, {"name": "lint-util-genai", "package": "util-genai", "tox_env": "lint-util-genai", "ui_name": "util-genai"}, {"name": "lint-loongsuite-instrumentation-litellm", "package": "loongsuite-instrumentation-litellm", "tox_env": "lint-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm"}, {"name": "lint-loongsuite-instrumentation-crewai", "package": "loongsuite-instrumentation-crewai", "tox_env": "lint-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai"}, {"name": "lint-loongsuite-instrumentation-a2a", "package": "loongsuite-instrumentation-a2a", "tox_env": "lint-loongsuite-instrumentation-a2a", "ui_name": "loongsuite-instrumentation-a2a"}, {"name": "lint-loongsuite-instrumentation-llama-index", "package": "loongsuite-instrumentation-llama-index", "tox_env": "lint-loongsuite-instrumentation-llama-index", "ui_name": "loongsuite-instrumentation-llama-index"}, {"name": "lint-loongsuite-instrumentation-qwenpaw", "package": "loongsuite-instrumentation-qwenpaw", "tox_env": "lint-loongsuite-instrumentation-qwenpaw", "ui_name": "loongsuite-instrumentation-qwenpaw"}, {"name": "lint-loongsuite-instrumentation-algotune", "package": "loongsuite-instrumentation-algotune", "tox_env": "lint-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune"}, {"name": "lint-loongsuite-instrumentation-bfclv4", "package": "loongsuite-instrumentation-bfclv4", "tox_env": "lint-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4"}, {"name": "lint-loongsuite-instrumentation-claw-eval", "package": "loongsuite-instrumentation-claw-eval", "tox_env": "lint-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval"}, {"name": "lint-loongsuite-instrumentation-minisweagent", "package": "loongsuite-instrumentation-minisweagent", "tox_env": "lint-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent"}, {"name": "lint-loongsuite-instrumentation-openhands", "package": "loongsuite-instrumentation-openhands", "tox_env": "lint-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands"}, {"name": "lint-loongsuite-instrumentation-slop-code", "package": "loongsuite-instrumentation-slop-code", "tox_env": "lint-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code"}, {"name": "lint-loongsuite-instrumentation-terminus2", "package": "loongsuite-instrumentation-terminus2", "tox_env": "lint-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2"}, {"name": "lint-loongsuite-instrumentation-vita", "package": "loongsuite-instrumentation-vita", "tox_env": "lint-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita"}, {"name": "lint-loongsuite-instrumentation-webarena", "package": "loongsuite-instrumentation-webarena", "tox_env": "lint-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena"}, {"name": "lint-loongsuite-instrumentation-widesearch", "package": "loongsuite-instrumentation-widesearch", "tox_env": "lint-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch"}, {"name": "lint-loongsuite-instrumentation-wildtool", "package": "loongsuite-instrumentation-wildtool", "tox_env": "lint-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool"}] + [{"name": "lint-loongsuite-instrumentation-agentscope", "package": "loongsuite-instrumentation-agentscope", "tox_env": "lint-loongsuite-instrumentation-agentscope", "ui_name": "loongsuite-instrumentation-agentscope"}, {"name": "lint-loongsuite-instrumentation-dashscope", "package": "loongsuite-instrumentation-dashscope", "tox_env": "lint-loongsuite-instrumentation-dashscope", "ui_name": "loongsuite-instrumentation-dashscope"}, {"name": "lint-loongsuite-instrumentation-claude-agent-sdk", "package": "loongsuite-instrumentation-claude-agent-sdk", "tox_env": "lint-loongsuite-instrumentation-claude-agent-sdk", "ui_name": "loongsuite-instrumentation-claude-agent-sdk"}, {"name": "lint-loongsuite-instrumentation-google-adk", "package": "loongsuite-instrumentation-google-adk", "tox_env": "lint-loongsuite-instrumentation-google-adk", "ui_name": "loongsuite-instrumentation-google-adk"}, {"name": "lint-loongsuite-instrumentation-google-genai", "package": "loongsuite-instrumentation-google-genai", "tox_env": "lint-loongsuite-instrumentation-google-genai", "ui_name": "loongsuite-instrumentation-google-genai"}, {"name": "lint-loongsuite-instrumentation-agno", "package": "loongsuite-instrumentation-agno", "tox_env": "lint-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno"}, {"name": "lint-loongsuite-instrumentation-langchain", "package": "loongsuite-instrumentation-langchain", "tox_env": "lint-loongsuite-instrumentation-langchain", "ui_name": "loongsuite-instrumentation-langchain"}, {"name": "lint-loongsuite-instrumentation-langgraph", "package": "loongsuite-instrumentation-langgraph", "tox_env": "lint-loongsuite-instrumentation-langgraph", "ui_name": "loongsuite-instrumentation-langgraph"}, {"name": "lint-loongsuite-instrumentation-autogen", "package": "loongsuite-instrumentation-autogen", "tox_env": "lint-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen"}, {"name": "lint-loongsuite-instrumentation-deepagents", "package": "loongsuite-instrumentation-deepagents", "tox_env": "lint-loongsuite-instrumentation-deepagents", "ui_name": "loongsuite-instrumentation-deepagents"}, {"name": "lint-loongsuite-instrumentation-microsoft-agent-framework", "package": "loongsuite-instrumentation-microsoft-agent-framework", "tox_env": "lint-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework"}, {"name": "lint-loongsuite-instrumentation-strands", "package": "loongsuite-instrumentation-strands", "tox_env": "lint-loongsuite-instrumentation-strands", "ui_name": "loongsuite-instrumentation-strands"}, {"name": "lint-loongsuite-instrumentation-qwen-agent", "package": "loongsuite-instrumentation-qwen-agent", "tox_env": "lint-loongsuite-instrumentation-qwen-agent", "ui_name": "loongsuite-instrumentation-qwen-agent"}, {"name": "lint-loongsuite-instrumentation-hermes-agent", "package": "loongsuite-instrumentation-hermes-agent", "tox_env": "lint-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent"}, {"name": "lint-loongsuite-instrumentation-mem0", "package": "loongsuite-instrumentation-mem0", "tox_env": "lint-loongsuite-instrumentation-mem0", "ui_name": "loongsuite-instrumentation-mem0"}, {"name": "lint-util-genai", "package": "util-genai", "tox_env": "lint-util-genai", "ui_name": "util-genai"}, {"name": "lint-loongsuite-instrumentation-litellm", "package": "loongsuite-instrumentation-litellm", "tox_env": "lint-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm"}, {"name": "lint-loongsuite-instrumentation-crewai", "package": "loongsuite-instrumentation-crewai", "tox_env": "lint-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai"}, {"name": "lint-loongsuite-instrumentation-a2a", "package": "loongsuite-instrumentation-a2a", "tox_env": "lint-loongsuite-instrumentation-a2a", "ui_name": "loongsuite-instrumentation-a2a"}, {"name": "lint-loongsuite-instrumentation-agentuniverse", "package": "loongsuite-instrumentation-agentuniverse", "tox_env": "lint-loongsuite-instrumentation-agentuniverse", "ui_name": "loongsuite-instrumentation-agentuniverse"}, {"name": "lint-loongsuite-instrumentation-llama-index", "package": "loongsuite-instrumentation-llama-index", "tox_env": "lint-loongsuite-instrumentation-llama-index", "ui_name": "loongsuite-instrumentation-llama-index"}, {"name": "lint-loongsuite-instrumentation-qwenpaw", "package": "loongsuite-instrumentation-qwenpaw", "tox_env": "lint-loongsuite-instrumentation-qwenpaw", "ui_name": "loongsuite-instrumentation-qwenpaw"}, {"name": "lint-loongsuite-instrumentation-algotune", "package": "loongsuite-instrumentation-algotune", "tox_env": "lint-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune"}, {"name": "lint-loongsuite-instrumentation-bfclv4", "package": "loongsuite-instrumentation-bfclv4", "tox_env": "lint-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4"}, {"name": "lint-loongsuite-instrumentation-claw-eval", "package": "loongsuite-instrumentation-claw-eval", "tox_env": "lint-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval"}, {"name": "lint-loongsuite-instrumentation-minisweagent", "package": "loongsuite-instrumentation-minisweagent", "tox_env": "lint-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent"}, {"name": "lint-loongsuite-instrumentation-openhands", "package": "loongsuite-instrumentation-openhands", "tox_env": "lint-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands"}, {"name": "lint-loongsuite-instrumentation-slop-code", "package": "loongsuite-instrumentation-slop-code", "tox_env": "lint-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code"}, {"name": "lint-loongsuite-instrumentation-terminus2", "package": "loongsuite-instrumentation-terminus2", "tox_env": "lint-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2"}, {"name": "lint-loongsuite-instrumentation-vita", "package": "loongsuite-instrumentation-vita", "tox_env": "lint-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita"}, {"name": "lint-loongsuite-instrumentation-webarena", "package": "loongsuite-instrumentation-webarena", "tox_env": "lint-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena"}, {"name": "lint-loongsuite-instrumentation-widesearch", "package": "loongsuite-instrumentation-widesearch", "tox_env": "lint-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch"}, {"name": "lint-loongsuite-instrumentation-wildtool", "package": "loongsuite-instrumentation-wildtool", "tox_env": "lint-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool"}] LOONGSUITE_FULL: ${{ steps.detect.outputs.full }} LOONGSUITE_PACKAGES: ${{ steps.detect.outputs.packages }} run: | diff --git a/.github/workflows/loongsuite_test_0.yml b/.github/workflows/loongsuite_test_0.yml index cc47c79e8..5c83091c5 100644 --- a/.github/workflows/loongsuite_test_0.yml +++ b/.github/workflows/loongsuite_test_0.yml @@ -84,7 +84,7 @@ jobs: shell: bash env: LOONGSUITE_ALL_JOBS: >- - [{"name": "py310-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agentscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agentscope-latest", "ui_name": "loongsuite-instrumentation-agentscope-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agentscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agentscope-latest", "ui_name": "loongsuite-instrumentation-agentscope-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-agentscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-agentscope-latest", "ui_name": "loongsuite-instrumentation-agentscope-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-deepagents-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-deepagents", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-deepagents-latest", "ui_name": "loongsuite-instrumentation-deepagents-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-deepagents-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-deepagents", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-deepagents-latest", "ui_name": "loongsuite-instrumentation-deepagents-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-deepagents-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-deepagents", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-deepagents-latest", "ui_name": "loongsuite-instrumentation-deepagents-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.13 ARC"}, {"name": "py39-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.9", "tox_env": "py39-test-util-genai", "ui_name": "util-genai 3.9 ARC"}, {"name": "py310-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.10", "tox_env": "py310-test-util-genai", "ui_name": "util-genai 3.10 ARC"}, {"name": "py311-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.11", "tox_env": "py311-test-util-genai", "ui_name": "util-genai 3.11 ARC"}, {"name": "py312-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.12", "tox_env": "py312-test-util-genai", "ui_name": "util-genai 3.12 ARC"}, {"name": "py313-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.13", "tox_env": "py313-test-util-genai", "ui_name": "util-genai 3.13 ARC"}, {"name": "py314-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.14", "tox_env": "py314-test-util-genai", "ui_name": "util-genai 3.14 ARC"}, {"name": "pypy3-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "pypy-3.9", "tox_env": "pypy3-test-util-genai", "ui_name": "util-genai pypy-3.9 ARC"}, {"name": "py311-test-detect-loongsuite-changes_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "detect-loongsuite-changes", "python_version": "3.11", "tox_env": "py311-test-detect-loongsuite-changes", "ui_name": "detect-loongsuite-changes 3.11 ARC"}, {"name": "py310-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-a2a_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-a2a", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-a2a", "ui_name": "loongsuite-instrumentation-a2a 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-a2a_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-a2a", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-a2a", "ui_name": "loongsuite-instrumentation-a2a 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-a2a_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-a2a", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-a2a", "ui_name": "loongsuite-instrumentation-a2a 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-a2a_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-a2a", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-a2a", "ui_name": "loongsuite-instrumentation-a2a 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-llama-index_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-llama-index", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-llama-index", "ui_name": "loongsuite-instrumentation-llama-index 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-llama-index_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-llama-index", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-llama-index", "ui_name": "loongsuite-instrumentation-llama-index 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-llama-index_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-llama-index", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-llama-index", "ui_name": "loongsuite-instrumentation-llama-index 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-llama-index_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-llama-index", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-llama-index", "ui_name": "loongsuite-instrumentation-llama-index 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwenpaw-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwenpaw-latest", "ui_name": "loongsuite-instrumentation-qwenpaw-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwenpaw-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwenpaw-latest", "ui_name": "loongsuite-instrumentation-qwenpaw-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwenpaw-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwenpaw-latest", "ui_name": "loongsuite-instrumentation-qwenpaw-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-widesearch_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-widesearch", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-widesearch_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-widesearch", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-widesearch_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-widesearch", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.13 ARC"}, {"name": "py39-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.9", "tox_env": "py39-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.9 ARC"}, {"name": "py310-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.10", "tox_env": "py310-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.10 ARC"}, {"name": "py311-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.11", "tox_env": "py311-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.11 ARC"}, {"name": "py312-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.12", "tox_env": "py312-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.12 ARC"}, {"name": "py313-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.13", "tox_env": "py313-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.13 ARC"}] + [{"name": "py310-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-agentscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-agentscope-oldest", "ui_name": "loongsuite-instrumentation-agentscope-oldest 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agentscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agentscope-latest", "ui_name": "loongsuite-instrumentation-agentscope-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agentscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agentscope-latest", "ui_name": "loongsuite-instrumentation-agentscope-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-agentscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-agentscope-latest", "ui_name": "loongsuite-instrumentation-agentscope-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-dashscope-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-dashscope-oldest", "ui_name": "loongsuite-instrumentation-dashscope-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-dashscope-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-dashscope", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-dashscope-latest", "ui_name": "loongsuite-instrumentation-dashscope-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-claude-agent-sdk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-claude-agent-sdk-oldest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-claude-agent-sdk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claude-agent-sdk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-claude-agent-sdk-latest", "ui_name": "loongsuite-instrumentation-claude-agent-sdk-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-adk-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-adk-oldest", "ui_name": "loongsuite-instrumentation-google-adk-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-adk-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-adk", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-adk-latest", "ui_name": "loongsuite-instrumentation-google-adk-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-genai-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-genai-oldest", "ui_name": "loongsuite-instrumentation-google-genai-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-google-genai-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-google-genai", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-google-genai-latest", "ui_name": "loongsuite-instrumentation-google-genai-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-agno_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agno", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-agno", "ui_name": "loongsuite-instrumentation-agno 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langchain-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langchain-oldest", "ui_name": "loongsuite-instrumentation-langchain-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langchain-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langchain", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langchain-latest", "ui_name": "loongsuite-instrumentation-langchain-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langgraph-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langgraph-oldest", "ui_name": "loongsuite-instrumentation-langgraph-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-langgraph-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-langgraph", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-langgraph-latest", "ui_name": "loongsuite-instrumentation-langgraph-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-autogen_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-autogen", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-autogen", "ui_name": "loongsuite-instrumentation-autogen 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-deepagents-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-deepagents", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-deepagents-latest", "ui_name": "loongsuite-instrumentation-deepagents-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-deepagents-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-deepagents", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-deepagents-latest", "ui_name": "loongsuite-instrumentation-deepagents-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-deepagents-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-deepagents", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-deepagents-latest", "ui_name": "loongsuite-instrumentation-deepagents-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-microsoft-agent-framework_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-microsoft-agent-framework", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-microsoft-agent-framework", "ui_name": "loongsuite-instrumentation-microsoft-agent-framework 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-strands-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-strands-oldest", "ui_name": "loongsuite-instrumentation-strands-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-strands-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-strands", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-strands-latest", "ui_name": "loongsuite-instrumentation-strands-latest 3.13 ARC"}, {"name": "py39-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.9 ARC"}, {"name": "py39-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.9", "tox_env": "py39-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.9 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwen-agent-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwen-agent-oldest", "ui_name": "loongsuite-instrumentation-qwen-agent-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwen-agent-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwen-agent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwen-agent-latest", "ui_name": "loongsuite-instrumentation-qwen-agent-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-hermes-agent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-hermes-agent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-hermes-agent", "ui_name": "loongsuite-instrumentation-hermes-agent 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-mem0-oldest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-mem0-oldest", "ui_name": "loongsuite-instrumentation-mem0-oldest 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-mem0-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-mem0", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-mem0-latest", "ui_name": "loongsuite-instrumentation-mem0-latest 3.13 ARC"}, {"name": "py39-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.9", "tox_env": "py39-test-util-genai", "ui_name": "util-genai 3.9 ARC"}, {"name": "py310-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.10", "tox_env": "py310-test-util-genai", "ui_name": "util-genai 3.10 ARC"}, {"name": "py311-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.11", "tox_env": "py311-test-util-genai", "ui_name": "util-genai 3.11 ARC"}, {"name": "py312-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.12", "tox_env": "py312-test-util-genai", "ui_name": "util-genai 3.12 ARC"}, {"name": "py313-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.13", "tox_env": "py313-test-util-genai", "ui_name": "util-genai 3.13 ARC"}, {"name": "py314-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "3.14", "tox_env": "py314-test-util-genai", "ui_name": "util-genai 3.14 ARC"}, {"name": "pypy3-test-util-genai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "util-genai", "python_version": "pypy-3.9", "tox_env": "pypy3-test-util-genai", "ui_name": "util-genai pypy-3.9 ARC"}, {"name": "py311-test-detect-loongsuite-changes_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "detect-loongsuite-changes", "python_version": "3.11", "tox_env": "py311-test-detect-loongsuite-changes", "ui_name": "detect-loongsuite-changes 3.11 ARC"}, {"name": "py310-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-litellm_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-litellm", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-litellm", "ui_name": "loongsuite-instrumentation-litellm 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-crewai_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-crewai", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-crewai", "ui_name": "loongsuite-instrumentation-crewai 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-a2a_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-a2a", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-a2a", "ui_name": "loongsuite-instrumentation-a2a 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-a2a_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-a2a", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-a2a", "ui_name": "loongsuite-instrumentation-a2a 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-a2a_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-a2a", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-a2a", "ui_name": "loongsuite-instrumentation-a2a 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-a2a_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-a2a", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-a2a", "ui_name": "loongsuite-instrumentation-a2a 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-agentuniverse_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentuniverse", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-agentuniverse", "ui_name": "loongsuite-instrumentation-agentuniverse 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-agentuniverse_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentuniverse", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-agentuniverse", "ui_name": "loongsuite-instrumentation-agentuniverse 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-agentuniverse_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-agentuniverse", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-agentuniverse", "ui_name": "loongsuite-instrumentation-agentuniverse 3.12 ARC"}, {"name": "py310-test-loongsuite-instrumentation-llama-index_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-llama-index", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-llama-index", "ui_name": "loongsuite-instrumentation-llama-index 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-llama-index_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-llama-index", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-llama-index", "ui_name": "loongsuite-instrumentation-llama-index 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-llama-index_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-llama-index", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-llama-index", "ui_name": "loongsuite-instrumentation-llama-index 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-llama-index_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-llama-index", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-llama-index", "ui_name": "loongsuite-instrumentation-llama-index 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwenpaw-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwenpaw-latest", "ui_name": "loongsuite-instrumentation-qwenpaw-latest 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwenpaw-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwenpaw-latest", "ui_name": "loongsuite-instrumentation-qwenpaw-latest 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwenpaw-latest_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwenpaw-latest", "ui_name": "loongsuite-instrumentation-qwenpaw-latest 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.10 ARC"}, {"name": "py310-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.11 ARC"}, {"name": "py311-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.12 ARC"}, {"name": "py312-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwenpaw-v1_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwenpaw-v1", "ui_name": "loongsuite-instrumentation-qwenpaw-v1 3.13 ARC"}, {"name": "py313-test-loongsuite-instrumentation-qwenpaw-legacy_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-qwenpaw", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-qwenpaw-legacy", "ui_name": "loongsuite-instrumentation-qwenpaw-legacy 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-algotune_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-algotune", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-algotune", "ui_name": "loongsuite-instrumentation-algotune 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-bfclv4_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-bfclv4", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-bfclv4", "ui_name": "loongsuite-instrumentation-bfclv4 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-claw-eval_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-claw-eval", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-claw-eval", "ui_name": "loongsuite-instrumentation-claw-eval 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-minisweagent_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-minisweagent", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-minisweagent", "ui_name": "loongsuite-instrumentation-minisweagent 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-openhands_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-openhands", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-openhands", "ui_name": "loongsuite-instrumentation-openhands 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-slop-code_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-slop-code", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-slop-code", "ui_name": "loongsuite-instrumentation-slop-code 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-terminus2_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-terminus2", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-terminus2", "ui_name": "loongsuite-instrumentation-terminus2 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-vita_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-vita", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-vita", "ui_name": "loongsuite-instrumentation-vita 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-webarena_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-webarena", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-webarena", "ui_name": "loongsuite-instrumentation-webarena 3.13 ARC"}, {"name": "py311-test-loongsuite-instrumentation-widesearch_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-widesearch", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-widesearch_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-widesearch", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-widesearch_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-widesearch", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-widesearch", "ui_name": "loongsuite-instrumentation-widesearch 3.13 ARC"}, {"name": "py310-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.10", "tox_env": "py310-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.10 ARC"}, {"name": "py311-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.11", "tox_env": "py311-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.11 ARC"}, {"name": "py312-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.12", "tox_env": "py312-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.12 ARC"}, {"name": "py313-test-loongsuite-instrumentation-wildtool_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-instrumentation-wildtool", "python_version": "3.13", "tox_env": "py313-test-loongsuite-instrumentation-wildtool", "ui_name": "loongsuite-instrumentation-wildtool 3.13 ARC"}, {"name": "py39-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.9", "tox_env": "py39-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.9 ARC"}, {"name": "py310-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.10", "tox_env": "py310-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.10 ARC"}, {"name": "py311-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.11", "tox_env": "py311-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.11 ARC"}, {"name": "py312-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.12", "tox_env": "py312-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.12 ARC"}, {"name": "py313-test-loongsuite-distro_loongsuite-python-arc", "os": "loongsuite-python-arc", "package": "loongsuite-distro", "python_version": "3.13", "tox_env": "py313-test-loongsuite-distro", "ui_name": "loongsuite-distro 3.13 ARC"}] LOONGSUITE_FULL: ${{ steps.detect.outputs.full }} LOONGSUITE_PACKAGES: ${{ steps.detect.outputs.packages }} run: | diff --git a/instrumentation-loongsuite/README.md b/instrumentation-loongsuite/README.md index dba0782ac..156c29582 100644 --- a/instrumentation-loongsuite/README.md +++ b/instrumentation-loongsuite/README.md @@ -2,6 +2,7 @@ | Instrumentation | Supported Packages | Metrics support | Semconv status | | --------------- | ------------------ | --------------- | -------------- | | [loongsuite-instrumentation-agentscope](./loongsuite-instrumentation-agentscope) | agentscope >= 1.0.0, < 3.0.0 | No | development +| [loongsuite-instrumentation-agentuniverse](./loongsuite-instrumentation-agentuniverse) | agentUniverse == 0.0.19.1 | Yes | development | [loongsuite-instrumentation-agno](./loongsuite-instrumentation-agno) | agno >= 2.0.0, < 3 | No | development | [loongsuite-instrumentation-algotune](./loongsuite-instrumentation-algotune) | algotune | No | development | [loongsuite-instrumentation-autogen](./loongsuite-instrumentation-autogen) | autogen-agentchat >= 0.7.0, < 0.8.0 | No | development @@ -32,4 +33,4 @@ | [loongsuite-instrumentation-vita](./loongsuite-instrumentation-vita) | vita >= 0.0.1 | No | development | [loongsuite-instrumentation-webarena](./loongsuite-instrumentation-webarena) | webarena >= 0.0.1 | No | development | [loongsuite-instrumentation-widesearch](./loongsuite-instrumentation-widesearch) | widesearch >= 0.1.0 | No | development -| [loongsuite-instrumentation-wildtool](./loongsuite-instrumentation-wildtool) | openai >= 1.0.0 | No | development \ No newline at end of file +| [loongsuite-instrumentation-wildtool](./loongsuite-instrumentation-wildtool) | openai >= 1.0.0 | No | development diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/CHANGELOG.md b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/CHANGELOG.md new file mode 100644 index 000000000..d2b0f81c9 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/CHANGELOG.md @@ -0,0 +1,44 @@ +# Changelog + +All notable changes to this project will be documented in this file. +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## Unreleased + +### Added + +- Independent Agent, LLM and Tool instrumentation using the framework's six + wrapper extension points, with shared GenAI attributes and 26 native metric + families. Existing framework wrappers are restored on uninstrumentation. +- Stable agentUniverse 0.0.19.1 API compatibility for sync/async calls and the + native LangChain ReAct executor. Optional alpha round and dispatcher hooks + are not required to enable the three core layers. +- Representative stable runtime coverage: two user turns per session, four + model decisions and three actual tools per turn, concurrent full-stream + conversations, public reasoning enabled/disabled and usage-only SSE tails. + +### Fixed + +- Install and restore wrapper/provider hooks transactionally, retaining later + hook owners and allowing a failed installation to be retried. +- Read response IDs, public reasoning, finish reasons and tool-call fragments + from the stable SDK's retained OpenAI payloads without changing business + objects. Read JSON arguments from native text ReAct tool dispatch. +- Use provider-inclusive Token totals without counting cache/reasoning subsets + twice, and aggregate each stream onto its parent exactly once. +- Keep Agent telemetry limited to its actual final answer. Preserve explicit + session IDs, message roles/history, channel provider/model and temperature zero. +- Preserve business exceptions and cancellation, chunk identity/order and + context restoration across tasks, unread close and collector failures. +- Keep the name-preserving proxy compatible with the OpenTelemetry Span API, + preserving trace/span correlation for GenAI events. +- Apply content controls to both native and GenAI content attributes. + +### Release blocker + +- The published agentUniverse 0.0.19.1 wheel pins `grpcio==1.63.0`, conflicting + with the aligned OTel exporter's `grpcio>=1.63.2` requirement. Runtime tests + with unchanged stable source and a visibly versioned metadata-only dependency + correction are diagnostic evidence. Normal installation and release remain + blocked until upstream supplies a compatible stable package. No dependency + bypass is advertised as supported installation. diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/LICENSE b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/README.md b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/README.md new file mode 100644 index 000000000..386cce213 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/README.md @@ -0,0 +1,342 @@ +# LoongSuite agentUniverse Instrumentation + +OpenTelemetry instrumentation for the +[agentUniverse](https://github.com/agentuniverse-ai/agentUniverse) multi-agent framework +(`agentUniverse`). + +This is an **independent implementation aligned with the capabilities of +agentUniverse's own instrumentation**. It does not import, delegate to or patch +the framework's instrumentors, span-attribute setters, span managers or metric +recorders. Instead it takes over the three documented wrapper extension points +of `agentuniverse.base.annotation.trace` -- the same seam the framework's own +instrumentation uses -- and produces the Agent, LLM and Tool telemetry itself: + +* `au.agent.{source}`, `au.llm.{source}` and `au.tool.{source}` spans + (`SpanKind.INTERNAL`), with the `au.*` compatibility attributes and 26 metrics + (9 agent, 9 LLM, 8 tool), streaming first-token timing, conversation-memory + recording, invocation-chain propagation, token aggregation and error handling + the framework's instrumentors emit, and +* the ARMS/LoongSuite GenAI semantic conventions (`gen_ai.*`) on those same + spans, through the shared `ExtendedTelemetryHandler` every LoongSuite + instrumentation uses. + +One agent call tree carries one span per agent, LLM request and tool call. +Stable text ReAct uses the framework's LangChain executor; its Agent, LLM and +Tool calls are recorded without requiring the alpha-only round hooks. The layer spans carry +both the `au.*` and the `gen_ai.*` namespaces. + +## Requirements + +The compatibility target is **agentUniverse 0.0.19.1**, Python 3.10-3.12. +Agent/LLM/Tool wrappers, native text ReAct, OpenAI JSON/SSE responses and public +reasoning use this stable release's actual APIs. Optional newer hooks do not +prevent stable activation. The alpha release is not required. + +**Release blocker: the published 0.0.19.1 package cannot currently be installed +with this integration's OpenTelemetry dependencies.** It requires +`grpcio==1.63.0`, while the aligned OTel gRPC exporter requires +`grpcio>=1.63.2`. The framework requires both the gRPC and HTTP exporter +distributions, so choosing HTTP does not remove the dependency conflict. +An upstream dependency correction and a new compatible stable package are +required before normal installation can be accepted. + +Runtime diagnostic tests use unchanged stable framework source with a visibly +versioned, metadata-only local dependency correction. This establishes API +behavior, not compatibility of the unmodified published wheel. Neither +`--no-deps` nor `skip_dep_check=True` is a supported installation workaround. +The framework's numpy/pyarrow requirements limit this matrix to Python 3.10-3.12. + +## Installation + +Normal installation is blocked as described above. The `instruments` extra +and runtime version check target `agentUniverse==0.0.19.1`; they do not silently +select a prerelease. Do not treat this unreleased candidate as ready to install +until the dependency blocker is resolved. + +## Usage + +```python +from opentelemetry.instrumentation.agentuniverse import ( + AgentUniverseInstrumentor, +) + +AgentUniverseInstrumentor().instrument() +``` + +One call instruments all three layers. Every agent, every `@trace_llm` method +and every tool is covered, including classes defined after `instrument()` is +called, because the decorators read the wrapper globals at call time. + +`tracer_provider` and `meter_provider` may be passed to `instrument()`; both +default to the OTel globals. `uninstrument()` removes the instrumentation +again. + +## What is instrumented + +### Agent + +| | | +| --- | --- | +| Span | `au.agent.{source}`, `SpanKind.INTERNAL` | +| Attributes | `au.span.kind=agent`, `au.agent.name`, `au.agent.input`, `au.agent.output`, `au.agent.status`, `au.agent.duration`, `au.agent.pair_id`, `au.agent.streaming`, `au.agent.first_token.duration`, `au.agent.error.type`, `au.agent.error.message`, `au.trace.caller_name`, `au.trace.caller_type`, `au.agent.usage.total_tokens/prompt_tokens/completion_tokens/detail_tokens` | +| Metrics | `agent_calls_total`, `agent_errors_total`, `agent_call_duration`, `agent_first_token_duration`, `agent_total_tokens`, `agent_prompt_tokens`, `agent_completion_tokens`, `agent_cached_tokens`, `agent_reasoning_tokens` | +| Behaviour | sync and async runs; streaming runs wrap the `output_stream` queue (`queue.Queue`, `queue.SimpleQueue` and `asyncio.Queue`) to time the first token; `ConversationMemoryModule` records the input and the result; the invocation chain carries the agent node; token usage of nested LLM calls is aggregated onto the agent span | + +### LLM + +| | | +| --- | --- | +| Span | `au.llm.{source}`, `SpanKind.INTERNAL` | +| Attributes | `au.span.kind=llm`, `au.llm.name`, `au.llm.channel_name`, `au.llm.input`, `au.llm.output`, `au.llm.llm_params`, `au.llm.streaming`, `au.llm.duration`, `au.llm.status`, `au.llm.first_token.duration`, `au.llm.error.type`, `au.llm.error.message`, `au.trace.caller_name`, `au.trace.caller_type`, `au.llm.usage.prompt_tokens/completion_tokens/total_tokens/detail_tokens` | +| Metrics | `llm_calls_total`, `llm_errors_total`, `llm_call_duration`, `llm_first_token_duration`, `llm_total_tokens`, `llm_prompt_tokens`, `llm_completion_tokens`, `llm_cached_tokens`, `llm_reasoning_tokens` | +| Behaviour | sync and async calls; a streaming call's span stays open until the returned iterator has been consumed, and is finalized exactly once (normal completion, error, or an early `close()` / `aclose()`); the real `LLMOutput.usage` is aggregated onto the parent agent; the `_llm_plugins` extension point of `agentuniverse.base.annotation.trace` is applied before the original call, so application plugins keep working | + +### Tool + +| | | +| --- | --- | +| Span | `au.tool.{source}`, `SpanKind.INTERNAL` | +| Attributes | `au.span.kind=tool`, `au.tool.name`, `au.tool.input`, `au.tool.output`, `au.tool.duration`, `au.tool.status`, `au.tool.pair_id`, `au.tool.error.type`, `au.tool.error.message`, `au.trace.caller_name`, `au.trace.caller_type`, `au.tool.usage.*` | +| Metrics | `tool_calls_total`, `tool_errors_total`, `tool_call_duration`, `tool_total_tokens`, `tool_prompt_tokens`, `tool_completion_tokens`, `tool_cached_tokens`, `tool_reasoning_tokens` | +| Behaviour | sync and async tools; `ConversationMemoryModule` records the input and the output; the invocation chain carries the tool node | + +### LoongSuite GenAI conventions + +The `gen_ai.*` namespace is written by the shared `ExtendedTelemetryHandler` on +the very same span. Because the handler normalises span names to its own +`gen_ai` form, the layer hands it a thin name-preserving proxy: the `au.*` +span name is kept, while the handler still writes its attributes, records its +metrics and ends the span. + +| Layer | `gen_ai.*` identity and request attributes | +| --- | --- | +| Agent | `gen_ai.operation.name=invoke_agent`, `gen_ai.span.kind=AGENT`, `gen_ai.agent.name`, `gen_ai.framework=agentuniverse` | +| LLM | `gen_ai.operation.name=chat`, `gen_ai.span.kind=LLM`, `gen_ai.request.model`, `gen_ai.provider.name`, `gen_ai.request.temperature` (only when the caller set one), `gen_ai.framework=agentuniverse` | +| Tool | `gen_ai.operation.name=execute_tool`, `gen_ai.span.kind=TOOL`, `gen_ai.tool.name`, `gen_ai.tool.type=function`, `gen_ai.tool.call.id`, `gen_ai.framework=agentuniverse` | + +On top of that the handler writes `gen_ai.usage.input_tokens/output_tokens/ +total_tokens` from the same numbers the `au.*.usage.*` attributes carry, +`gen_ai.response.time_to_first_token` for agent and LLM calls, the LLM's +`gen_ai.response.finish_reasons`, the tool call carriers +`gen_ai.tool.call.arguments` / `gen_ai.tool.call.result` and, with content +capture on, role-preserving `gen_ai.input.messages` / +`gen_ai.output.messages` / `gen_ai.system_instructions`. + +The handler additionally records its own client metrics on those spans, +`gen_ai.client.operation.duration` and `gen_ai.client.token.usage`, next to the +26 `au.*` metrics. The framework's own instrumentation has no equivalent of +those two. In event-enabled modes, the Span-compatible name proxy preserves +the real trace/span IDs on the shared handler's LLM and Agent events. + +## Coexistence with agentUniverse's own instrumentation + +`agentUniverse` ships its own OpenTelemetry instrumentors (`AgentInstrumentor`, +`LLMInstrumentor`, `ToolInstrumentor`), started either directly or through +`TelemetryManager.init_from_config()`. Both implementations take over the same +six module-level wrapper globals, so they must not both be active. + +This package is written for that: **LoongSuite takes priority while it is +installed, and the framework's instrumentation is restored afterwards.** + +* `instrument()` first snapshots all six extension points + (`_agent_wrapper_sync` / `_agent_wrapper_async`, `_llm_wrapper_sync` / + `_llm_wrapper_async`, `_tool_wrapper_sync` / `_tool_wrapper_async`) as they + are at that moment -- plain framework defaults, or bound methods of a native + instrumentor the application already started. +* It then replaces all six with its own wrappers and never calls the previous + wrapper, so no call can be wrapped twice and no second span or metric can be + produced. This is what keeps a three-instrumentor application at exactly one + span per layer when LoongSuite is added. +* `uninstrument()` puts the saved objects back, extension point by extension + point. If something else replaced a wrapper while LoongSuite was installed, + that new state is left untouched instead of being overwritten. +* The framework's instrumentor objects are never called and their + `BaseInstrumentor` state is never modified: LoongSuite only overrides the + module-level extension points, reversibly. A native instrumentor the + application started stays "instrumented" as far as the framework is + concerned and resumes working as soon as LoongSuite restores the wrappers. +* Installing is transactional. If any extension point cannot be replaced, the + wrappers already installed in that attempt are rolled back to the snapshot + and the application keeps using the original wrappers -- a partially + installed state, which could double-wrap a layer, never survives an install. +* Instrument and uninstrument are sentinelled, so repeating either is a no-op, + and mixing native and LoongSuite instrumentation in either order is covered + by tests. + +An application that prefers the framework's telemetry can keep using it alone; +this package adds no telemetry unless it is instrumented. + +## Content capture and privacy + +Content capture follows the shared GenAI switch, read per call from the same +configuration every LoongSuite instrumentation reads. An absent or invalid +value defaults to `NO_CONTENT`, so prompts and results are never exported +without opt-in: + +```bash +# Record message content on spans (default is NO_CONTENT): +export OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=SPAN_ONLY +``` + +| | capture off (default) | capture on (`SPAN_ONLY` / `SPAN_AND_EVENT`) | +| --- | --- | --- | +| `au.agent.input/output`, `au.llm.input/output`, `au.tool.input/output` | not written, on any layer | written | +| `gen_ai.input.messages`, `gen_ai.output.messages`, `gen_ai.system_instructions`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result` | not written | written | +| structure (identity, caller, timing, status, pairing, usage, errors) and all 26 metrics | unchanged | unchanged | + +Capture is all-or-nothing per layer: either the input and the output are both +on the span, or neither is. `EVENT_ONLY` counts as off for span content, +while GenAI LLM events follow the shared handler and the application's log exporter configuration. + +Two attribute families need extra care, so they get the same treatment: + +* **`au.llm.llm_params`** may carry prompt-like values. With capture off only + the scalar request parameters that cannot hold content (for example + `temperature`) are kept; with capture on the whole parameter mapping is + written. +* **`au.*.error.message`** is derived from the failing exception. With capture + off it is the exception type alone, and the span status description is the + same string, because an exception message or a traceback can quote the values + that flowed through the failing call. With capture on the message includes + `str(error)`. The framework's own instrumentation writes the full traceback + unconditionally. + +### A difference that is intentional + +The framework's own instrumentation writes `au.*` content carriers +unconditionally: its attribute setters have no notion of the capture switch, so +with content capture at its default `NO_CONTENT` a native-instrumented +application still exports the raw prompt and the raw result. This package +instead treats `NO_CONTENT` as a privacy guarantee for every span it produces. +This deliberate difference is covered in both the native baseline and complex runtime tests. + +## Session propagation + +The explicit `session_id` on an Agent run is recorded as `gen_ai.session.id` +on that Agent and its nested LLM and Tool spans. This context remains isolated +through the stable native ReAct executor and concurrent async conversations, +without changing global providers or propagators. + + +Session ID propagation (`au.trace.session.id`, `AUSessionPropagator`) is part of +agentUniverse's telemetry setup, not of the instrumentation: it comes from +`SessionSpanProcessor` and `AUSessionPropagator`, which +`TelemetryManager.init_from_config()` registers. This package activates only +the Agent, LLM and Tool instrumentation and never touches global propagator or +tracer provider state, so an application that needs session propagation should +use `TelemetryManager` (which can also load this instrumentation by class +path), or register the processor and the propagator itself: + +```python +from agentuniverse.base.tracing.otel.telemetry_manager import TelemetryManager + +TelemetryManager().init_from_config({ + "service_name": "my-agent-app", + "instrumentations": [ + "opentelemetry.instrumentation.agentuniverse:AgentUniverseInstrumentor", + ], +}) +``` + +That path is verified for real: a test runs `TelemetryManager.init_from_config()` +with an in-memory exporter in a separate process, sets the session id through +the framework, and asserts that the agent and LLM spans both carry +`au.trace.session.id`, that injecting into a carrier writes `AU-SessionId` and +`auSessionId`, and that extracting such a carrier restores the session id. +Running it out of process keeps the one-shot global provider and propagator out +of the pytest process -- which the main process asserts too: instrumenting +through this package leaves `trace.get_tracer_provider()` and +`propagate.get_global_textmap()` untouched. + +## Token usage + +The framework's `TokenUsage` sums disjoint text, image, audio, cache and +reasoning buckets. OpenAI-compatible provider usage instead includes cache +reads in input totals and reasoning in output totals. This instrumentation +observes the raw provider usage at conversion time and creates its own corrected +bookkeeping value; it returns the original framework objects unchanged. + +For provider usage of 20 input and 10 output tokens, with 5 cached input tokens +and 3 reasoning output tokens, every LLM span records 20 / 10 / 30 rather than +25 / 13 / 38. The same corrected totals reach Agent spans, the native metric +families and the shared GenAI metrics. Cache reads remain a subset reported in +`gen_ai.usage.cache_read.input_tokens`; reasoning remains a separate public +message part and a native usage detail/metric. Usage-only stream tails replace +cumulative snapshots instead of being added again. + +Direct user-created `TokenUsage` values without a provider payload retain their +framework bucket semantics. No provider values are guessed from response text. +The stable framework's native instrumentation counts stream usage twice onto +its parent. This integration aggregates each stream once. + +## Divergence from the framework's own instrumentation + +| Area | Framework's instrumentors | This package | +| --- | --- | --- | +| Span names, kinds, `au.*` attribute keys, metric names and labels | baseline | identical; a test compares a native run and a LoongSuite run of the same workload item by item | +| Content capture | always writes `au.*` content | honours the capture switch; `NO_CONTENT` writes no content on any layer (intentional) | +| Streamed LLM usage into the parent | counted twice in 0.0.19.1 | counted once | +| OpenAI cache/reasoning subsets | may be added to inclusive totals twice | corrected from the raw provider totals without mutating business results | +| `au.*.error.message` | full traceback | `str(error)` when capture is on, the exception type when it is off | +| `au.llm.llm_params` | source helper may replace configured temperature with -1 | actual call/channel temperature, including zero; safe scalar parameters when capture is off | +| Session processor and propagator | registered by `TelemetryManager` | globals unchanged; explicit Agent session ID is also mapped to GenAI attributes | + +## Fail-safe telemetry + +Instrumentation never changes agent behaviour. Every attribute write, metric +record, memory call, invocation-chain push and content serialization is +guarded: a failure inside telemetry is logged and swallowed, the instrumented +call still runs, and its own exception -- including `asyncio.CancelledError` -- +is re-raised unchanged after the span has been closed with an error status. A +layer that cannot even be constructed calls the original function directly. + +## Modules + +| Module | Responsibility | +| --- | --- | +| `__init__.py` | `AgentUniverseInstrumentor`: snapshot, transactional takeover of the six extension points, rollback, restore, tracer/meter/handler assembly | +| `_common.py` | shared span lifecycle, serialization, privacy, GenAI identity, token bookkeeping, metrics and fail-safe helpers | +| `_agent.py` | the agent layer: span, attributes, nine metrics, streaming first token, memory, aggregation | +| `_llm.py` | the LLM layer: span, attributes, nine metrics, deferred stream finalization, plugin hook, usage | +| `_step.py` | native ReAct round spans | +| `_hooks.py` | reversible provider conversion and native tool dispatcher hooks | +| `_usage.py` | provider totals and call-local usage capture | +| `_tool.py` | the tool layer: span, attributes, eight metrics, memory | + +## Tests + +The suite runs against the declared real framework version. Fast wrapper, +coexistence, privacy, error and lifecycle checks are supplemented by +`tests/runtime_app.py` and `tests/stable_runtime_provider.py`: + +- One native sync and two concurrent streaming conversations, each with two + user turns and four model decisions per turn. +- The stable native ReAct/AgentExecutor dispatches stock, price and + multiplication tools, then produces the final answer. Arguments, actual + results and second-turn history are verified in real SDK traffic and spans. + Text ReAct does not supply provider tool-call IDs or parallel calls within + one model round; telemetry does not invent either capability. +- Nonempty synthetic public reasoning with `reasoning_effort=high`, reasoning + disabled, fragmented text/reasoning and usage-only SSE tails. Only the + provider HTTP boundary is deterministic; SDK parsing, framework loops, + tool dispatch and telemetry are real. +- Separate real stable-channel SDK calls verify raw tool-call JSON and + fragmented tool arguments in SSE. This checks model-output mapping, without + claiming that native text ReAct dispatch supplies provider tool-call IDs. +- `SPAN_ONLY` and `NO_CONTENT`, correct inclusive Token totals and metrics, + original business result identity, isolated sessions and parent trees. +- Atomic installation failure/retry, later hook ownership, unread stream close, + cross-task consumption, cancellation, collector faults and clean siblings. + +Run from the repository root: + +```bash +tox -c tox-loongsuite.ini -e py312-test-loongsuite-instrumentation-agentuniverse +``` + +The test matrix uses the exporters from the same pinned OpenTelemetry core +revision as the SDK. Published exporters constrain released SDK versions and +cannot be mixed with that development SDK. The normal tox installation remains +blocked on the official framework dependency conflict; diagnostic runs are +reported separately. Six alpha-API checks are inapplicable to the stable +release and do not count as executed stable feature coverage. diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/pyproject.toml new file mode 100644 index 000000000..2f6aab659 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/pyproject.toml @@ -0,0 +1,58 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "loongsuite-instrumentation-agentuniverse" +dynamic = ["version"] +description = "LoongSuite agentUniverse Instrumentation" +supports_metrics = true +readme = "README.md" +license = "Apache-2.0" +# The stable agentUniverse release pins numpy<2 and pyarrow<17, with +# no cp313 wheels, so the ``instruments`` extra cannot be installed on 3.13. +# Keep this in step with the tox matrix in tox-loongsuite.ini. +requires-python = ">=3.10,<3.13" +authors = [ + { name = "OpenTelemetry Authors", email = "cncf-opentelemetry-contributors@lists.cncf.io" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "opentelemetry-api >= 1.37.0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", + "opentelemetry-util-genai", +] + +[project.optional-dependencies] +instruments = [ + "agentUniverse == 0.0.19.1", +] + +[project.entry-points.opentelemetry_instrumentor] +agentuniverse = "opentelemetry.instrumentation.agentuniverse:AgentUniverseInstrumentor" + +[project.urls] +Homepage = "https://github.com/alibaba/loongsuite-python/tree/main/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse" +Repository = "https://github.com/alibaba/loongsuite-python" + +[tool.hatch.version] +path = "src/opentelemetry/instrumentation/agentuniverse/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/opentelemetry"] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/__init__.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/__init__.py new file mode 100644 index 000000000..64992552a --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/__init__.py @@ -0,0 +1,312 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OpenTelemetry instrumentation for agentUniverse (LoongSuite). + +This package instruments the agentUniverse Agent, LLM and Tool layers by +claiming the six module-level wrapper extension points of +``agentuniverse.base.annotation.trace``. The ``@trace_agent``, ``@trace_llm`` +and ``@trace_tool`` decorators resolve those globals on every call, so replacing +them intercepts every call without touching the decorated functions. + +It is an independent implementation: it never imports, calls, patches or +delegates to agentUniverse's own OTel instrumentors (``AgentInstrumentor``, +``LLMInstrumentor``, ``ToolInstrumentor``, their attribute setters, span and +metric managers or queue wrappers). Everything an instrumented call needs -- +spans, the ``au.*`` compatibility attributes, metrics, token aggregation, +streaming finalization and error recording -- is produced here, together with +the LoongSuite ``gen_ai.*`` semantic conventions written onto the same span by +the shared ``ExtendedTelemetryHandler``. + +Instrumentation is transactional: the six extension points are snapshotted +first, replaced only after every layer is ready, rolled back as a whole if any +step fails, and restored to their original objects on uninstrument. If +agentUniverse's own instrumentors were active before this package was +instrumented, their wrappers are suppressed (never called) while LoongSuite owns +the extension points, and are restored verbatim afterwards. +""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, Optional, Sequence + +from agentuniverse.base.annotation import trace as trace_module + +from opentelemetry import metrics, trace +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.util.genai.extended_handler import ExtendedTelemetryHandler + +from ._agent import AgentLayer +from ._common import ( + INSTRUMENTOR_NAME, + LAYER_WRAPPER_GLOBALS, + WRAPPER_GLOBAL_NAMES, + describe_wrapper_owner, +) +from ._hooks import install_auxiliary_hooks, restore_auxiliary_hooks +from ._llm import LLMLayer +from ._step import ( + ReactStepLayer, + install_react_step_hooks, + remove_react_step_hooks, +) +from ._tool import ToolLayer +from .version import __version__ + +logger = logging.getLogger(__name__) + +__all__ = ["AgentUniverseInstrumentor"] + + +class _InstallState: + """Which instrumentor instance currently owns the extension points.""" + + owner: Optional["AgentUniverseInstrumentor"] = None + + +_install_state = _InstallState() + + +class AgentUniverseInstrumentor(BaseInstrumentor): + """LoongSuite instrumentation for the agentUniverse Agent, LLM and Tool.""" + + _snapshot: Dict[str, Any] = {} + _wrappers: Dict[str, Any] = {} + _hooks: Dict[str, Any] = {} + _auxiliary_hooks: list = [] + _owns_install = False + #: Set when ``_instrument`` rolled back, so ``instrument`` can undo the + #: instrumented mark ``BaseInstrumentor`` applies after it returns. + _install_failed = False + + def instrumentation_dependencies(self) -> Sequence[str]: + return ("agentUniverse >= 0.0.19.1",) + + # -- install ------------------------------------------------------ + + def instrument(self, **kwargs: Any) -> None: + """Instrument the framework, keeping a rolled-back install retryable. + + ``BaseInstrumentor.instrument`` marks the instrumentor as instrumented + as soon as ``_instrument`` returns, including when it returned because + it had rolled the install back again. Clearing that mark keeps the + bookkeeping honest: nothing of ours is installed, and a later attempt + can run. + """ + super().instrument(**kwargs) + if self._install_failed: + self._install_failed = False + self._is_instrumented_by_opentelemetry = False + + def _instrument(self, **kwargs: Any) -> None: + owner = self._installed_owner() + if owner is not None: + logger.warning( + "The agentUniverse wrapper extension points are already owned " + "by %s; skipping this install so calls are not wrapped twice", + owner, + ) + self._install_failed = True + return + + snapshot = { + name: getattr(trace_module, name, None) + for name in WRAPPER_GLOBAL_NAMES + } + self._log_suppressed_wrappers(snapshot) + + tracer_provider = kwargs.get("tracer_provider") + meter_provider = kwargs.get("meter_provider") + + try: + tracer = trace.get_tracer( + INSTRUMENTOR_NAME, __version__, tracer_provider + ) + meter = metrics.get_meter( + INSTRUMENTOR_NAME, __version__, meter_provider + ) + handler = ExtendedTelemetryHandler( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + ) + step_layer = ReactStepLayer(handler) + agent_layer = AgentLayer(tracer, handler, meter, step_layer) + llm_layer = LLMLayer(tracer, handler, meter) + tool_layer = ToolLayer(tracer, handler, meter) + wrappers: Dict[str, Any] = { + "_agent_wrapper_sync": agent_layer.wrap_sync, + "_agent_wrapper_async": agent_layer.wrap_async, + "_llm_wrapper_sync": llm_layer.wrap_sync, + "_llm_wrapper_async": llm_layer.wrap_async, + "_tool_wrapper_sync": tool_layer.wrap_sync, + "_tool_wrapper_async": tool_layer.wrap_async, + } + except Exception: + logger.warning( + "Could not build the LoongSuite agentUniverse instrumentation; " + "the application keeps using the original wrappers", + exc_info=True, + ) + self._install_failed = True + return + + installed: list[str] = [] + hooks = {} + auxiliary_hooks = [] + try: + for name, wrapper in wrappers.items(): + installed.append(name) + self._install_wrapper(name, wrapper) + hooks = install_react_step_hooks(step_layer) + auxiliary_hooks = install_auxiliary_hooks() + except Exception: + restore_auxiliary_hooks(auxiliary_hooks) + remove_react_step_hooks(hooks) + self._rollback(snapshot, wrappers, installed) + logger.warning( + "Installing the LoongSuite agentUniverse wrappers failed; " + "rolled back to the original wrappers", + exc_info=True, + ) + self._install_failed = True + return + + self._snapshot = snapshot + self._wrappers = wrappers + self._hooks = hooks + self._auxiliary_hooks = auxiliary_hooks + self._owns_install = True + _install_state.owner = self + + def _install_wrapper(self, name: str, wrapper: Any) -> None: + """Replace one extension point (a seam, so install can be tested).""" + setattr(trace_module, name, wrapper) + + def _rollback( + self, + snapshot: Dict[str, Any], + wrappers: Dict[str, Any], + installed: list[str], + ) -> None: + """Put the snapshotted wrappers back after a failed install. + + Only extension points this install still holds are restored; anything + another instrumentation has taken over in the meantime is left alone. + """ + for name in installed: + if getattr(trace_module, name, None) is not wrappers.get(name): + continue + try: + setattr(trace_module, name, snapshot[name]) + except Exception: # pragma: no cover - defensive + logger.warning( + "Could not restore the agentUniverse wrapper %s", + name, + exc_info=True, + ) + + def _log_suppressed_wrappers(self, snapshot: Dict[str, Any]) -> None: + """Report third-party wrappers that this install takes over.""" + for name in WRAPPER_GLOBAL_NAMES: + owner = describe_wrapper_owner(snapshot.get(name)) + if owner is None: + continue + logger.info( + "Suppressing the pre-existing %s wrapper for %s; it will be " + "restored when this instrumentation is removed", + owner, + name, + ) + + @staticmethod + def _installed_owner() -> Optional["AgentUniverseInstrumentor"]: + """The instance that currently owns the extension points, if any. + + The sentinel can go stale: an application (or a test harness) may put the + wrappers back without going through this class. It therefore only counts + while the extension points still hold that instance's wrappers. + """ + owner = _install_state.owner + if owner is None: + return None + if getattr( + trace_module, "_agent_wrapper_sync", None + ) is owner._wrappers.get("_agent_wrapper_sync"): + return owner + _install_state.owner = None + owner._owns_install = False + return None + + # -- uninstall ---------------------------------------------------- + + def _uninstrument(self, **kwargs: Any) -> None: + if not self._owns_install: + logger.debug( + "Nothing to remove: this instance does not own the " + "agentUniverse wrapper extension points" + ) + return + + failures = [] + replaced = [] + for name, original in self._snapshot.items(): + if getattr(trace_module, name, None) is not self._wrappers.get( + name + ): + replaced.append(name) + continue + try: + setattr(trace_module, name, original) + except Exception: # pragma: no cover - defensive + failures.append(name) + if replaced: + logger.warning( + "Left %s untouched: another instrumentation replaced the " + "LoongSuite wrappers after they were installed", + ", ".join(replaced), + ) + if failures: + logger.warning( + "Could not restore the agentUniverse wrapper extension " + "points: %s", + ", ".join(failures), + ) + + if _install_state.owner is self: + _install_state.owner = None + self._owns_install = False + self._snapshot = {} + self._wrappers = {} + auxiliary_hooks, self._auxiliary_hooks = self._auxiliary_hooks, [] + restore_auxiliary_hooks(auxiliary_hooks) + hooks, self._hooks = self._hooks, {} + if hooks: + remove_react_step_hooks(hooks) + + # -- introspection (used by tests and diagnostics) ----------------- + + @property + def wrapper_snapshot(self) -> Dict[str, Any]: + """The wrapper objects captured before this install.""" + return dict(self._snapshot) + + @property + def installed_wrappers(self) -> Dict[str, Any]: + """The wrapper objects this install put in place.""" + return dict(self._wrappers) + + @property + def layer_globals(self) -> tuple[tuple[str, str], ...]: + return LAYER_WRAPPER_GLOBALS diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_agent.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_agent.py new file mode 100644 index 000000000..7415d1c44 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_agent.py @@ -0,0 +1,535 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The agent layer of the LoongSuite agentUniverse instrumentation. + +This module claims the ``_agent_wrapper_sync``/``_agent_wrapper_async`` +extension points the ``@trace_agent`` decorator resolves, and builds the +``au.agent.*`` span, metrics and token bookkeeping itself. The ``gen_ai.*`` +attributes on that same span come from the shared ``ExtendedTelemetryHandler``, +which finalizes the invocation this layer starts. + +No agentUniverse OTel class is imported or called here. +""" + +from __future__ import annotations + +import asyncio +import logging +import queue +from typing import Any, Dict, Optional + +from agentuniverse.agent.memory.conversation_memory.conversation_memory_module import ( # noqa: E501 + ConversationMemoryModule, +) +from agentuniverse.base.annotation.trace import _get_agent_info +from agentuniverse.llm.llm_output import TokenUsage + +from opentelemetry.metrics import Meter +from opentelemetry.trace import SpanKind +from opentelemetry.util.genai.extended_handler import ExtendedTelemetryHandler +from opentelemetry.util.genai.extended_types import InvokeAgentInvocation +from opentelemetry.util.genai.types import Error + +from ._common import ( + FRAMEWORK_NAME, + SUCCESS_STATUS, + AgentAttributes, + LayerSpan, + MetricLabels, + NamePreservingSpan, + agent_name_attributes, + build_safely, + caller_labels, + detach_context_token, + error_status_text, + final_answer_of, + input_messages_for, + maybe_content, + monotonic_now, + output_messages_for, + pop_current_agent_name, + pop_invocation_chain, + promote_token_usage_to_parent, + push_current_agent_name, + push_invocation_chain, + run_safely, + set_caller_attributes, + set_error_status, + set_layer_error_attributes, + set_usage_attributes, + token_usage_for, +) +from ._step import ( + ReactStepLayer, + current_react_step_tracker, + pop_react_step_tracker, + push_react_step_tracker, +) + +logger = logging.getLogger(__name__) + +SPAN_KIND_VALUE = "agent" + + +def _memory_module() -> Any: + """The framework's conversation memory, or ``None`` when unavailable. + + The framework's own instrumentation builds this module unguarded; this + package never lets a telemetry side effect break a working agent call, so a + module that cannot be built (or a memory call that fails) is only logged. + """ + try: + return ConversationMemoryModule() + except Exception: # pragma: no cover - application config not ready + logger.debug("ConversationMemoryModule unavailable", exc_info=True) + return None + + +class AgentMetrics: + """The agentUniverse agent metric set, recorded by this package.""" + + def __init__(self, meter: Meter) -> None: + self._calls = meter.create_counter( + name="agent_calls_total", + description="Total number of Agent calls", + unit="1", + ) + self._errors = meter.create_counter( + name="agent_errors_total", + description="Total number of Agent errors", + unit="1", + ) + self._duration = meter.create_histogram( + name="agent_call_duration", + description="Duration of Agent calls in seconds", + unit="s", + ) + self._first_token = meter.create_histogram( + name="agent_first_token_duration", + description="Duration of Agent first token in seconds", + unit="s", + ) + self._total_tokens = meter.create_histogram( + name="agent_total_tokens", + description="Total token nums used in agent", + unit="1", + ) + self._prompt_tokens = meter.create_histogram( + name="agent_prompt_tokens", + description="Prompt token nums used in agent", + unit="1", + ) + self._completion_tokens = meter.create_histogram( + name="agent_completion_tokens", + description="Completion token nums used in agent", + unit="1", + ) + self._cached_tokens = meter.create_histogram( + name="agent_cached_tokens", + description="Cached token nums used in agent", + unit="1", + ) + self._reasoning_tokens = meter.create_histogram( + name="agent_reasoning_tokens", + description="Reasoning token nums used in agent", + unit="1", + ) + + def record_call(self, labels: Dict[str, Any]) -> None: + # Older SDKs retain the attribute mapping rather than copying it. + self._calls.add(1, dict(labels)) + + def record_duration(self, duration: float, labels: Dict[str, Any]) -> None: + self._duration.record(duration, labels) + + def record_first_token( + self, duration: float, labels: Dict[str, Any] + ) -> None: + self._first_token.record(duration, labels) + + def record_error( + self, error: BaseException, duration: float, labels: Dict[str, Any] + ) -> None: + error_labels = { + **labels, + MetricLabels.AGENT_STATUS: type(error).__name__, + } + self._errors.add(1, error_labels) + self._duration.record(duration, error_labels) + + def record_tokens(self, usage: TokenUsage, labels: Dict[str, Any]) -> None: + self._total_tokens.record(usage.total_tokens, labels) + self._completion_tokens.record(usage.completion_tokens, labels) + self._prompt_tokens.record(usage.prompt_tokens, labels) + self._cached_tokens.record(usage.cached_tokens, labels) + self._reasoning_tokens.record(usage.reasoning_tokens, labels) + + +class FirstTokenStream: + """Proxy for the agent's ``output_stream`` queue. + + The agent writes its first token straight onto the caller's queue, so the + proxy is what turns that write into the first-token timing. Everything else + is delegated to the wrapped queue. + """ + + def __init__(self, wrapped: Any, on_first_item: Any) -> None: + self._wrapped = wrapped + self._on_first_item = on_first_item + self._fired = False + + def _fire(self) -> None: + if self._fired: + return + self._fired = True + run_safely(self._on_first_item) + + def put(self, item: Any, block: bool = True, timeout: Any = None) -> Any: + self._fire() + return self._wrapped.put(item, block, timeout) + + def put_nowait(self, item: Any) -> Any: + self._fire() + return self._wrapped.put_nowait(item) + + def __getattr__(self, name: str) -> Any: + return getattr(self._wrapped, name) + + +class AsyncFirstTokenStream(FirstTokenStream): + """``FirstTokenStream`` for an ``asyncio.Queue``.""" + + async def put(self, item: Any) -> Any: # type: ignore[override] + self._fire() + return await self._wrapped.put(item) + + def put_nowait(self, item: Any) -> Any: # type: ignore[override] + self._fire() + return self._wrapped.put_nowait(item) + + +def wrap_first_token_stream(output_stream: Any, on_first_item: Any) -> Any: + """Wrap a recognised stream queue, leaving anything else untouched.""" + if isinstance(output_stream, asyncio.Queue): + return AsyncFirstTokenStream(output_stream, on_first_item) + if isinstance(output_stream, (queue.Queue, queue.SimpleQueue)): + return FirstTokenStream(output_stream, on_first_item) + return output_stream + + +class _AgentCall: + """Everything one wrapped agent call needs, sync or async.""" + + def __init__( + self, + layer: "AgentLayer", + func: Any, + args: tuple, + kwargs: Dict[str, Any], + ) -> None: + self._layer = layer + ( + self.agent_instance, + self.source, + self.agent_input, + self.caller_info, + self.pair_id, + ) = _get_agent_info(func, *args, **kwargs) + self.span_name = f"au.agent.{self.source}" + self.labels: Dict[str, Any] = { + MetricLabels.AGENT_NAME: self.source, + **caller_labels(self.caller_info), + MetricLabels.AGENT_STATUS: SUCCESS_STATUS, + } + self.layer_span: Optional[LayerSpan] = None + session_id = kwargs.get("session_id") + self.session_id = ( + session_id + if isinstance(session_id, str) and session_id + else agent_name_attributes().get("gen_ai.session.id") + ) + self.invocation = InvokeAgentInvocation( + provider=FRAMEWORK_NAME, + agent_name=self.source, + attributes={"gen_ai.session.id": self.session_id} + if self.session_id + else {}, + ) + self.streaming = False + self.finished = False + #: Only a node this call actually added may be popped again. + self.invocation_chain_pushed = False + #: The ReAct step tracker of this call, pushed while it runs. + self.react_step_token = None + + # -- lifecycle ---------------------------------------------------- + + def start(self, kwargs: Dict[str, Any]) -> None: + self.layer_span = self._layer.start_span(self.span_name) + # The chain is keyed by the current trace id, so the span has to be + # current before this layer's node is added; children read it there. + if push_invocation_chain(self.source, "agent"): + self.invocation_chain_pushed = True + span = self.layer_span.span + # The shared handler finalizes this span on behalf of the invocation, so + # it has to know the span and the context token it may detach. The proxy + # keeps the framework-native span name while it does so. + self.invocation.span = NamePreservingSpan(span) + span.set_attribute(AgentAttributes.SPAN_KIND, SPAN_KIND_VALUE) + span.set_attribute(AgentAttributes.NAME, self.source) + content = maybe_content(self.agent_input) + if content is not None: + span.set_attribute(AgentAttributes.INPUT, content) + set_caller_attributes(span, self.caller_info) + span.set_attribute(AgentAttributes.PAIR_ID, self.pair_id) + self.invocation.input_messages = input_messages_for(self.agent_input) + + run_safely(self._layer.metrics.record_call, self.labels) + + # The ReAct round hooks run inside this call's context, so the tracker + # they read is the one pushed here. + self.react_step_token = push_react_step_tracker( + self._layer.react_step_tracker() + ) + + kwargs["memory_source_info"] = self.caller_info + memory = _memory_module() + if memory is not None: + run_safely( + memory.add_agent_input_info, + self.caller_info, + self.agent_instance, + self.agent_input, + self.pair_id, + ) + + output_stream = kwargs.get("output_stream") + if output_stream is not None: + self.streaming = True + self.labels[MetricLabels.AGENT_STREAMING] = True + span.set_attribute(AgentAttributes.STREAMING, True) + kwargs["output_stream"] = wrap_first_token_stream( + output_stream, self._on_first_token + ) + else: + self.labels[MetricLabels.AGENT_STREAMING] = False + span.set_attribute(AgentAttributes.STREAMING, False) + + def _on_first_token(self) -> None: + if self.layer_span is None: + return + duration = self.layer_span.record_first_token() + run_safely( + self._layer.metrics.record_first_token, duration, self.labels + ) + self.layer_span.span.set_attribute( + AgentAttributes.FIRST_TOKEN_DURATION, duration + ) + + def finish_success(self, result: Any) -> None: + if self.layer_span is None: + return + span = self.layer_span.span + duration = self.layer_span.elapsed() + usage = token_usage_for(self.layer_span.span_id) + + run_safely(self._layer.metrics.record_duration, duration, self.labels) + run_safely(self._layer.metrics.record_tokens, usage, self.labels) + if not self.streaming: + run_safely( + self._layer.metrics.record_first_token, duration, self.labels + ) + span.set_attribute(AgentAttributes.FIRST_TOKEN_DURATION, duration) + + span.set_attribute(AgentAttributes.DURATION, duration) + span.set_attribute(AgentAttributes.STATUS, SUCCESS_STATUS) + # ``AgentTemplate.customized_execute`` returns the caller's whole agent + # input with the answer under ``output``. The agent's output is that + # answer alone: publishing the envelope would echo the prompt and the + # framework's own bookkeeping as the agent's output. + answer = final_answer_of(result) + payload = answer.to_dict() if hasattr(answer, "to_dict") else answer + if payload is not None: + content = maybe_content(payload) + if content is not None: + span.set_attribute(AgentAttributes.OUTPUT, content) + set_usage_attributes(span, AgentAttributes, usage) + + self.invocation.output_messages = output_messages_for(payload) + if usage.prompt_tokens: + self.invocation.input_tokens = usage.prompt_tokens + if usage.completion_tokens: + self.invocation.output_tokens = usage.completion_tokens + self.layer_span.apply_timing(self.invocation) + memory = _memory_module() + if memory is not None: + run_safely( + memory.add_agent_result_info, + self.agent_instance, + result, + self.caller_info, + self.pair_id, + ) + + def finish_error(self, error: BaseException) -> None: + if self.layer_span is None: + return + span = self.layer_span.span + duration = self.layer_span.elapsed() + run_safely( + self._layer.metrics.record_error, error, duration, self.labels + ) + run_safely( + self._layer.metrics.record_tokens, + token_usage_for(self.layer_span.span_id), + self.labels, + ) + set_layer_error_attributes(span, AgentAttributes, error, duration) + set_usage_attributes( + span, AgentAttributes, token_usage_for(self.layer_span.span_id) + ) + set_error_status(span, error_status_text(error)) + + def finalize(self, error: Optional[BaseException] = None) -> None: + """Promote token usage, hand the span to the handler and end it.""" + if self.finished: + return + self.finished = True + # A round the loop never closed -- an LLM call that raised, an aborted + # loop -- still leaves a step span behind, and a step whose span never + # ends stays open for the rest of the process. Close it first: it is a + # child of this span, so it has to end before the agent span does. + run_safely(self._close_open_react_step, error) + self._pop_react_step_tracker() + layer_span = self.layer_span + if layer_span is None: + return + + run_safely( + promote_token_usage_to_parent, + token_usage_for(layer_span.span_id), + layer_span.parent_span_id, + ) + if self.invocation_chain_pushed: + self.invocation_chain_pushed = False + pop_invocation_chain() + self.invocation.context_token = layer_span.take_context_token() + self.invocation.monotonic_end_s = monotonic_now() + if error is None: + run_safely(self._layer.handler.stop_invoke_agent, self.invocation) + else: + run_safely( + self._layer.handler.fail_invoke_agent, + self.invocation, + Error(message=error_status_text(error), type=type(error)), + ) + self._release_context_token(layer_span) + layer_span.end() + + def _close_open_react_step( + self, error: Optional[BaseException] = None + ) -> None: + """Close the step of the round this call ended in the middle of.""" + tracker = current_react_step_tracker() + if tracker is not None: + tracker.close(error) + + def _pop_react_step_tracker(self) -> None: + token, self.react_step_token = self.react_step_token, None + pop_react_step_tracker(token) + + def _release_context_token(self, layer_span: LayerSpan) -> None: + """Detach the span context unless the handler already did. + + The shared handler detaches the token and ends the span itself whenever + it finalizes an invocation; detaching it a second time would raise. If + the handler did not finish the span, the token is still ours to drop. + """ + if not layer_span.span.is_recording(): + self.invocation.context_token = None + leftover = self.invocation.context_token + if leftover is not None: + self.invocation.context_token = None + run_safely(detach_context_token, leftover) + + +class AgentLayer: + """The agent wrapper pair plus the spans and metrics it owns.""" + + def __init__( + self, + tracer: Any, + handler: ExtendedTelemetryHandler, + meter: Meter, + step_layer: Optional[ReactStepLayer] = None, + ) -> None: + self._tracer = tracer + self.handler = handler + self.metrics = AgentMetrics(meter) + #: The STEP spans this call's Reasoning-Acting rounds are recorded as. + self.step_layer = step_layer or ReactStepLayer(handler) + + def react_step_tracker(self) -> Any: + """A tracker the ReAct round hooks of this call report into.""" + return self.step_layer.tracker() + + def start_span(self, span_name: str) -> LayerSpan: + return LayerSpan(self._tracer, span_name, SpanKind.INTERNAL).attach() + + def wrap_sync(self, func: Any, *args: Any, **kwargs: Any) -> Any: + call = build_safely(_AgentCall, self, func, args, kwargs) + if call is None: + return func(*args, **kwargs) + error: Optional[BaseException] = None + agent_name_token = None + try: + run_safely(call.start, kwargs) + agent_name_token = push_current_agent_name( + call.source, call.session_id + ) + result = func(*args, **kwargs) + except BaseException as exc: + error = exc + run_safely(call.finish_error, exc) + raise + else: + run_safely(call.finish_success, result) + return result + finally: + run_safely(call.finalize, error) + pop_current_agent_name(agent_name_token) + + async def wrap_async(self, func: Any, *args: Any, **kwargs: Any) -> Any: + call = build_safely(_AgentCall, self, func, args, kwargs) + if call is None: + return await func(*args, **kwargs) + error: Optional[BaseException] = None + agent_name_token = None + try: + run_safely(call.start, kwargs) + agent_name_token = push_current_agent_name( + call.source, call.session_id + ) + result = await func(*args, **kwargs) + except BaseException as exc: + error = exc + run_safely(call.finish_error, exc) + raise + else: + run_safely(call.finish_success, result) + return result + finally: + run_safely(call.finalize, error) + pop_current_agent_name(agent_name_token) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_common.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_common.py new file mode 100644 index 000000000..8877a0cb4 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_common.py @@ -0,0 +1,891 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared helpers for the LoongSuite agentUniverse instrumentation. + +Nothing in this module -- or anywhere else in this package -- imports or +delegates to agentUniverse's own OTel instrumentors. The wrapper extension +points in ``agentuniverse.base.annotation.trace`` are claimed by this package, +and the span, attribute, metric and token-usage work is done here from scratch. +Only the framework's non-telemetry business surfaces are reused: the +``_get_*_info`` argument readers, ``ConversationMemoryModule``, +``InvocationChainContext``/``Monitor`` and the ``AuTraceManager`` token API. +""" + +from __future__ import annotations + +import base64 +import datetime +import decimal +import json +import logging +import time +import traceback +from contextvars import ContextVar, Token +from typing import Any, Callable, Dict, Optional, Type + +from agentuniverse.base.tracing.au_trace_manager import ( + add_current_token_usage, + add_current_token_usage_to_parent, + get_current_token_usage, + init_new_token_usage, +) +from agentuniverse.base.util.monitor.monitor import Monitor +from agentuniverse.llm.llm_output import TokenUsage + +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.trace import Span, SpanKind, Status, StatusCode +from opentelemetry.util.genai.extended_semconv.gen_ai_extended_attributes import ( + GEN_AI_SESSION_ID, +) +from opentelemetry.util.genai.types import ( + InputMessage, + OutputMessage, + Reasoning, + Text, + ToolCall, + ToolCallResponse, +) +from opentelemetry.util.genai.utils import ( + ContentCapturingMode, + get_content_capturing_mode, + is_experimental_mode, +) + +logger = logging.getLogger(__name__) + +INSTRUMENTOR_NAME = "loongsuite-instrumentation-agentuniverse" + +#: Framework name reported on every ``gen_ai.*`` span this package creates. +FRAMEWORK_NAME = "agentuniverse" + +#: ``gen_ai.framework`` is a LoongSuite extension attribute. The shared handler +#: does not set it, so each layer records it here. +GEN_AI_FRAMEWORK = "gen_ai.framework" + +#: The wrapper extension points this package claims. These are the module-level +#: globals the ``@trace_agent``/``@trace_llm``/``@trace_tool`` decorators resolve +#: on every call, so replacing them intercepts every call. +LAYER_WRAPPER_GLOBALS: tuple[tuple[str, str], ...] = ( + ("_agent_wrapper_sync", "_agent_wrapper_async"), + ("_llm_wrapper_sync", "_llm_wrapper_async"), + ("_tool_wrapper_sync", "_tool_wrapper_async"), +) + +WRAPPER_GLOBAL_NAMES: tuple[str, ...] = tuple( + name for pair in LAYER_WRAPPER_GLOBALS for name in pair +) + +#: Span status values used by the framework's ``au.*.status`` attribute. +SUCCESS_STATUS = "success" +ERROR_STATUS = "error" + + +# --------------------------------------------------------------------------- +# Serialization +# --------------------------------------------------------------------------- + + +def _fallback(value: Any) -> Any: + """JSON fallback for values the stdlib encoder rejects.""" + if isinstance(value, (datetime.datetime, datetime.date)): + return value.isoformat() + if isinstance(value, bytes): + return base64.b64encode(value).decode() + if isinstance(value, (set, frozenset)): + return list(value) + if isinstance(value, decimal.Decimal): + return float(value) + return str(value) + + +def safe_json_dumps(value: Any) -> str: + """Serialize ``value`` for a span attribute, never raising.""" + try: + return json.dumps(value, default=_fallback, ensure_ascii=False) + except Exception: # pragma: no cover - defensive + return str(value) + + +def error_to_string(error: BaseException) -> str: + """Render an exception for the ``au.*.error.message`` attribute. + + The framework's own instrumentation always writes the full traceback. A + traceback quotes the values flowing through the failing call, and so does an + exception's own message, so with content capture disabled this package + reports the exception type alone -- see the privacy section of the README. + """ + if not captures_span_content(): + return type(error).__name__ + try: + return "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + except Exception: # pragma: no cover - defensive + return type(error).__name__ + + +def error_status_text(error: BaseException) -> str: + """The span status description for a failed call. + + Uses the same privacy rule as the ``au.*.error.message`` attribute, so a + status description can never carry content the attributes may not. + """ + return error_to_string(error) + + +def build_safely( + factory: Callable[..., Any], *args: Any, **kwargs: Any +) -> Any: + """Build one layer's call record, or ``None`` when that is not possible. + + Reading the framework's call information can fail -- an unusual call shape, a + component that is only half configured. Telemetry must never turn that into a + failing application call, so the caller runs the original function untouched + when this returns ``None``. + """ + try: + return factory(*args, **kwargs) + except Exception: + logger.warning( + "Could not prepare agentUniverse instrumentation for a call; " + "running it without instrumentation", + exc_info=True, + ) + return None + + +# --------------------------------------------------------------------------- +# Content capture / privacy +# --------------------------------------------------------------------------- + + +def content_capturing_mode() -> ContentCapturingMode: + """The effective GenAI content capturing mode, fail-safe. + + The shared util raises when the GenAI stability opt-in is not experimental, + which is exactly the "no content" case here. + """ + try: + if not is_experimental_mode(): + return ContentCapturingMode.NO_CONTENT + return get_content_capturing_mode() + except Exception: # pragma: no cover - defensive + return ContentCapturingMode.NO_CONTENT + + +def captures_span_content() -> bool: + """Whether user content may be written to span attributes. + + Evaluated per call from the same shared configuration the GenAI handler + reads, so the ``au.*`` compatibility carriers and the handler's + ``gen_ai.*`` content attributes are always gated identically. ``EVENT_ONLY`` + keeps span content redacted. + """ + return content_capturing_mode() in ( + ContentCapturingMode.SPAN_ONLY, + ContentCapturingMode.SPAN_AND_EVENT, + ) + + +def maybe_content(value: Any) -> Optional[str]: + """Serialize ``value`` for a span content carrier, or ``None``.""" + if not captures_span_content(): + return None + return safe_json_dumps(value) + + +def maybe_content_text(value: Any) -> Optional[str]: + """Render ``value`` as span content text, or ``None`` when redacted.""" + if not captures_span_content(): + return None + if isinstance(value, str): + return value + return safe_json_dumps(value) + + +#: LLM request parameters that are safe to report with content capture disabled. +#: They are scalars by nature, so they cannot carry prompt or message content. +SAFE_LLM_PARAM_KEYS = frozenset( + { + "temperature", + "top_p", + "top_k", + "max_tokens", + "max_new_tokens", + "presence_penalty", + "frequency_penalty", + "repetition_penalty", + "n", + "seed", + "stream", + "logprobs", + "top_logprobs", + "response_format", + "stop", + } +) + + +def llm_params_attribute(params: Any) -> str: + """The ``au.llm.llm_params`` payload for a call. + + With content capture enabled this is the whole parameter mapping, exactly + what the framework's own instrumentation reports. With capture disabled only + the scalar request parameters in :data:`SAFE_LLM_PARAM_KEYS` are kept, so a + call that passes content through its parameters cannot leak it. + """ + if captures_span_content(): + return safe_json_dumps(params) + safe: Dict[str, Any] = {} + if isinstance(params, dict): + for key, value in params.items(): + if key not in SAFE_LLM_PARAM_KEYS: + continue + if isinstance(value, (int, float, bool)) or value is None: + safe[key] = value + return safe_json_dumps(safe) + + +# --------------------------------------------------------------------------- +# Token usage (framework API; the usage bookkeeping belongs to our spans) +# --------------------------------------------------------------------------- + + +def reset_token_usage(span_id: Optional[int] = None) -> None: + init_new_token_usage(span_id) + + +def token_usage_for(span_id: Optional[int] = None) -> TokenUsage: + """Token usage accumulated for ``span_id``, defaulting to an empty one.""" + try: + return get_current_token_usage(span_id) + except Exception: # pragma: no cover - defensive + return TokenUsage() + + +def record_token_usage( + usage: TokenUsage, span_id: Optional[int] = None +) -> None: + """Accumulate ``usage`` onto ``span_id`` exactly once.""" + if usage is None: + return + try: + add_current_token_usage(usage, span_id) + except Exception: # pragma: no cover - defensive + logger.debug("Failed to accumulate token usage", exc_info=True) + + +def promote_token_usage_to_parent( + usage: Optional[TokenUsage] = None, + parent_span_id: Optional[int] = None, +) -> None: + """Move a finished layer span's usage up to its parent span.""" + try: + add_current_token_usage_to_parent(usage, parent_span_id) + except Exception: # pragma: no cover - defensive + logger.debug("Failed to promote token usage", exc_info=True) + + +# --------------------------------------------------------------------------- +# Span lifecycle +# --------------------------------------------------------------------------- + + +class LayerSpan: + """One LoongSuite-owned layer span plus its attached context token.""" + + def __init__( + self, + tracer: trace.Tracer, + span_name: str, + span_kind: SpanKind = SpanKind.INTERNAL, + ) -> None: + self.name = span_name + self.span: Span = tracer.start_span( + name=span_name, + kind=span_kind, + context=otel_context.get_current(), + ) + self.context_token = None + self.start_time = time.time() + self.monotonic_start_s = time.monotonic() + self.first_token_time: Optional[float] = None + self.first_token_monotonic: Optional[float] = None + + def attach(self) -> "LayerSpan": + self.context_token = otel_context.attach( + trace.set_span_in_context(self.span) + ) + try: + # The framework's token bookkeeping is keyed by the span that is + # current when a layer starts, so seed it right after attaching. + reset_token_usage(self.span.context.span_id) + self.mark_gen_ai_identity() + except BaseException: + # A half-attached span is worse than no span: the context token + # would stay attached and the span open, so every later span of + # this trace would be filed under a span that never finished. + # Undo both, then let the caller run the call uninstrumented. + self.detach() + self.end() + raise + return self + + def take_context_token(self) -> Any: + """Hand the context token to the handler that will detach it.""" + token, self.context_token = self.context_token, None + return token + + @property + def span_id(self) -> Optional[int]: + return self.span.context.span_id + + @property + def parent_span_id(self) -> Optional[int]: + parent = self.span.parent + return parent.span_id if parent is not None else None + + def elapsed(self) -> float: + """Seconds since the layer started, measured off the call clock.""" + return time.time() - self.start_time + + def record_first_token(self) -> float: + """Record and return the elapsed seconds up to the first token.""" + self.first_token_monotonic = time.monotonic() + self.first_token_time = time.time() + return self.first_token_time - self.start_time + + def apply_timing(self, invocation: Any) -> None: + """Hand the shared handler the clocks behind its timing attributes. + + ``gen_ai.response.time_to_first_token`` is derived by the shared handler + from the monotonic start and first-token times, so both are recorded + here. A call that never saw a streamed token treats its response as the + first token, measured when the call finishes. + """ + invocation.monotonic_start_s = self.monotonic_start_s + if self.first_token_monotonic is None: + self.first_token_monotonic = time.monotonic() + invocation.monotonic_first_token_s = self.first_token_monotonic + + def mark_gen_ai_identity(self) -> None: + self.span.set_attribute(GEN_AI_FRAMEWORK, FRAMEWORK_NAME) + session_id = agent_name_attributes().get(GEN_AI_SESSION_ID) + if session_id: + self.span.set_attribute(GEN_AI_SESSION_ID, session_id) + + def end(self) -> None: + if self.span.is_recording(): + self.span.end() + + def detach(self) -> None: + token, self.context_token = self.context_token, None + detach_context_token(token) + + +def detach_context_token(token: Any) -> None: + """Detach a context token once, never raising.""" + if token is None: + return + try: + otel_context.detach(token) + except Exception: # pragma: no cover - cross-context teardown + logger.debug("Failed to detach span context", exc_info=True) + + +def monotonic_now() -> float: + return time.monotonic() + + +class NamePreservingSpan(Span): + """Keeps the framework-native span name while the shared handler finalizes. + + The shared GenAI handler normalises a span name to its own ``gen_ai`` form + (``invoke_agent X``/``chat ...``/``execute_tool X``) and ends the span + itself, which would replace the ``au.agent.*``/``au.llm.*``/``au.tool.*`` + span name that agentUniverse's own instrumentation exports. Handing the + handler this thin proxy instead -- every other call is forwarded to the real + span, ``end()`` included -- keeps the native span-name contract while the + handler still writes its ``gen_ai.*`` attributes, records its own metrics + and closes the span. + """ + + def __init__(self, span: Span) -> None: + self._span = span + + def update_name(self, name: str) -> None: + """Drop the handler's name normalisation.""" + + def get_span_context(self): + return self._span.get_span_context() + + def set_attribute(self, key, value): + return self._span.set_attribute(key, value) + + def set_attributes(self, attributes): + return self._span.set_attributes(attributes) + + def add_event(self, *args, **kwargs): + return self._span.add_event(*args, **kwargs) + + def set_status(self, *args, **kwargs): + return self._span.set_status(*args, **kwargs) + + def record_exception(self, *args, **kwargs): + return self._span.record_exception(*args, **kwargs) + + def is_recording(self): + return self._span.is_recording() + + def end(self, *args, **kwargs): + return self._span.end(*args, **kwargs) + + def __getattr__(self, item: str) -> Any: + return getattr(self._span, item) + + +def set_error_status(span: Span, message: str) -> None: + span.set_status(Status(StatusCode.ERROR, message)) + + +# --------------------------------------------------------------------------- +# The agentUniverse ``au.*`` attribute contract, per layer +# --------------------------------------------------------------------------- + + +class AgentAttributes: + SPAN_KIND = "au.span.kind" + NAME = "au.agent.name" + INPUT = "au.agent.input" + OUTPUT = "au.agent.output" + DURATION = "au.agent.duration" + STATUS = "au.agent.status" + PAIR_ID = "au.agent.pair_id" + STREAMING = "au.agent.streaming" + FIRST_TOKEN_DURATION = "au.agent.first_token.duration" + ERROR_TYPE = "au.agent.error.type" + ERROR_MESSAGE = "au.agent.error.message" + CALLER_NAME = "au.trace.caller_name" + CALLER_TYPE = "au.trace.caller_type" + USAGE_TOTAL_TOKENS = "au.agent.usage.total_tokens" + USAGE_PROMPT_TOKENS = "au.agent.usage.prompt_tokens" + USAGE_COMPLETION_TOKENS = "au.agent.usage.completion_tokens" + USAGE_DETAIL_TOKENS = "au.agent.usage.detail_tokens" + + +class LLMAttributes: + SPAN_KIND = "au.span.kind" + NAME = "au.llm.name" + CHANNEL_NAME = "au.llm.channel_name" + INPUT = "au.llm.input" + OUTPUT = "au.llm.output" + LLM_PARAMS = "au.llm.llm_params" + STREAMING = "au.llm.streaming" + DURATION = "au.llm.duration" + STATUS = "au.llm.status" + FIRST_TOKEN_DURATION = "au.llm.first_token.duration" + ERROR_TYPE = "au.llm.error.type" + ERROR_MESSAGE = "au.llm.error.message" + CALLER_NAME = "au.trace.caller_name" + CALLER_TYPE = "au.trace.caller_type" + USAGE_TOTAL_TOKENS = "au.llm.usage.total_tokens" + USAGE_PROMPT_TOKENS = "au.llm.usage.prompt_tokens" + USAGE_COMPLETION_TOKENS = "au.llm.usage.completion_tokens" + USAGE_DETAIL_TOKENS = "au.llm.usage.detail_tokens" + + +class ToolAttributes: + SPAN_KIND = "au.span.kind" + NAME = "au.tool.name" + INPUT = "au.tool.input" + OUTPUT = "au.tool.output" + DURATION = "au.tool.duration" + STATUS = "au.tool.status" + PAIR_ID = "au.tool.pair_id" + ERROR_TYPE = "au.tool.error.type" + ERROR_MESSAGE = "au.tool.error.message" + CALLER_NAME = "au.trace.caller_name" + CALLER_TYPE = "au.trace.caller_type" + USAGE_TOTAL_TOKENS = "au.tool.usage.total_tokens" + USAGE_PROMPT_TOKENS = "au.tool.usage.prompt_tokens" + USAGE_COMPLETION_TOKENS = "au.tool.usage.completion_tokens" + USAGE_DETAIL_TOKENS = "au.tool.usage.detail_tokens" + + +class MetricLabels: + """Metric label keys, shared by the three layers.""" + + CALLER_NAME = "au_trace_caller_name" + CALLER_TYPE = "au_trace_caller_type" + AGENT_NAME = "au_agent_name" + AGENT_STATUS = "au_agent_status" + AGENT_STREAMING = "au_agent_streaming" + LLM_NAME = "au_llm_name" + LLM_CHANNEL_NAME = "au_llm_channel_name" + LLM_STATUS = "au_llm_status" + LLM_STREAMING = "au_llm_streaming" + TOOL_NAME = "au_tool_name" + TOOL_STATUS = "au_tool_status" + + +def caller_labels(caller_info: Dict[str, Any]) -> Dict[str, Any]: + """The caller labels every layer puts on its metrics.""" + return { + MetricLabels.CALLER_NAME: caller_info.get("source", "unknown"), + MetricLabels.CALLER_TYPE: caller_info.get("type", "user"), + } + + +def set_caller_attributes(span: Span, caller_info: Dict[str, Any]) -> None: + span.set_attribute( + AgentAttributes.CALLER_NAME, caller_info.get("source", "unknown") + ) + span.set_attribute( + AgentAttributes.CALLER_TYPE, caller_info.get("type", "user") + ) + + +def set_usage_attributes( + span: Span, names: Type[Any], usage: TokenUsage +) -> None: + """Record the ``au.*.usage.*`` attributes for a layer.""" + span.set_attribute(names.USAGE_TOTAL_TOKENS, usage.total_tokens) + span.set_attribute(names.USAGE_PROMPT_TOKENS, usage.prompt_tokens) + span.set_attribute(names.USAGE_COMPLETION_TOKENS, usage.completion_tokens) + span.set_attribute( + names.USAGE_DETAIL_TOKENS, safe_json_dumps(usage.to_dict()) + ) + + +def set_layer_error_attributes( + span: Span, names: Type[Any], error: BaseException, duration: float +) -> None: + """Record the ``au.*`` error contract for a layer.""" + span.set_attribute(names.STATUS, ERROR_STATUS) + span.set_attribute(names.ERROR_TYPE, type(error).__name__) + span.set_attribute(names.ERROR_MESSAGE, error_to_string(error)) + span.set_attribute(names.DURATION, duration) + + +# --------------------------------------------------------------------------- +# GenAI helpers +# --------------------------------------------------------------------------- + + +def run_safely(action: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: + """Run one telemetry step without ever breaking the application. + + Telemetry must not turn a working agent call into a failing one, so every + tracing/metrics step the layers run goes through here. Anything that goes + wrong is logged and dropped. + """ + try: + return action(*args, **kwargs) + except Exception: # pragma: no cover - telemetry must never raise + logger.debug( + "agentUniverse instrumentation step failed", exc_info=True + ) + return None + + +def user_message(content: str) -> InputMessage: + return InputMessage(role="user", parts=[Text(content=content)]) + + +def assistant_message( + content: str, finish_reason: str = "stop" +) -> OutputMessage: + return OutputMessage( + role="assistant", + parts=[Text(content=content)], + finish_reason=finish_reason, + ) + + +def input_messages_for(payload: Any) -> list: + """Preserve public message roles, tool IDs and the actual user input.""" + if isinstance(payload, dict): + nested = payload.get("kwargs") + values = nested if isinstance(nested, dict) else payload + for key in ("messages", "prompt", "input"): + if key in values: + return input_messages_for(values[key]) + positional = payload.get("args") + if isinstance(positional, (list, tuple)) and positional: + return input_messages_for(positional[0]) + if isinstance(payload, (list, tuple)): + messages = [] + for message in payload: + role = message_field(message, "role") or message_field( + message, "type" + ) + role = getattr(role, "value", role) + if role not in ( + "user", + "assistant", + "system", + "developer", + "tool", + ): + continue + content = message_field(message, "content") + if role == "tool": + parts = [ + ToolCallResponse( + response=content, + id=message_field(message, "tool_call_id"), + ) + ] + else: + parts = content_parts(content) + reasoning = message_field(message, "reasoning_content") + if isinstance(reasoning, str) and reasoning: + parts.insert(0, Reasoning(content=reasoning)) + parts.extend( + tool_call_parts(message_field(message, "tool_calls")) + ) + messages.append(InputMessage(role=role, parts=parts)) + return messages + if isinstance(payload, str): + text = payload + else: + text = safe_json_dumps(payload) + return [user_message(text)] if text else [] + + +def message_field(message: Any, name: str) -> Any: + return ( + message.get(name) + if isinstance(message, dict) + else getattr(message, name, None) + ) + + +def content_parts(content: Any) -> list: + if isinstance(content, str): + return [Text(content=content)] if content else [] + if isinstance(content, list): + return [ + Text(content=item["text"]) + for item in content + if isinstance(item, dict) + and item.get("type") in ("text", "input_text", "output_text") + and isinstance(item.get("text"), str) + ] + return [] + + +def parse_tool_arguments(value: Any) -> Any: + """Parse the framework's JSON tool arguments for a GenAI tool call. + + The framework carries tool arguments as the provider's JSON string. The + shared GenAI type holds them parsed, so a string that is valid JSON becomes + the object it encodes and anything else is passed through unchanged. + """ + if not isinstance(value, str): + return value + text = value.strip() + if not text: + return value + try: + parsed = json.loads(text) + except Exception: + return value + return parsed if isinstance(parsed, dict) else value + + +def tool_call_function(call: Any) -> Any: + """The ``FunctionCall`` of a framework tool call, when it has one. + + The framework's own ``ToolCall`` keeps the name and the JSON arguments one + level down, in ``function`` (``{"id": ..., "type": "function", + "function": {"name": ..., "arguments": ...}}``), which is also the shape the + provider sends. + """ + if isinstance(call, dict): + return call.get("function") + return getattr(call, "function", None) + + +def tool_call_field(call: Any, name: str) -> Any: + """One field of a tool call, wherever its shape keeps it.""" + function = tool_call_function(call) + if function is not None: + value = ( + function.get(name) + if isinstance(function, dict) + else getattr(function, name, None) + ) + if value is not None: + return value + if isinstance(call, dict): + return call.get(name) + return getattr(call, name, None) + + +def tool_call_parts(payload: Any) -> list: + """Map the framework's tool calls onto the shared GenAI tool-call parts.""" + if not payload: + return [] + parts = [] + for call in payload: + parts.append( + ToolCall( + arguments=parse_tool_arguments( + tool_call_field(call, "arguments") + ), + name=str(tool_call_field(call, "name") or ""), + id=tool_call_field(call, "id"), + ) + ) + return parts + + +def output_messages_for( + payload: Any, + finish_reason: str = "stop", + tool_calls: Any = None, + reasoning: Optional[str] = None, +) -> list: + """The assistant message a layer reports for its own output. + + A round that answers with tool calls has no text: the calls themselves are + the output, so ``tool_calls`` becomes a tool-call part of the same message. + """ + parts: list = [] + if isinstance(reasoning, str) and reasoning: + parts.append(Reasoning(content=reasoning)) + if payload is not None: + text = ( + payload if isinstance(payload, str) else safe_json_dumps(payload) + ) + if text: + parts.append(Text(content=text)) + parts.extend(tool_call_parts(tool_calls)) + if not parts: + return [] + return [ + OutputMessage( + role="assistant", parts=parts, finish_reason=finish_reason + ) + ] + + +def final_answer_of(result: Any) -> Any: + """The agent's final answer, without the framework's own call envelope. + + ``AgentTemplate.customized_execute`` returns the caller's whole agent input + with the answer under ``output``. Only that answer is the agent's output; + echoing the input back would publish the prompt and the framework's + bookkeeping as the agent's output. + """ + if isinstance(result, dict): + return result.get("output") + getter = getattr(result, "get_data", None) + if callable(getter): + answer = run_safely(getter, "output") + if answer is not None: + return answer + return getattr(result, "output", result) + + +_current_agent_name: ContextVar[tuple] = ContextVar( + "agentuniverse_current_agent_name", default=(None, None) +) + + +def push_current_agent_name( + name: Optional[str], session_id: Optional[str] = None +) -> Optional[Token]: + """Track the agent a nested LLM/tool call belongs to.""" + if not name: + return None + try: + if session_id is None: + session_id = _current_agent_name.get()[1] + return _current_agent_name.set((name, session_id)) + except Exception: # pragma: no cover - defensive + return None + + +def pop_current_agent_name(token: Optional[Token]) -> None: + if token is None: + return + try: + _current_agent_name.reset(token) + except Exception: # pragma: no cover - defensive + logger.debug("Failed to reset current agent name", exc_info=True) + + +def agent_name_attributes() -> Dict[str, Any]: + """``gen_ai.agent.name`` for a nested layer span, when there is one.""" + name, session_id = _current_agent_name.get() + attributes = {GenAI.GEN_AI_AGENT_NAME: name} if name else {} + if session_id: + attributes[GEN_AI_SESSION_ID] = session_id + return attributes + + +def push_invocation_chain(source: str, node_type: str) -> bool: + """Add this layer's node to the framework's invocation chain. + + The chain lives in the framework context under the current trace id, so the + layer's span has to be the current span before its node is added: pushing it + earlier would file the node under a different trace id than the one the + instrumented call's own children read. ``True`` means this call added the + node and is therefore the one that may remove it again. + """ + try: + Monitor.init_invocation_chain() + Monitor.add_invocation_chain({"source": source, "type": node_type}) + except Exception: + logger.debug( + "Could not extend the agentUniverse invocation chain", + exc_info=True, + ) + return False + return True + + +def pop_invocation_chain() -> None: + """Remove this layer's node; on failure a parent's entry stays put.""" + try: + Monitor.pop_invocation_chain() + except Exception: + logger.debug( + "Could not leave the agentUniverse invocation chain", + exc_info=True, + ) + + +def describe_wrapper_owner(wrapper: Any) -> Optional[str]: + """Name the class a wrapper extension point is bound to, if any. + + The wrapper extension points are plain module globals, so whoever + instrumented the framework before this package either installed a plain + function or bound a method of its instrumentor class. This is used for + logging only -- nothing is imported, constructed or called. + """ + if wrapper is None: + return None + owner = getattr(wrapper, "__self__", None) + if owner is None: + return None + owner_type = type(owner) + return f"{owner_type.__module__}.{owner_type.__name__}" diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_hooks.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_hooks.py new file mode 100644 index 000000000..0f4e0d14e --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_hooks.py @@ -0,0 +1,138 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reversible hooks at provider conversion and native tool dispatch boundaries.""" + +import inspect +from contextvars import ContextVar +from functools import wraps + +from agentuniverse.agent.agent import Agent +from agentuniverse.llm.llm_output import TokenUsage + +from ._common import ( + pop_current_agent_name, + push_current_agent_name, + run_safely, +) +from ._usage import capture_provider_usage + +_tool_call_id: ContextVar[str | None] = ContextVar( + "agentuniverse_tool_call_id", default=None +) + + +def current_tool_call_id(): + return _tool_call_id.get() + + +def _usage_hook(original, provider): + @wraps(original.__func__) + def converted(cls, payload, *args, **kwargs): + # The SDK conversion stays outside telemetry error handling and its + # exact result object is returned, even when our collector fails. + result = original.__get__(None, cls)(payload, *args, **kwargs) + run_safely(capture_provider_usage, payload, provider) + return result + + return classmethod(converted) + + +def _dispatch_hook(original, asynchronous): + if asynchronous: + + @wraps(original) + async def dispatch(self, tool_call, *args, **kwargs): + identifier = run_safely(getattr, tool_call, "id", None) + token = _tool_call_id.set( + identifier if isinstance(identifier, str) else None + ) + name, session = run_safely( + _dispatch_metadata, self, args, kwargs + ) or (None, None) + agent_token = push_current_agent_name(name, session) + try: + return await original(self, tool_call, *args, **kwargs) + finally: + pop_current_agent_name(agent_token) + _tool_call_id.reset(token) + else: + + @wraps(original) + def dispatch(self, tool_call, *args, **kwargs): + identifier = run_safely(getattr, tool_call, "id", None) + token = _tool_call_id.set( + identifier if isinstance(identifier, str) else None + ) + name, session = run_safely( + _dispatch_metadata, self, args, kwargs + ) or (None, None) + agent_token = push_current_agent_name(name, session) + try: + return original(self, tool_call, *args, **kwargs) + finally: + pop_current_agent_name(agent_token) + _tool_call_id.reset(token) + + return dispatch + + +def _dispatch_metadata(agent, args, kwargs): + # The framework's thread pool propagates OTel context, rather than every + # ContextVar. Its explicit AgentContext is authoritative in each worker. + current = kwargs.get("agent_context") or (args[0] if args else None) + session = getattr(current, "session_id", None) + info = getattr(getattr(agent, "agent_model", None), "info", None) + name = info.get("name") if isinstance(info, dict) else None + return ( + name if isinstance(name, str) else None, + session if isinstance(session, str) else None, + ) + + +def restore_auxiliary_hooks(hooks): + for cls, name, original, wrapper in reversed(hooks): + if inspect.getattr_static(cls, name, None) is wrapper: + run_safely(setattr, cls, name, original) + + +def install_auxiliary_hooks(): + hooks = [] + try: + for name, provider in ( + ("from_openai", "openai"), + ("from_anthropic", "anthropic"), + ): + original = inspect.getattr_static(TokenUsage, name, None) + if original is None: + continue + if not isinstance(original, classmethod): + raise TypeError("Unsupported TokenUsage converter") + wrapper = _usage_hook(original, provider) + hooks.append((TokenUsage, name, original, wrapper)) + setattr(TokenUsage, name, wrapper) + for name, asynchronous in ( + ("execute_tool_call", False), + ("async_execute_tool_call", True), + ): + original = inspect.getattr_static(Agent, name, None) + if original is None: + continue + wrapper = _dispatch_hook(original, asynchronous) + hooks.append((Agent, name, original, wrapper)) + setattr(Agent, name, wrapper) + except BaseException: + restore_auxiliary_hooks(hooks) + raise + return hooks diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_llm.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_llm.py new file mode 100644 index 000000000..ae7da4c21 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_llm.py @@ -0,0 +1,1011 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The LLM layer of the LoongSuite agentUniverse instrumentation. + +This module claims the ``_llm_wrapper_sync``/``_llm_wrapper_async`` extension +points the ``@trace_llm`` decorator resolves and builds the ``au.llm.*`` span, +metrics and token bookkeeping itself. ``gen_ai.*`` attributes and metrics on the +same span come from the shared ``ExtendedTelemetryHandler``. + +Streaming keeps one intentional difference from the framework's own +instrumentation: the token usage reported by the stream is aggregated onto the +parent span exactly once (see README). + +No agentUniverse OTel class is imported or called here. +""" + +from __future__ import annotations + +import logging +from typing import Any, AsyncGenerator, Dict, Generator, Optional + +from agentuniverse.base.annotation import trace as trace_module +from agentuniverse.base.annotation.trace import _get_llm_info +from agentuniverse.base.util.monitor.monitor import Monitor +from agentuniverse.llm.llm_output import LLMOutput, TokenUsage + +from opentelemetry.metrics import Meter +from opentelemetry.trace import SpanKind +from opentelemetry.util.genai.extended_handler import ExtendedTelemetryHandler +from opentelemetry.util.genai.types import ( + Error, + FunctionToolDefinition, + LLMInvocation, +) + +from ._common import ( + SUCCESS_STATUS, + LayerSpan, + LLMAttributes, + MetricLabels, + NamePreservingSpan, + agent_name_attributes, + build_safely, + caller_labels, + error_status_text, + input_messages_for, + llm_params_attribute, + maybe_content, + maybe_content_text, + monotonic_now, + output_messages_for, + pop_invocation_chain, + promote_token_usage_to_parent, + push_invocation_chain, + record_token_usage, + run_safely, + set_caller_attributes, + set_error_status, + set_layer_error_attributes, + set_usage_attributes, + token_usage_for, +) +from ._usage import provider_usage, usage_call_context + +logger = logging.getLogger(__name__) + +SPAN_KIND_VALUE = "llm" + + +def usage_of(result: Any) -> Optional[TokenUsage]: + """Prefer inclusive provider totals; never mutate SDK usage objects.""" + raw = getattr(result, "raw", None) + if isinstance(raw, dict): + payload = raw.get("usage") + provider = ( + "anthropic" + if isinstance(payload, dict) + and ( + "cache_read_input_tokens" in payload + or "cache_creation_input_tokens" in payload + ) + else "openai" + ) + derived = provider_usage(payload, provider) + if derived is not None: + return derived + return getattr(result, "usage", None) + + +def chunk_record(chunk: Any) -> tuple: + """One streamed chunk as ``(text, usage)``, from either framework shape. + + A chunk of the stable ``LLMOutput`` shape carries its text in ``text``, and + an empty string there is a real value: a role-only or usage-only chunk + contributes nothing rather than its repr. From 0.1.0a2 on a streamed chunk -- + or the ``LLMStreamEvent`` wrapper around it -- carries the delta in + ``text_delta`` instead, and ``LLMStreamEvent.text`` is a class method rather + than the payload, so a reader that only knows ``text`` drops the whole + stream's output. A chunk with neither field is not a framework payload and is + rendered as it is. + """ + text = "" + for name in ("text", "text_delta"): + value = getattr(chunk, name, None) + if isinstance(value, str): + text = value + break + else: + if not hasattr(chunk, "text") and not hasattr(chunk, "text_delta"): + text = str(chunk) + return text, getattr(chunk, "usage", None) + + +def chunk_text(chunk: Any) -> str: + """The text one streamed chunk contributes to the LLM span's output.""" + return chunk_record(chunk)[0] + + +def raw_choice(result: Any) -> dict: + """The first public OpenAI choice retained by stable LLMOutput.raw.""" + raw = getattr(result, "raw", None) + choices = raw.get("choices") if isinstance(raw, dict) else None + if isinstance(choices, list) and choices and isinstance(choices[0], dict): + return choices[0] + return {} + + +def raw_message(result: Any) -> dict: + choice = raw_choice(result) + message = choice.get("delta") or choice.get("message") + return message if isinstance(message, dict) else {} + + +def tool_calls_of(result: Any) -> list: + """The tool calls an LLM result asked for, from either framework shape. + + ``LLMOutput.get_tool_calls()`` is the 0.1.0a2 accessor; the stable shape + keeps the provider message in ``LLMOutput.message``. A plain answer has + neither and reports no calls. + """ + getter = getattr(result, "get_tool_calls", None) + if callable(getter): + calls = run_safely(getter) + if calls: + return list(calls) + calls = getattr(result, "tool_calls", None) + if calls: + return list(calls) + message = getattr(result, "message", None) + if isinstance(message, dict) and message.get("tool_calls"): + return list(message["tool_calls"]) + return list(raw_message(result).get("tool_calls") or []) + + +def _streamed_tool_call_deltas(chunk: Any) -> list: + """The tool-call deltas one streamed chunk carries, if any.""" + delta = getattr(chunk, "tool_call_delta", None) + if delta is not None: + return [delta] + deltas = getattr(chunk, "tool_calls", None) + if deltas: + return list(deltas) + message = getattr(chunk, "message", None) + if isinstance(message, dict) and message.get("tool_calls"): + return list(message["tool_calls"]) + return list(raw_message(chunk).get("tool_calls") or []) + + +class ToolCallAccumulator: + """Assembles the tool calls a stream spreads across its chunks. + + A streamed tool call arrives in fragments: the first delta carries the id + and the name, every later one carries another slice of the JSON arguments. + The calls are kept in the order of their indices, which is the order the + provider numbered them. + """ + + def __init__(self) -> None: + self._calls: Dict[int, Dict[str, Any]] = {} + + def feed(self, chunk: Any) -> None: + for position, delta in enumerate(_streamed_tool_call_deltas(chunk)): + index = getattr(delta, "index", None) + identifier = getattr(delta, "id", None) + name = getattr(delta, "name", None) + fragment = getattr(delta, "arguments_delta", None) + if isinstance(delta, dict): + index = delta.get("index", index) + identifier = delta.get("id", identifier) + name = delta.get("name", name) + fragment = delta.get("arguments_delta", fragment) + function = delta.get("function") + if isinstance(function, dict): + name = function.get("name", name) + fragment = function.get("arguments", fragment) + if not isinstance(index, int): + index = len(self._calls) if index is None else position + call = self._calls.setdefault( + index, {"id": None, "name": "", "arguments": ""} + ) + if identifier: + call["id"] = identifier + if name: + call["name"] = name + if isinstance(fragment, str): + call["arguments"] += fragment + + def result(self) -> list: + return [self._calls[index] for index in sorted(self._calls)] + + +class ClaimedSyncStream: + """Finalize an explicitly closed stream even before its first iteration.""" + + def __init__(self, call, stream, original): + self._call = call + self._stream = stream + self._original = original + self._started = False + self._closed = False + + def __iter__(self): + return self + + def __next__(self): + self._started = True + return next(self._stream) + + def send(self, value): + self._started = True + return self._stream.send(value) + + def throw(self, *args): + self._started = True + return self._stream.throw(*args) + + def close(self): + if self._closed: + return + self._closed = True + try: + close = getattr(self._original, "close", None) + if callable(close): + close() + finally: + try: + self._stream.close() + finally: + if not self._started: + run_safely(self._call.finish_stream_success, "", None) + run_safely(self._call.finalize) + + +class ClaimedAsyncStream: + """The async twin of ``process_sync_stream``: claimed, then finalized once. + + ``process_async_stream`` cannot itself be an async generator, because an + async generator's body -- ``mark_streaming()`` included -- only runs on the + caller's first ``__anext__``. A stream that is awaited and then closed or + dropped without being read would therefore keep the caller's context token + attached and its span recording. This iterator claims the stream when it is + created, so the caller's context is released before it resumes, and it + finalizes the span as soon as consumption stops, however it stops. + """ + + def __init__( + self, + call: "_LLMCall", + stream: AsyncGenerator[Any, None], + original: Any, + ) -> None: + self._call = call + self._stream = stream + self._original = original + self._started = False + self._closed = False + + def __aiter__(self) -> "ClaimedAsyncStream": + return self + + def __anext__(self) -> Any: + self._started = True + return self._stream.__anext__() + + def asend(self, value: Any) -> Any: + self._started = True + return self._stream.asend(value) + + def athrow(self, *args: Any, **kwargs: Any) -> Any: + self._started = True + return self._stream.athrow(*args, **kwargs) + + async def aclose(self) -> None: + """Stop the stream early; the span closes exactly once.""" + if self._closed: + return + self._closed = True + try: + close = getattr(self._original, "aclose", None) + if callable(close): + await close() + finally: + try: + await self._stream.aclose() + finally: + if not self._started: + run_safely(self._call.finish_stream_success, "", None) + run_safely(self._call.finalize) + + +def _apply_llm_plugins(func: Any) -> Any: + """Apply the framework's LLM plugin hook, resolved per call. + + The framework looks its plugin hook up through the trace module's globals, + so it is resolved here on every call too: an application that replaces it -- + or a test that spies on it -- sees the call. A missing or failing hook falls + back to the plain function, so telemetry can never swallow the call. + """ + try: + plugins = getattr(trace_module, "_llm_plugins", None) + return plugins(func) if plugins else func + except Exception: + logger.warning("Could not apply the LLM plugin hook", exc_info=True) + return func + + +class LLMMetrics: + """The agentUniverse LLM metric set, recorded by this package.""" + + def __init__(self, meter: Meter) -> None: + self._calls = meter.create_counter( + name="llm_calls_total", + description="Total number of LLM calls", + unit="1", + ) + self._errors = meter.create_counter( + name="llm_errors_total", + description="Total number of LLM errors", + unit="1", + ) + self._duration = meter.create_histogram( + name="llm_call_duration", + description="Duration of LLM calls in seconds", + unit="s", + ) + self._first_token = meter.create_histogram( + name="llm_first_token_duration", + description="Duration of LLM first token in seconds", + unit="s", + ) + self._total_tokens = meter.create_histogram( + name="llm_total_tokens", + description="Total token nums used in llm", + unit="1", + ) + self._prompt_tokens = meter.create_histogram( + name="llm_prompt_tokens", + description="Prompt token nums used in llm", + unit="1", + ) + self._completion_tokens = meter.create_histogram( + name="llm_completion_tokens", + description="Completion token nums used in llm", + unit="1", + ) + self._cached_tokens = meter.create_histogram( + name="llm_cached_tokens", + description="Cached token nums used in llm", + unit="1", + ) + self._reasoning_tokens = meter.create_histogram( + name="llm_reasoning_tokens", + description="Reasoning token nums used in llm", + unit="1", + ) + + def record_call(self, labels: Dict[str, Any]) -> None: + self._calls.add(1, labels) + + def record_duration(self, duration: float, labels: Dict[str, Any]) -> None: + self._duration.record(duration, labels) + + def record_first_token( + self, duration: float, labels: Dict[str, Any], streaming: bool + ) -> None: + self._first_token.record( + duration, {**labels, MetricLabels.LLM_STREAMING: streaming} + ) + + def record_error( + self, error: BaseException, duration: float, labels: Dict[str, Any] + ) -> None: + error_labels = { + **labels, + MetricLabels.LLM_STATUS: type(error).__name__, + } + self._errors.add(1, error_labels) + self._duration.record(duration, error_labels) + + def record_tokens(self, usage: TokenUsage, labels: Dict[str, Any]) -> None: + self._total_tokens.record(usage.total_tokens, labels) + self._completion_tokens.record(usage.completion_tokens, labels) + self._prompt_tokens.record(usage.prompt_tokens, labels) + self._cached_tokens.record(usage.cached_tokens, labels) + self._reasoning_tokens.record(usage.reasoning_tokens, labels) + + +class _LLMCall: + """Everything one wrapped LLM call needs, sync, async or streaming.""" + + def __init__( + self, + layer: "LLMLayer", + func: Any, + args: tuple, + kwargs: Dict[str, Any], + ) -> None: + self._layer = layer + #: Kept because the per-call overrides are read from the call's own + #: arguments: ``_get_llm_info`` only reports what the component is + #: configured with. + self.kwargs = dict(kwargs or {}) + ( + self.llm_instance, + self.source, + self.channel_name, + self.llm_input, + self.params, + self.caller_info, + ) = _get_llm_info(func, *args, **kwargs) + self.span_name = f"au.llm.{self.source}" + self.labels: Dict[str, Any] = { + MetricLabels.LLM_NAME: self.source, + **caller_labels(self.caller_info), + MetricLabels.LLM_STATUS: SUCCESS_STATUS, + } + self.layer_span: Optional[LayerSpan] = None + self.streaming = False + self.provider_usage: Optional[TokenUsage] = None + self.reasoning_chunks: list[str] = [] + self.stream_finish_reason: Optional[str] = None + self.finished = False + #: Only a node this call actually added may be popped again. + self.invocation_chain_pushed = False + self.invocation = LLMInvocation( + request_model=self._request_model(), + provider=self.channel_name or None, + attributes=agent_name_attributes(), + temperature=self._temperature(), + ) + + def _call_model_override(self) -> Optional[str]: + """The model this one call asks for, when it names one itself. + + Every framework LLM implementation pops ``model`` out of the call's own + arguments and falls back to its configured name + (``model=kwargs.pop("model", self.model_name)``), so a call that names a + model really does go to that model. Telemetry has to report the model the + call went to, not the one the component defaults to. + """ + for source in ( + self.kwargs, + self.llm_input.get("kwargs"), + self.llm_input, + ): + if not isinstance(source, dict): + continue + value = source.get("model") + if isinstance(value, str) and value: + return value + return None + + def _request_model(self) -> Optional[str]: + """The model the call asks for, for ``gen_ai.request.model``. + + The framework's ``LLM.model_name`` carries the provider model (``gpt-4`` + and friends); a component that does not configure one reports its own + name, which is what the framework's instrumentation uses as well. + ``au.llm.name`` keeps that component name in either case. + """ + override = self._call_model_override() + if override: + return override + channel = getattr(self.llm_instance, "_channel_instance", None) + model_name = ( + getattr(channel, "channel_model_name", None) + or getattr(self.llm_instance, "channel_model_name", None) + or getattr(self.llm_instance, "model_name", None) + ) + if isinstance(model_name, str) and model_name: + return model_name + return None + + def _temperature(self) -> Optional[float]: + """The configured temperature, ignoring the framework's -1 sentinel.""" + channel = ( + getattr(self.llm_instance, "_channel_instance", None) + or self.llm_instance + ) + config = getattr(channel, "channel_model_config", None) or {} + nested = self.llm_input.get("kwargs") or {} + for values in (self.kwargs, nested, config): + if "temperature" in values: + temperature = values["temperature"] + return ( + temperature + if isinstance(temperature, (int, float)) + and temperature != -1 + else None + ) + temperature = getattr(self.llm_instance, "temperature", None) + return ( + temperature + if isinstance(temperature, (int, float)) and temperature != -1 + else None + ) + + def _refresh_request(self) -> None: + # A channel may be lazily initialized inside the first business call. + channel = ( + getattr(self.llm_instance, "_channel_instance", None) + or self.llm_instance + ) + provider = getattr(channel, "channel_name", None) + self.invocation.provider = ( + provider if provider and provider != "default_channel" else None + ) + self.invocation.request_model = self._request_model() + tools = ( + self.kwargs.get("tools") + or (self.llm_input.get("kwargs") or {}).get("tools") + or [] + ) + self.invocation.tool_definitions = [ + FunctionToolDefinition( + name=tool["function"]["name"], + description=tool["function"].get("description"), + parameters=tool["function"].get("parameters"), + ) + for tool in tools + if isinstance(tool, dict) + and tool.get("type") == "function" + and isinstance(tool.get("function"), dict) + and isinstance(tool["function"].get("name"), str) + ] + self.invocation.temperature = self._temperature() + if self.invocation.temperature is not None: + self.params["temperature"] = self.invocation.temperature + if self.layer_span is not None: + self.layer_span.span.set_attribute( + LLMAttributes.LLM_PARAMS, llm_params_attribute(self.params) + ) + if self.invocation.provider: + self.layer_span.span.set_attribute( + LLMAttributes.CHANNEL_NAME, self.invocation.provider + ) + + def _collect_chunk_metadata(self, chunk: Any) -> None: + raw = getattr(chunk, "raw", None) + raw = raw if isinstance(raw, dict) else {} + reasoning = getattr(chunk, "reasoning_delta", None) or raw_message( + chunk + ).get("reasoning_content") + if isinstance(reasoning, str) and reasoning: + self.reasoning_chunks.append(reasoning) + finish = getattr(chunk, "finish_reason", None) or raw_choice( + chunk + ).get("finish_reason") + if isinstance(finish, str) and finish: + self.stream_finish_reason = finish + identifier = getattr(chunk, "response_id", None) or raw.get("id") + if isinstance(identifier, str) and identifier: + self.invocation.response_id = identifier + if isinstance(raw.get("model"), str): + self.invocation.response_model_name = raw["model"] + + # -- lifecycle ---------------------------------------------------- + + def start(self) -> None: + run_safely(Monitor.trace_llm_input, self.source, self.llm_input) + + self.layer_span = self._layer.start_span(self.span_name) + # The chain is keyed by the current trace id, so the span has to be + # current before this layer's node is added; children read it there. + if push_invocation_chain(self.source, "llm"): + self.invocation_chain_pushed = True + span = self.layer_span.span + # The shared handler finalizes this span on behalf of the invocation, so + # it has to know the span and the context token it may detach. The proxy + # keeps the framework-native span name while it does so. + self.invocation.span = NamePreservingSpan(span) + span.set_attribute(LLMAttributes.SPAN_KIND, SPAN_KIND_VALUE) + span.set_attribute(LLMAttributes.NAME, self.source) + span.set_attribute(LLMAttributes.CHANNEL_NAME, self.channel_name) + content = maybe_content(self.llm_input) + if content is not None: + span.set_attribute(LLMAttributes.INPUT, content) + span.set_attribute( + LLMAttributes.LLM_PARAMS, llm_params_attribute(self.params) + ) + set_caller_attributes(span, self.caller_info) + self.invocation.input_messages = self._input_messages() + self._refresh_request() + + run_safely(self._layer.metrics.record_call, self.labels) + + def _input_messages(self) -> list: + return input_messages_for(self.llm_input) + + def mark_streaming(self) -> None: + """Mark the span streaming and release the context for the caller. + + The streaming flag only rides on the first-token metric, exactly like + the framework's own instrumentation; the remaining metrics keep the + labels the call started with. + """ + self.streaming = True + self.layer_span.span.set_attribute(LLMAttributes.STREAMING, True) + self.invocation.context_token = self.layer_span.take_context_token() + run_safely(self._layer.handler.detach_llm_context, self.invocation) + + def _finish_first_token(self) -> None: + duration = self.layer_span.record_first_token() + run_safely( + self._layer.metrics.record_first_token, + duration, + self.labels, + True, + ) + self.layer_span.span.set_attribute( + LLMAttributes.FIRST_TOKEN_DURATION, duration + ) + + def _record_usage(self, usage: TokenUsage) -> None: + """Record usage on the metrics and accumulate it exactly once.""" + run_safely(self._layer.metrics.record_tokens, usage, self.labels) + run_safely(record_token_usage, usage, self.layer_span.span_id) + + def finish_success(self, result: LLMOutput) -> None: + if self.layer_span is None: + return + span = self.layer_span.span + span.set_attribute(LLMAttributes.STREAMING, False) + duration = self.layer_span.elapsed() + usage = self.provider_usage or usage_of(result) + raw = getattr(result, "raw", None) + self.invocation.response_id = getattr(result, "response_id", None) or ( + raw.get("id") if isinstance(raw, dict) else None + ) + if isinstance(raw, dict) and isinstance(raw.get("model"), str): + self.invocation.response_model_name = raw["model"] + + run_safely( + Monitor.trace_llm_invocation, + self.source, + self.llm_input, + result.text, + duration, + ) + run_safely( + Monitor().trace_llm_token_usage, + self.llm_instance, + self.llm_input, + result, + ) + + run_safely(self._layer.metrics.record_duration, duration, self.labels) + run_safely( + self._layer.metrics.record_first_token, + duration, + self.labels, + False, + ) + if usage: + self._record_usage(usage) + + span.set_attribute(LLMAttributes.FIRST_TOKEN_DURATION, duration) + span.set_attribute(LLMAttributes.DURATION, duration) + span.set_attribute(LLMAttributes.STATUS, SUCCESS_STATUS) + content = maybe_content_text(result.text) + if content is not None: + span.set_attribute(LLMAttributes.OUTPUT, content) + if usage: + set_usage_attributes(span, LLMAttributes, usage) + + self._set_invocation_success( + result.text, + usage, + getattr(result, "finish_reason", None) + or raw_choice(result).get("finish_reason"), + tool_calls_of(result), + getattr(result, "reasoning_text", None) + or raw_message(result).get("reasoning_content"), + ) + + def finish_stream_success( + self, + output: str, + usage: Optional[TokenUsage], + tool_calls: Optional[list] = None, + ) -> None: + """Finalize a consumed stream. Usage reaches the parent once. + + ``tool_calls`` carries the calls a streamed tool-planning round assembled: + that round has no text at all, and its calls are its output. + """ + if self.layer_span is None: + return + span = self.layer_span.span + duration = self.layer_span.elapsed() + usage = self.provider_usage or usage + + run_safely( + Monitor().trace_llm_invocation, + source=self.source, + llm_input=self.llm_input, + llm_output=output, + cost_time=duration, + ) + pseudo_result = LLMOutput(text=output, usage=usage) + run_safely( + Monitor().trace_llm_token_usage, + self.llm_instance, + self.llm_input, + pseudo_result, + ) + + run_safely(self._layer.metrics.record_duration, duration, self.labels) + if usage: + self._record_usage(usage) + + span.set_attribute(LLMAttributes.DURATION, duration) + span.set_attribute(LLMAttributes.STATUS, SUCCESS_STATUS) + content = maybe_content_text(output) + if content is not None: + span.set_attribute(LLMAttributes.OUTPUT, content) + if usage: + set_usage_attributes(span, LLMAttributes, usage) + + self._set_invocation_success( + output, + usage, + self.stream_finish_reason, + tool_calls, + "".join(self.reasoning_chunks), + ) + + def _set_invocation_success( + self, + output: str, + usage: Optional[TokenUsage], + finish_reason: Optional[str] = None, + tool_calls: Optional[list] = None, + reasoning: Optional[str] = None, + ) -> None: + """Hand the shared handler this call's output, calls included. + + A round that answers with tool calls carries no text; without the calls + its ``gen_ai.output.messages`` would be empty even though the round did + produce the tool plan. + """ + reason = finish_reason or ("tool_calls" if tool_calls else "stop") + self.invocation.output_messages = output_messages_for( + output, reason, tool_calls, reasoning + ) + self.invocation.finish_reasons = [reason] + if usage is not None: + self.invocation.input_tokens = usage.prompt_tokens + self.invocation.output_tokens = usage.completion_tokens + if usage.cached_in: + self.invocation.usage_cache_read_input_tokens = usage.cached_in + self._refresh_request() + self.layer_span.apply_timing(self.invocation) + + def finish_error(self, error: BaseException) -> None: + if self.layer_span is None: + return + span = self.layer_span.span + duration = self.layer_span.elapsed() + run_safely( + self._layer.metrics.record_error, error, duration, self.labels + ) + set_layer_error_attributes(span, LLMAttributes, error, duration) + set_error_status(span, error_status_text(error)) + + def finalize(self, error: Optional[BaseException] = None) -> None: + if self.finished: + return + self.finished = True + layer_span = self.layer_span + if layer_span is None: + return + + run_safely( + promote_token_usage_to_parent, + token_usage_for(layer_span.span_id), + layer_span.parent_span_id, + ) + + if self.invocation.context_token is None: + self.invocation.context_token = layer_span.take_context_token() + self.invocation.monotonic_end_s = monotonic_now() + if error is None: + run_safely(self._layer.handler.stop_llm, self.invocation) + else: + run_safely( + self._layer.handler.fail_llm, + self.invocation, + Error(message=error_status_text(error), type=type(error)), + ) + # Everything left over is closed by the handler's own cleanup rather + # than by hand: ``abandon_llm`` detaches whatever token is still pending + # and ends the span if it is still recording, and it is a no-op on an + # invocation ``stop_llm``/``fail_llm`` already finalized. The plugin + # therefore hands the token over before finalizing and owns neither + # step afterwards. + run_safely(self._layer.handler.abandon_llm, self.invocation) + if self.invocation_chain_pushed: + self.invocation_chain_pushed = False + pop_invocation_chain() + + # -- streaming ---------------------------------------------------- + + def process_sync_stream(self, result: Any) -> ClaimedSyncStream: + self.mark_streaming() + return ClaimedSyncStream(self, self._sync_stream(result), result) + + def _sync_stream(self, result: Any) -> Generator[Any, None, None]: + output: list[str] = [] + usage: Optional[TokenUsage] = None + tool_calls = ToolCallAccumulator() + error: Optional[BaseException] = None + first = True + try: + iterator = iter(result) + while True: + try: + with usage_call_context(self): + chunk = next(iterator) + except StopIteration: + break + if first: + first = False + run_safely(self._finish_first_token) + # The chunk is read here, before the caller receives it, so every + # read is guarded: a collector that fails must not turn a working + # stream into a failing one, and must not disturb the chunk the + # caller is waiting for -- it is yielded unchanged and in order. + text, chunk_usage = run_safely(chunk_record, chunk) or ( + "", + None, + ) + if text: + output.append(text) + if chunk_usage: + usage = chunk_usage + run_safely(tool_calls.feed, chunk) + run_safely(self._collect_chunk_metadata, chunk) + yield chunk + except GeneratorExit: + # The caller stopped consuming the stream early: the span still + # closes exactly once, with whatever the stream produced so far. + raise + except BaseException as exc: + error = exc + run_safely(self.finish_error, exc) + raise + finally: + if error is None: + run_safely( + self.finish_stream_success, + "".join(output), + usage, + tool_calls.result(), + ) + run_safely(self.finalize, error) + + def process_async_stream(self, result: Any) -> ClaimedAsyncStream: + """Claim the stream before the caller gets it, like the sync twin. + + ``mark_streaming`` releases the caller's context token, so it has to run + now rather than on the caller's first iteration; the returned iterator + finalizes the span whenever consumption stops. + """ + self.mark_streaming() + return ClaimedAsyncStream(self, self._async_stream(result), result) + + async def _async_stream(self, result: Any) -> AsyncGenerator[Any, None]: + output: list[str] = [] + usage: Optional[TokenUsage] = None + tool_calls = ToolCallAccumulator() + error: Optional[BaseException] = None + first = True + try: + iterator = result.__aiter__() + while True: + try: + with usage_call_context(self): + chunk = await iterator.__anext__() + except StopAsyncIteration: + break + if first: + first = False + run_safely(self._finish_first_token) + # Same rule as the sync twin: the collector is guarded, and the + # chunk the caller awaited is delivered unchanged and in order. + text, chunk_usage = run_safely(chunk_record, chunk) or ( + "", + None, + ) + if text: + output.append(text) + if chunk_usage: + usage = chunk_usage + run_safely(tool_calls.feed, chunk) + run_safely(self._collect_chunk_metadata, chunk) + yield chunk + except GeneratorExit: + # The caller stopped consuming the stream early: the span still + # closes exactly once, with whatever the stream produced so far. + raise + except BaseException as exc: + error = exc + run_safely(self.finish_error, exc) + raise + finally: + if error is None: + run_safely( + self.finish_stream_success, + "".join(output), + usage, + tool_calls.result(), + ) + run_safely(self.finalize, error) + + +class LLMLayer: + """The LLM wrapper pair plus the spans and metrics it owns.""" + + def __init__( + self, + tracer: Any, + handler: ExtendedTelemetryHandler, + meter: Meter, + ) -> None: + self._tracer = tracer + self.handler = handler + self.metrics = LLMMetrics(meter) + + def start_span(self, span_name: str) -> LayerSpan: + return LayerSpan(self._tracer, span_name, SpanKind.INTERNAL).attach() + + def wrap_sync(self, func: Any, *args: Any, **kwargs: Any) -> Any: + call = build_safely(_LLMCall, self, func, args, kwargs) + if call is None: + return _apply_llm_plugins(func)(*args, **kwargs) + run_safely(call.start) + try: + with usage_call_context(call): + result = _apply_llm_plugins(func)(*args, **kwargs) + except BaseException as exc: + run_safely(call.finish_error, exc) + run_safely(call.finalize, exc) + raise + if isinstance(result, LLMOutput): + run_safely(call.finish_success, result) + run_safely(call.finalize) + return result + if call.layer_span is None: + return result + stream = run_safely(call.process_sync_stream, result) + if stream is None: + run_safely(call.finalize) + return result + return stream + + async def wrap_async(self, func: Any, *args: Any, **kwargs: Any) -> Any: + call = build_safely(_LLMCall, self, func, args, kwargs) + if call is None: + return await _apply_llm_plugins(func)(*args, **kwargs) + run_safely(call.start) + try: + with usage_call_context(call): + result = await _apply_llm_plugins(func)(*args, **kwargs) + except BaseException as exc: + run_safely(call.finish_error, exc) + run_safely(call.finalize, exc) + raise + if isinstance(result, LLMOutput): + run_safely(call.finish_success, result) + run_safely(call.finalize) + return result + if call.layer_span is None: + return result + stream = run_safely(call.process_async_stream, result) + if stream is None: + run_safely(call.finalize) + return result + return stream diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_step.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_step.py new file mode 100644 index 000000000..cac901da7 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_step.py @@ -0,0 +1,340 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The ReAct STEP layer of the LoongSuite agentUniverse instrumentation. + +A tool-calling agentUniverse agent is a Reasoning-Acting loop: every round calls +the LLM, and a round whose answer carries tool calls is followed by the tools. +LoongSuite models one such iteration as its own ``STEP`` span +(``gen_ai.span.kind=STEP``, ``gen_ai.operation.name=react``, +``gen_ai.react.round`` and ``gen_ai.react.finish_reason``), which is what this +module creates. + +The interception point is the framework's own round hooks, +``Agent.on_before_llm_round``/``Agent.on_after_llm_round``: the tool-calling loop +calls them around every round, the async loop delegates to them, and no +framework class overrides them. They are patched on the class for the duration of +an install and restored verbatim on uninstrument -- the same transactional rule +the layer wrappers follow. + +Each step span is a child of the agent span, and its token usage bucket is +promoted to the agent when the step closes, so a round's LLM usage still reaches +the agent exactly once. + +No agentUniverse OTel class is imported or called here. +""" + +from __future__ import annotations + +import logging +from contextvars import ContextVar, Token +from typing import Any, Dict, Optional + +from agentuniverse.agent.agent import Agent + +from opentelemetry.util.genai.extended_handler import ExtendedTelemetryHandler +from opentelemetry.util.genai.extended_types import ReactStepInvocation +from opentelemetry.util.genai.types import Error + +from ._common import ( + AgentAttributes, + agent_name_attributes, + detach_context_token, + error_status_text, + promote_token_usage_to_parent, + reset_token_usage, + run_safely, + token_usage_for, +) + +logger = logging.getLogger(__name__) + +#: ``au.span.kind`` carrier value for a ReAct step, like the other layers. +STEP_SPAN_KIND_VALUE = "step" + +#: Used when the round answered without naming a reason of its own. +DEFAULT_FINISH_REASON = "stop" + +#: The reason a round that answered with tool calls gives, per the framework's +#: own ``finish_reason`` vocabulary. +TOOL_CALL_FINISH_REASON = "tool_calls" + + +class ReactStep: + """One Reasoning-Acting iteration, as a LoongSuite STEP span.""" + + def __init__( + self, layer: "ReactStepLayer", round_index: Optional[int] + ) -> None: + self._layer = layer + number: Optional[int] = None + if isinstance(round_index, int) and round_index >= 0: + # The framework counts rounds from zero, the convention from one. + number = round_index + 1 + self.invocation = ReactStepInvocation( + round=number, attributes=agent_name_attributes() + ) + self.span = None + self.finished = False + + @property + def started(self) -> bool: + return self.span is not None + + def start(self) -> None: + """Start the step span and attach it as the round's parent context.""" + run_safely(self._layer.handler.start_react_step, self.invocation) + self.span = self.invocation.span + if self.span is None: + # ``start_react_step`` attaches the context token and the span + # together; a token without a span would stay attached to this + # context for the rest of the process, so it is released here. + token, self.invocation.context_token = ( + self.invocation.context_token, + None, + ) + run_safely(detach_context_token, token) + return + run_safely( + self.span.set_attribute, + AgentAttributes.SPAN_KIND, + STEP_SPAN_KIND_VALUE, + ) + # The framework's own span bookkeeping seeds a fresh usage bucket for + # every span it starts; the shared handler does not, so the step gets its + # own bucket before its children report into it. + run_safely(reset_token_usage, self.span.context.span_id) + + def finish( + self, + error: Optional[BaseException] = None, + finish_reason: Optional[str] = None, + ) -> None: + """Close the step span exactly once, whatever ends the round.""" + if self.finished: + return + self.finished = True + span = self.span + if span is None: + return + + if finish_reason and not self.invocation.finish_reason: + self.invocation.finish_reason = finish_reason + + # What the round's children accumulated belongs to the agent span: the + # step is a span of its own, not a second place usage is counted. + parent = span.parent + run_safely( + promote_token_usage_to_parent, + token_usage_for(span.context.span_id), + parent.span_id if parent is not None else None, + ) + + if error is None: + run_safely(self._layer.handler.stop_react_step, self.invocation) + else: + run_safely( + self._layer.handler.fail_react_step, + self.invocation, + Error(message=error_status_text(error), type=type(error)), + ) + # Both handlers detach the context token and end the span. Ending is + # checked first because a span that is no longer recording is exactly the + # one they already closed; the token is theirs to release, and releasing + # it twice raises out of ``contextvars``. + if span.is_recording(): + run_safely(span.end) + + +class ReactStepTracker: + """Opens one STEP span per round of the agent call it belongs to.""" + + def __init__(self, layer: "ReactStepLayer") -> None: + self._layer = layer + self._open: Optional[ReactStep] = None + + def begin(self, round_index: Optional[int]) -> None: + if self._open is not None: + return + step = ReactStep(self._layer, round_index) + step.start() + if step.started: + self._open = step + + def end( + self, + llm_output: Any = None, + error: Optional[BaseException] = None, + ) -> None: + step, self._open = self._open, None + if step is None: + return + step.finish(error, self._finish_reason(llm_output)) + + @staticmethod + def _finish_reason(llm_output: Any) -> Optional[str]: + if llm_output is None: + return None + reason = getattr(llm_output, "finish_reason", None) + if isinstance(reason, str) and reason: + return reason + has_tool_calls = getattr(llm_output, "has_tool_calls", None) + if callable(has_tool_calls) and run_safely(has_tool_calls): + return TOOL_CALL_FINISH_REASON + return DEFAULT_FINISH_REASON + + def close(self, error: Optional[BaseException] = None) -> None: + """Close a step the loop left open, as an aborted round does.""" + self.end(None, error) + + +class ReactStepLayer: + """The step spans an agent call's rounds are recorded as.""" + + def __init__(self, handler: ExtendedTelemetryHandler) -> None: + self.handler = handler + + def tracker(self) -> ReactStepTracker: + return ReactStepTracker(self) + + +_current_tracker: ContextVar[Optional[ReactStepTracker]] = ContextVar( + "agentuniverse_react_step_tracker", default=None +) + + +def current_react_step_tracker() -> Optional[ReactStepTracker]: + """The tracker of the agent call that is running in this context.""" + return _current_tracker.get() + + +def push_react_step_tracker( + tracker: Optional[ReactStepTracker], +) -> Optional[Token]: + if tracker is None: + return None + try: + return _current_tracker.set(tracker) + except Exception: # pragma: no cover - defensive + return None + + +def pop_react_step_tracker(token: Optional[Token]) -> None: + if token is None: + return + try: + _current_tracker.reset(token) + except Exception: # pragma: no cover - defensive + logger.debug("Failed to reset the ReAct step tracker", exc_info=True) + + +#: The class attributes this package claims for the step spans. +REACT_STEP_HOOKS: tuple[str, ...] = ( + "on_before_llm_round", + "on_after_llm_round", +) + + +def install_react_step_hooks(layer: ReactStepLayer) -> Dict[str, Any]: + """Claim the framework's round hooks; return the originals it replaced. + + The hooks stay no-ops for the application: whatever the framework (or + anything that claimed them before) does around a round still happens, in the + same order. Only the step span is added. + """ + # Stable 0.0.19.1 uses a LangChain ReAct executor, without these newer + # native round hooks. Their absence must not disable Agent/LLM/Tool spans. + # A partially present hook pair is still an invalid install and rolls back. + if not any(hasattr(Agent, name) for name in REACT_STEP_HOOKS): + return {} + originals: Dict[str, Any] = { + name: getattr(Agent, name) for name in REACT_STEP_HOOKS + } + + def before( + self: Any, + context: Any, + round_index: int = 0, + *args: Any, + **kwargs: Any, + ) -> Any: + tracker = _current_tracker.get() + if tracker is not None: + run_safely(tracker.begin, round_index) + return originals["on_before_llm_round"]( + self, context, round_index=round_index, *args, **kwargs + ) + + def after( + self: Any, + context: Any, + llm_output: Any = None, + round_index: int = 0, + *args: Any, + **kwargs: Any, + ) -> Any: + try: + return originals["on_after_llm_round"]( + self, + context, + llm_output, + round_index=round_index, + *args, + **kwargs, + ) + finally: + tracker = _current_tracker.get() + if tracker is not None: + run_safely(tracker.end, llm_output) + + try: + Agent.on_before_llm_round = before + Agent.on_after_llm_round = after + except Exception: + logger.warning( + "Could not claim the agentUniverse round hooks; no ReAct STEP " + "spans will be recorded", + exc_info=True, + ) + remove_react_step_hooks(originals) + raise + return { + name: (originals[name], wrapper) + for name, wrapper in ( + ("on_before_llm_round", before), + ("on_after_llm_round", after), + ) + } + + +def remove_react_step_hooks(originals: Dict[str, Any]) -> None: + """Restore the round hooks this package replaced.""" + for name in REACT_STEP_HOOKS: + saved = originals.get(name) + if saved is None: + continue + # Installation rollback also accepts the initial bare snapshot. + original, wrapper = ( + saved if isinstance(saved, tuple) else (saved, None) + ) + if wrapper is not None and getattr(Agent, name, None) is not wrapper: + continue + try: + setattr(Agent, name, original) + except Exception: # pragma: no cover - defensive + logger.warning( + "Could not restore the agentUniverse round hook %s", + name, + exc_info=True, + ) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_tool.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_tool.py new file mode 100644 index 000000000..350d7ff34 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_tool.py @@ -0,0 +1,383 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The tool layer of the LoongSuite agentUniverse instrumentation. + +This module claims the ``_tool_wrapper_sync``/``_tool_wrapper_async`` extension +points the ``@trace_tool`` decorator resolves and builds the ``au.tool.*`` span, +metrics and token bookkeeping itself. ``gen_ai.*`` attributes and metrics on the +same span come from the shared ``ExtendedTelemetryHandler``. + +No agentUniverse OTel class is imported or called here. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any, Dict, Optional + +from agentuniverse.agent.memory.conversation_memory.conversation_memory_module import ( # noqa: E501 + ConversationMemoryModule, +) +from agentuniverse.base.annotation.trace import _get_tool_info +from agentuniverse.llm.llm_output import TokenUsage + +from opentelemetry.metrics import Meter +from opentelemetry.trace import SpanKind +from opentelemetry.util.genai.extended_handler import ExtendedTelemetryHandler +from opentelemetry.util.genai.extended_types import ExecuteToolInvocation +from opentelemetry.util.genai.types import Error + +from ._common import ( + SUCCESS_STATUS, + LayerSpan, + MetricLabels, + NamePreservingSpan, + ToolAttributes, + agent_name_attributes, + build_safely, + caller_labels, + detach_context_token, + error_status_text, + maybe_content, + monotonic_now, + pop_invocation_chain, + promote_token_usage_to_parent, + push_invocation_chain, + run_safely, + set_caller_attributes, + set_error_status, + set_layer_error_attributes, + set_usage_attributes, + token_usage_for, +) +from ._hooks import current_tool_call_id + +logger = logging.getLogger(__name__) + +SPAN_KIND_VALUE = "tool" + +#: The framework exposes tools as plain callables to the model. +TOOL_TYPE = "function" + + +def _memory_module() -> Any: + """The framework's conversation memory, or ``None`` when unavailable.""" + try: + return ConversationMemoryModule() + except Exception: # pragma: no cover - application config not ready + logger.debug("ConversationMemoryModule unavailable", exc_info=True) + return None + + +class ToolMetrics: + """The agentUniverse tool metric set, recorded by this package.""" + + def __init__(self, meter: Meter) -> None: + self._calls = meter.create_counter( + name="tool_calls_total", + description="Total number of Tool calls", + unit="1", + ) + self._errors = meter.create_counter( + name="tool_errors_total", + description="Total number of Tool errors", + unit="1", + ) + self._duration = meter.create_histogram( + name="tool_call_duration", + description="Duration of Tool calls in seconds", + unit="s", + ) + self._total_tokens = meter.create_histogram( + name="tool_total_tokens", + description="Total token nums used in tool", + unit="1", + ) + self._prompt_tokens = meter.create_histogram( + name="tool_prompt_tokens", + description="Prompt token nums used in tool", + unit="1", + ) + self._completion_tokens = meter.create_histogram( + name="tool_completion_tokens", + description="Completion token nums used in tool", + unit="1", + ) + self._cached_tokens = meter.create_histogram( + name="tool_cached_tokens", + description="Cached token nums used in tool", + unit="1", + ) + self._reasoning_tokens = meter.create_histogram( + name="tool_reasoning_tokens", + description="Reasoning token nums used in tool", + unit="1", + ) + + def record_call(self, labels: Dict[str, Any]) -> None: + self._calls.add(1, labels) + + def record_duration(self, duration: float, labels: Dict[str, Any]) -> None: + self._duration.record(duration, labels) + + def record_error( + self, error: BaseException, duration: float, labels: Dict[str, Any] + ) -> None: + error_labels = { + **labels, + MetricLabels.TOOL_STATUS: type(error).__name__, + } + self._errors.add(1, error_labels) + self._duration.record(duration, error_labels) + + def record_tokens(self, usage: TokenUsage, labels: Dict[str, Any]) -> None: + self._total_tokens.record(usage.total_tokens, labels) + self._completion_tokens.record(usage.completion_tokens, labels) + self._prompt_tokens.record(usage.prompt_tokens, labels) + self._cached_tokens.record(usage.cached_tokens, labels) + self._reasoning_tokens.record(usage.reasoning_tokens, labels) + + +class _ToolCall: + """Everything one wrapped tool call needs, sync or async.""" + + def __init__( + self, + layer: "ToolLayer", + func: Any, + args: tuple, + kwargs: Dict[str, Any], + ) -> None: + self._layer = layer + ( + self.tool_instance, + self.source, + self.tool_input, + self.caller_info, + self.pair_id, + ) = _get_tool_info(func, *args, **kwargs) + self.span_name = f"au.tool.{self.source}" + self.labels: Dict[str, Any] = { + MetricLabels.TOOL_NAME: self.source, + **caller_labels(self.caller_info), + MetricLabels.TOOL_STATUS: SUCCESS_STATUS, + } + self.layer_span: Optional[LayerSpan] = None + self.finished = False + #: Only a node this call actually added may be popped again. + self.invocation_chain_pushed = False + self.invocation = ExecuteToolInvocation( + tool_name=self.source, + tool_type=TOOL_TYPE, + tool_call_id=current_tool_call_id(), + attributes=agent_name_attributes(), + ) + + # -- lifecycle ---------------------------------------------------- + + def start(self) -> None: + self.layer_span = self._layer.start_span(self.span_name) + # The chain is keyed by the current trace id, so the span has to be + # current before this layer's node is added; children read it there. + if push_invocation_chain(self.source, "tool"): + self.invocation_chain_pushed = True + span = self.layer_span.span + # The shared handler finalizes this span on behalf of the invocation, so + # it has to know the span and the context token it may detach. The proxy + # keeps the framework-native span name while it does so. + self.invocation.span = NamePreservingSpan(span) + span.set_attribute(ToolAttributes.SPAN_KIND, SPAN_KIND_VALUE) + span.set_attribute(ToolAttributes.NAME, self.source) + content = maybe_content(self.tool_input) + if content is not None: + span.set_attribute(ToolAttributes.INPUT, content) + set_caller_attributes(span, self.caller_info) + span.set_attribute(ToolAttributes.PAIR_ID, self.pair_id) + + run_safely(self._layer.metrics.record_call, self.labels) + memory = _memory_module() + if memory is not None: + run_safely( + memory.add_tool_input_info, + self.caller_info, + self.source, + self.tool_input, + self.pair_id, + ) + + def finish_success(self, result: Any) -> None: + if self.layer_span is None: + return + span = self.layer_span.span + duration = self.layer_span.elapsed() + usage = token_usage_for(self.layer_span.span_id) + + memory = _memory_module() + if memory is not None: + run_safely( + memory.add_tool_output_info, + self.caller_info, + self.source, + params=result, + pair_id=self.pair_id, + ) + run_safely(self._layer.metrics.record_duration, duration, self.labels) + run_safely(self._layer.metrics.record_tokens, usage, self.labels) + + span.set_attribute(ToolAttributes.DURATION, duration) + span.set_attribute(ToolAttributes.STATUS, SUCCESS_STATUS) + content = maybe_content(result) + if content is not None: + span.set_attribute(ToolAttributes.OUTPUT, content) + set_usage_attributes(span, ToolAttributes, usage) + + arguments = self.tool_input.get("kwargs", self.tool_input) + positional = self.tool_input.get("args") + # Stable native ReAct calls Tool.langchain_run with one JSON string; + # it is the actual tool input, whereas its kwargs contain no arguments. + if ( + not arguments + and isinstance(positional, (tuple, list)) + and len(positional) == 1 + and isinstance(positional[0], str) + ): + try: + decoded = json.loads(positional[0]) + except (TypeError, ValueError): + decoded = None + if isinstance(decoded, dict): + arguments = decoded + self.invocation.tool_call_arguments = { + key: value + for key, value in arguments.items() + if key != "agent_context" + } + self.invocation.tool_call_result = result + + def finish_error(self, error: BaseException) -> None: + if self.layer_span is None: + return + span = self.layer_span.span + duration = self.layer_span.elapsed() + usage = token_usage_for(self.layer_span.span_id) + run_safely( + self._layer.metrics.record_error, error, duration, self.labels + ) + run_safely( + self._layer.metrics.record_tokens, + usage, + self.labels, + ) + set_layer_error_attributes(span, ToolAttributes, error, duration) + # The framework records the (zero) usage of a failed tool call too. + set_usage_attributes(span, ToolAttributes, usage) + set_error_status(span, error_status_text(error)) + + def finalize(self, error: Optional[BaseException] = None) -> None: + if self.finished: + return + self.finished = True + layer_span = self.layer_span + if layer_span is None: + return + + run_safely( + promote_token_usage_to_parent, + token_usage_for(layer_span.span_id), + layer_span.parent_span_id, + ) + if self.invocation_chain_pushed: + self.invocation_chain_pushed = False + pop_invocation_chain() + self.invocation.context_token = layer_span.take_context_token() + self.invocation.monotonic_end_s = monotonic_now() + if error is None: + run_safely(self._layer.handler.stop_execute_tool, self.invocation) + else: + run_safely( + self._layer.handler.fail_execute_tool, + self.invocation, + Error(message=error_status_text(error), type=type(error)), + ) + self._release_context_token(layer_span) + layer_span.end() + + def _release_context_token(self, layer_span: LayerSpan) -> None: + """Detach the span context unless the handler already did. + + The shared handler detaches the token and ends the span itself whenever + it finalizes an invocation; detaching it a second time would raise. If + the handler did not finish the span, the token is still ours to drop. + """ + if not layer_span.span.is_recording(): + self.invocation.context_token = None + leftover = self.invocation.context_token + if leftover is not None: + self.invocation.context_token = None + run_safely(detach_context_token, leftover) + + +class ToolLayer: + """The tool wrapper pair plus the spans and metrics it owns.""" + + def __init__( + self, + tracer: Any, + handler: ExtendedTelemetryHandler, + meter: Meter, + ) -> None: + self._tracer = tracer + self.handler = handler + self.metrics = ToolMetrics(meter) + + def start_span(self, span_name: str) -> LayerSpan: + return LayerSpan(self._tracer, span_name, SpanKind.INTERNAL).attach() + + def wrap_sync(self, func: Any, *args: Any, **kwargs: Any) -> Any: + call = build_safely(_ToolCall, self, func, args, kwargs) + if call is None: + return func(*args, **kwargs) + error: Optional[BaseException] = None + try: + run_safely(call.start) + result = func(*args, **kwargs) + except BaseException as exc: + error = exc + run_safely(call.finish_error, exc) + raise + else: + run_safely(call.finish_success, result) + return result + finally: + run_safely(call.finalize, error) + + async def wrap_async(self, func: Any, *args: Any, **kwargs: Any) -> Any: + call = build_safely(_ToolCall, self, func, args, kwargs) + if call is None: + return await func(*args, **kwargs) + error: Optional[BaseException] = None + try: + run_safely(call.start) + result = await func(*args, **kwargs) + except BaseException as exc: + error = exc + run_safely(call.finish_error, exc) + raise + else: + run_safely(call.finish_success, result) + return result + finally: + run_safely(call.finalize, error) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_usage.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_usage.py new file mode 100644 index 000000000..6ff090eef --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/_usage.py @@ -0,0 +1,118 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read provider totals without changing the framework's business objects.""" + +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any, Optional + +from agentuniverse.llm.llm_output import TokenUsage + +_current_call: ContextVar[Any] = ContextVar( + "agentuniverse_llm_call", default=None +) + + +@contextmanager +def usage_call_context(call): + """Only held while calling the framework or advancing its iterator.""" + token = _current_call.set(call) + try: + yield + finally: + _current_call.reset(token) + + +def _count(value: Any) -> bool: + return ( + isinstance(value, int) and not isinstance(value, bool) and value >= 0 + ) + + +def provider_usage( + payload: Any, provider: str = "openai" +) -> Optional[TokenUsage]: + if not isinstance(payload, dict): + return None + if provider == "anthropic": + # Anthropic input_tokens excludes cache reads and writes. + values = [ + payload.get(key, 0) + for key in ( + "input_tokens", + "output_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + ) + ] + if not all(_count(value) for value in values): + return None + if "input_tokens" not in payload or "output_tokens" not in payload: + return None + inputs, outputs, reads, writes = values + return TokenUsage( + text_in=inputs + writes, cached_in=reads, text_out=outputs + ) + inputs = payload.get("prompt_tokens", payload.get("input_tokens")) + outputs = payload.get("completion_tokens", payload.get("output_tokens")) + if not _count(inputs) or not _count(outputs): + return None + details_in = ( + payload.get("prompt_tokens_details") + or payload.get("input_tokens_details") + or {} + ) + details_out = ( + payload.get("completion_tokens_details") + or payload.get("output_tokens_details") + or {} + ) + if not isinstance(details_in, dict) or not isinstance(details_out, dict): + return None + fields = { + "image_in": details_in.get("image_tokens", 0), + "audio_in": details_in.get("audio_tokens", 0), + "cached_in": details_in.get("cached_tokens", 0), + "image_out": details_out.get("image_tokens", 0), + "audio_out": details_out.get("audio_tokens", 0), + "cached_out": details_out.get("cached_tokens", 0), + "reasoning_out": details_out.get("reasoning_tokens", 0), + } + fields = { + key: 0 if value is None else value for key, value in fields.items() + } + if not all(_count(value) for value in fields.values()): + return None + # These details are subsets of provider totals. TokenUsage's properties + # sum disjoint buckets, so only the remainder belongs in its text buckets. + fields["text_in"] = inputs - sum( + fields[key] for key in ("image_in", "audio_in", "cached_in") + ) + fields["text_out"] = outputs - sum( + fields[key] + for key in ("image_out", "audio_out", "cached_out", "reasoning_out") + ) + if fields["text_in"] < 0 or fields["text_out"] < 0: + return None + return TokenUsage(**fields) + + +def capture_provider_usage(payload: Any, provider: str) -> None: + call = _current_call.get() + if call is not None: + usage = provider_usage(payload, provider) + if usage is not None: + # Usage events are cumulative snapshots, including usage-only tails. + call.provider_usage = usage diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/package.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/package.py new file mode 100644 index 000000000..59aeb5e76 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/package.py @@ -0,0 +1,16 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +_instruments = ("agentUniverse >= 0.0.19.1",) +_supports_metrics = True diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/version.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/version.py new file mode 100644 index 000000000..3a48c29e5 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/src/opentelemetry/instrumentation/agentuniverse/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.10.0.dev" diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/test-requirements.txt b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/test-requirements.txt new file mode 100644 index 000000000..da65bab58 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/test-requirements.txt @@ -0,0 +1,18 @@ +pytest +pytest-asyncio +pytest-forked==1.6.0 +setuptools +# agentUniverse 0.0.19.1 hard-pins grpcio==1.63.0, which conflicts with the +# pinned core otlp-proto-grpc (>=1.63.2). 0.1.0a2 drops the grpcio pin and +# requires opentelemetry-api/sdk>=1.30.0, both satisfied by the core checkout. +# The instrumentation code is compatible with both lines; this pin selects the +# installable prerelease for the formal CI install path (no --no-deps override). +agentUniverse == 0.1.0a2 + +-e opentelemetry-instrumentation +-e util/opentelemetry-util-genai +-e instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse +# Chroma's transitive FastAPI instrumentation must match this source checkout. +-e instrumentation/opentelemetry-instrumentation-fastapi +-e instrumentation/opentelemetry-instrumentation-asgi +-e util/opentelemetry-util-http \ No newline at end of file diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/__init__.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/__init__.py new file mode 100644 index 000000000..b0a6f4284 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/baseline.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/baseline.py new file mode 100644 index 000000000..a931b8d6d --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/baseline.py @@ -0,0 +1,273 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers that read telemetry and compare it with the framework's own. + +The tests run the same fixture twice -- once under agentUniverse's own Agent, +LLM and Tool instrumentors and once under this package -- and compare the two +runs. Values that cannot be equal across processes (durations, generated ids) +are compared by type and sign instead of by value; everything else is compared +literally. +""" + +import json +import re +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, Optional, Sequence + +AU_PREFIX = "au." +GEN_AI_PREFIX = "gen_ai." + +#: Attributes whose value differs between two runs of the same call. +DYNAMIC_SUFFIXES = ( + "duration", + "pair_id", + "detail_tokens", + "time_to_first_token", +) +DYNAMIC_EXACT = ( + "au.agent.first_token.duration", + "au.llm.first_token.duration", +) + +#: ``au.*.error.message`` embeds a traceback, whose line numbers are stable for +#: a given call but whose absolute paths are not interesting; compare the tail. +_LAST_LINE = re.compile(r"\n\s*") + + +def is_dynamic_attribute(key: str) -> bool: + if key in DYNAMIC_EXACT: + return True + return any(key.endswith(f".{suffix}") for suffix in DYNAMIC_SUFFIXES) + + +#: ``au.agent.output`` is the one ``au.*`` carrier with an intentional +#: difference: the framework's own instrumentation publishes whatever +#: ``customized_execute`` returned -- the caller's whole agent input with the +#: answer under ``output`` -- while this package publishes the answer itself (see +#: the README). Comparing the two means comparing the answers. +AGENT_OUTPUT_ATTRIBUTE = "au.agent.output" +_ENVELOPE_ANSWER_KEY = "output" + + +def agent_answer(value: Any) -> Any: + """The agent's answer inside an ``au.agent.output`` carrier value.""" + if not isinstance(value, str): + return value + try: + decoded = json.loads(value) + except Exception: + return value + if isinstance(decoded, dict) and _ENVELOPE_ANSWER_KEY in decoded: + return decoded[_ENVELOPE_ANSWER_KEY] + return decoded + + +def normalise_attribute_value(key: str, value: Any) -> Any: + """Reduce one attribute to the part two runs can be expected to share.""" + if key == AGENT_OUTPUT_ATTRIBUTE: + return agent_answer(value) + if not is_dynamic_attribute(key): + if ( + isinstance(value, str) + and "Traceback (most recent call last)" in value + ): + return _LAST_LINE.split(value.strip())[-1] + return value + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return f"number>={0}" if value >= 0 else "number<0" + if isinstance(value, str): + return "non-empty-string" if value else "empty-string" + return f"type:{type(value).__name__}" + + +def framework_version() -> tuple: + """The installed agentUniverse version, as comparable numbers.""" + from importlib.metadata import version + + text = version("agentuniverse") + return tuple(int(part) for part in re.findall(r"\d+", text)[:3]) + + +def native_stream_usage_doubling() -> bool: + """Whether the framework's own instrumentation doubles streamed usage. + + 0.0.19.1 counts a streamed response onto its parent twice: its stream loop + calls ``add_current_token_usage`` and its ``_finalize_streaming_result`` + calls it again for the same usage. From 0.1.0a2 on that second call is gone + (the source says so in a comment: it would double count). The tests assert + the value this build of the framework actually produces instead of assuming + one of the two behaviours. + """ + return framework_version() < (0, 1, 0) + + +@dataclass(frozen=True) +class DataPoint: + """One metric data point, flattened for assertions.""" + + value: Any + count: int + labels: Dict[str, Any] + + +def _point_value(point: Any) -> Any: + if hasattr(point, "value") and not hasattr(point, "sum"): + return point.value + if hasattr(point, "sum"): + return point.sum + return getattr(point, "value", None) + + +def collect_metrics(reader: Any) -> Dict[str, List[DataPoint]]: + """Every data point the reader holds, grouped by metric name.""" + metrics: Dict[str, List[DataPoint]] = {} + data = reader.get_metrics_data() + if data is None: + return metrics + for resource_metrics in data.resource_metrics: + for scope_metrics in resource_metrics.scope_metrics: + for metric in scope_metrics.metrics: + points = metrics.setdefault(metric.name, []) + for point in metric.data.data_points: + points.append( + DataPoint( + value=_point_value(point), + count=getattr(point, "count", 1), + labels=dict(point.attributes), + ) + ) + return metrics + + +def points_for( + metrics: Dict[str, List[DataPoint]], name: str, **labels: Any +) -> List[DataPoint]: + """The points of one metric whose labels contain ``labels``.""" + return [ + point + for point in metrics.get(name, []) + if all(point.labels.get(key) == value for key, value in labels.items()) + ] + + +def counter_value( + metrics: Dict[str, List[DataPoint]], name: str, **labels: Any +) -> float: + """The value of a counter, summing the points that match ``labels``.""" + return sum(point.value for point in points_for(metrics, name, **labels)) + + +def histogram_values( + metrics: Dict[str, List[DataPoint]], name: str, **labels: Any +) -> List[Any]: + return [point.value for point in points_for(metrics, name, **labels)] + + +def metric_names(metrics: Dict[str, List[DataPoint]]) -> set: + return set(metrics) + + +@dataclass(frozen=True) +class SpanRecord: + """The parts of a finished span the matrix compares.""" + + name: str + kind: str + status: str + description: str + parent: Optional[str] + attributes: Dict[str, Any] + + def prefix(self, prefix: str) -> Dict[str, Any]: + return { + key: value + for key, value in self.attributes.items() + if key.startswith(prefix) + } + + @property + def au(self) -> Dict[str, Any]: + return self.prefix(AU_PREFIX) + + @property + def gen_ai(self) -> Dict[str, Any]: + return self.prefix(GEN_AI_PREFIX) + + +def snapshot(spans: Iterable[Any]) -> List[SpanRecord]: + """Turn finished spans into comparable records, children after parents.""" + spans = list(spans) + names = {span.context.span_id: span.name for span in spans} + return [ + SpanRecord( + name=span.name, + kind=span.kind.name, + status=span.status.status_code.name, + description=span.status.description or "", + parent=names.get(span.parent.span_id) if span.parent else None, + attributes=dict(span.attributes or {}), + ) + for span in spans + ] + + +def find(records: Sequence[SpanRecord], name: str) -> SpanRecord: + matches = [record for record in records if record.name == name] + assert len(matches) == 1, f"expected one {name!r} span, got {len(matches)}" + return matches[0] + + +def names(records: Sequence[SpanRecord]) -> List[str]: + return [record.name for record in records] + + +def assert_same_au_contract( + native: SpanRecord, loongsuite: SpanRecord, ignore: Sequence[str] = () +) -> None: + """The ``au.*`` contract of both implementations must line up. + + The attribute key set has to match exactly, and every value that cannot + differ between two identical runs has to match too. Only the keys listed in + ``ignore`` are allowed to differ in value. + """ + native_au = native.au + loongsuite_au = loongsuite.au + assert set(native_au) == set(loongsuite_au), ( + f"{native.name}: au.* key sets differ; " + f"only native: {sorted(set(native_au) - set(loongsuite_au))}, " + f"only LoongSuite: {sorted(set(loongsuite_au) - set(native_au))}" + ) + for key in sorted(native_au): + expected = normalise_attribute_value(key, native_au[key]) + actual = normalise_attribute_value(key, loongsuite_au[key]) + if key in ignore: + assert actual is not None or expected is None + continue + assert expected == actual, ( + f"{native.name}: {key} differs ({native_au[key]!r} vs {loongsuite_au[key]!r})" + ) + + +def assert_same_span_shape( + native: SpanRecord, loongsuite: SpanRecord, ignore_au: Sequence[str] = () +) -> None: + assert native.kind == loongsuite.kind, native.name + assert native.status == loongsuite.status, native.name + assert native.parent == loongsuite.parent, ( + f"{native.name}: parent differs ({native.parent!r} vs {loongsuite.parent!r})" + ) + assert_same_au_contract(native, loongsuite, ignore=ignore_au) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/conftest.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/conftest.py new file mode 100644 index 000000000..2f6ead306 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/conftest.py @@ -0,0 +1,619 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fixtures for the agentUniverse instrumentation tests. + +These tests run against a real ``agentUniverse`` install, with no stand-in for +the framework: real ``Agent``/``Tool`` subclasses and a real ``@trace_llm`` +method drive the extension points this package claims, and the framework's own +instrumentors drive the same calls in the baseline runs the results are compared +with. + +The agents, LLM and tool below are real framework objects, so the wrappers, +``au.*`` attributes, metrics and token aggregation all run for real. +""" + +import os +import sys +import time +from pathlib import Path + +# The shared GenAI util only exposes its capture switch in experimental mode, +# and the instrumentation reads that switch at instrument() time, so opt in +# before any test runs. +os.environ.setdefault( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" +) + +# Import the package under test from its source tree, as the other +# instrumentation test suites do. +_SRC = Path(__file__).resolve().parents[1] / "src" +if _SRC.is_dir() and str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + +from collections.abc import Callable, Iterator # noqa: E402 +from typing import Any # noqa: E402 + +import pytest # noqa: E402 +from agentuniverse.agent.action.tool.tool import Tool # noqa: E402 +from agentuniverse.agent.agent import Agent # noqa: E402 +from agentuniverse.agent.agent_model import AgentModel # noqa: E402 +from agentuniverse.base.annotation import trace as trace_module # noqa: E402 +from agentuniverse.base.annotation.trace import trace_llm # noqa: E402 +from agentuniverse.base.config.application_configer.app_configer import ( # noqa: E402 + AppConfiger, +) +from agentuniverse.base.config.application_configer.application_config_manager import ( # noqa: E402 + ApplicationConfigManager, +) +from agentuniverse.base.tracing.otel.instrumentation.agent.agent_instrumentor import ( # noqa: E402 + AgentInstrumentor, +) +from agentuniverse.base.tracing.otel.instrumentation.llm.llm_instrumentor import ( # noqa: E402 + LLMInstrumentor, +) +from agentuniverse.base.tracing.otel.instrumentation.tool.tool_instrumentor import ( # noqa: E402 + ToolInstrumentor, +) +from agentuniverse.llm.llm_output import LLMOutput, TokenUsage # noqa: E402 + +from opentelemetry.instrumentation.agentuniverse import ( # noqa: E402 + AgentUniverseInstrumentor, +) +from opentelemetry.sdk.metrics import MeterProvider # noqa: E402 +from opentelemetry.sdk.metrics.export import ( # noqa: E402 + InMemoryMetricReader, +) +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 + InMemorySpanExporter, +) + +from . import baseline # noqa: E402 + +# The native instrumentor builds a ``ConversationMemoryModule`` at call time, +# which reads the application config manager on construction. Seed it before +# any agent runs, or the native wrapper fails on ``app_configer is None``. +ApplicationConfigManager().app_configer = AppConfiger() + +# Each native instrumentor lives by swapping a pair of module-level globals in +# ``agentuniverse.base.annotation.trace``; the pair is how the instrumentor that +# owns a layer is recognised. +NATIVE_LAYER_GLOBALS: dict[type, tuple[str, str]] = { + AgentInstrumentor: ("_agent_wrapper_sync", "_agent_wrapper_async"), + LLMInstrumentor: ("_llm_wrapper_sync", "_llm_wrapper_async"), + ToolInstrumentor: ("_tool_wrapper_sync", "_tool_wrapper_async"), +} +NATIVE_WRAPPER_GLOBALS: tuple[str, ...] = tuple( + name for pair in NATIVE_LAYER_GLOBALS.values() for name in pair +) + +# The three layers the bridge covers, as short keys the tests can loop over. +AGENT_LAYER = "agent" +LLM_LAYER = "llm" +TOOL_LAYER = "tool" + + +# --------------------------------------------------------------------------- +# Real agents +# --------------------------------------------------------------------------- + + +class StubAgent(Agent): + """A real agentUniverse Agent whose body needs no planner or LLM.""" + + def input_keys(self) -> list[str]: + return ["input"] + + def output_keys(self) -> list[str]: + return ["output"] + + def parse_input(self, input_object: Any, agent_input: dict) -> dict: + return agent_input + + def parse_result(self, agent_result: dict) -> dict: + return agent_result + + def execute(self, input_object: Any, agent_input: dict) -> dict: + return {"output": f"echo:{input_object.get_data('input')}"} + + async def async_execute( + self, input_object: Any, agent_input: dict + ) -> dict: + return {"output": f"echo:{input_object.get_data('input')}"} + + +class FailingAgent(StubAgent): + """``StubAgent`` whose body raises, to exercise the error path.""" + + def execute(self, input_object: Any, agent_input: dict) -> dict: + raise RuntimeError("agent exploded") + + +class StreamingAgent(StubAgent): + """``StubAgent`` that pushes one token onto ``output_stream``. + + The native wrapper swaps the caller's queue for one that records the first + put, so putting an item here is what exercises the first-token hook -- on + this path the native wrapper does not fall back to the end-of-call timing. + """ + + def execute(self, input_object: Any, agent_input: dict) -> dict: + stream = input_object.get_data("output_stream") + if stream is not None: + # ``time.time()`` on Windows is only accurate to about 15 ms, so + # put the token late enough that the first-token duration is + # unambiguously positive. + time.sleep(0.05) + stream.put("first-token") + return {"output": "streamed"} + + +class RichAgent(StubAgent): + """An agent that really calls an LLM and a tool, as a real one would.""" + + def execute(self, input_object: Any, agent_input: dict) -> dict: + prompt = input_object.get_data("input") + llm_result = StubLLM().call(prompt=prompt) + tool_result = StubTool().run(query=prompt) + return {"output": f"{llm_result.text}|{tool_result}"} + + +class AsyncRichAgent(StubAgent): + """``RichAgent`` over the async wrappers of all three layers.""" + + async def async_execute( + self, input_object: Any, agent_input: dict + ) -> dict: + prompt = input_object.get_data("input") + llm_result = await AsyncLLM().call(prompt=prompt) + tool_result = await StubTool().async_run(query=prompt) + return {"output": f"{llm_result.text}|{tool_result}"} + + +class LLMAgent(StubAgent): + """An agent whose only child call is a single LLM call.""" + + def execute(self, input_object: Any, agent_input: dict) -> dict: + return { + "output": StubLLM() + .call(prompt=input_object.get_data("input")) + .text + } + + +class StreamingLLMAgent(StubAgent): + """An agent whose only child call is a streaming LLM call.""" + + def execute(self, input_object: Any, agent_input: dict) -> dict: + stream = StreamingLLM().call(prompt=input_object.get_data("input")) + return {"output": "".join(chunk.text for chunk in stream)} + + +class ToolAgent(StubAgent): + """An agent whose only child call is a single tool call.""" + + def execute(self, input_object: Any, agent_input: dict) -> dict: + return {"output": StubTool().run(query=input_object.get_data("input"))} + + +# --------------------------------------------------------------------------- +# Real LLMs +# --------------------------------------------------------------------------- + + +class StubLLM: + """A minimal faithful LLM: a real ``@trace_llm`` method is all the native + ``LLMInstrumentor`` needs. + + ``_get_llm_info`` reads ``self.name``, ``self.channel_name`` and + ``self.channel_model_config``, and ``_llm_plugins`` walks an (empty by + default) plugin set from the application config, so no LLM base class is + required to exercise the real native LLM wrapper. + """ + + name = "stub_llm" + channel_name = "test_channel" + channel_model_config: dict = {"temperature": 0.7} + + @trace_llm + def call(self, prompt: str, **kwargs) -> LLMOutput: + return LLMOutput( + text=f"llm:{prompt}", + usage=TokenUsage(text_in=3, text_out=5), + finish_reason="stop", + ) + + +class AsyncLLM(StubLLM): + """The async twin, taken by the native async LLM wrapper.""" + + name = "async_llm" + + @trace_llm + async def call(self, prompt: str, **kwargs) -> LLMOutput: + return LLMOutput( + text=f"async:{prompt}", + usage=TokenUsage(text_in=2, text_out=4), + finish_reason="stop", + ) + + +class StreamingLLM(StubLLM): + """A ``@trace_llm`` generator, which the native wrapper treats as a stream. + + The sleep makes the first-token timing unambiguously positive, and the + usage rides on the last chunk, which is where the native + ``StreamingResultProcessor`` looks for it. + """ + + name = "stream_llm" + + @trace_llm + def call(self, prompt: str, **kwargs): + time.sleep(0.005) + yield LLMOutput(text="part-1") + yield LLMOutput(text="part-2", usage=TokenUsage(text_in=4, text_out=6)) + + +class FailingLLM(StubLLM): + """An LLM call that raises, to exercise the native LLM error path.""" + + name = "failing_llm" + + @trace_llm + def call(self, prompt: str, **kwargs) -> LLMOutput: + raise RuntimeError("llm exploded") + + +class MessagesLLM(StubLLM): + """An LLM called with a chat ``messages`` list instead of a prompt.""" + + name = "messages_llm" + + @trace_llm + def call(self, messages: list, **kwargs) -> LLMOutput: + return LLMOutput( + text="answered", + usage=TokenUsage(text_in=1, text_out=2), + finish_reason="stop", + ) + + +# --------------------------------------------------------------------------- +# Real tools +# --------------------------------------------------------------------------- + + +class StubTool(Tool): + """A real agentUniverse ``Tool``; ``run``/``async_run`` are native seams.""" + + name: str = "stub_tool" + input_keys: list = ["query"] + + def execute(self, query: str) -> str: + return f"tool-output:{query}" + + +class FailingTool(StubTool): + """A tool whose body raises, to exercise the native tool error path.""" + + name: str = "failing_tool" + + def execute(self, query: str) -> str: + raise RuntimeError("tool exploded") + + +def build_agent( + name: str = "test_agent", agent_cls: type = StubAgent +) -> Agent: + """Build an agent the native instrumentor will name ``name``.""" + agent = agent_cls() + agent.agent_model = AgentModel(info={"name": name}) + return agent + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def span_exporter() -> Iterator[InMemorySpanExporter]: + exporter = InMemorySpanExporter() + yield exporter + exporter.clear() + + +@pytest.fixture +def tracer_provider( + span_exporter: InMemorySpanExporter, +) -> Iterator[TracerProvider]: + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + yield provider + provider.shutdown() + + +@pytest.fixture +def metric_reader() -> Iterator[InMemoryMetricReader]: + yield InMemoryMetricReader() + + +@pytest.fixture +def meter_provider( + metric_reader: InMemoryMetricReader, +) -> Iterator[MeterProvider]: + """A meter provider the native instrumentors can record their metrics into. + + The native instrumentors read the provider from ``instrument(...)``, so the + metrics tests hand them this one instead of the global no-op provider. + """ + provider = MeterProvider(metric_readers=[metric_reader]) + yield provider + provider.shutdown() + + +@pytest.fixture +def make_agent() -> Callable[..., Agent]: + return build_agent + + +# --------------------------------------------------------------------------- +# Harnesses: one run under each instrumentation setup +# --------------------------------------------------------------------------- + + +class Harness: + """One instrumentation setup and the telemetry it recorded.""" + + def __init__( + self, + span_exporter: InMemorySpanExporter, + metric_reader: InMemoryMetricReader, + close: Callable[[], None], + ) -> None: + self.span_exporter = span_exporter + self.metric_reader = metric_reader + self._close = close + + def close(self) -> None: + self._close() + + # -- what the run produced --------------------------------------- + + def spans(self) -> list: + return list(self.span_exporter.get_finished_spans()) + + def records(self) -> list: + return baseline.snapshot(self.spans()) + + def record(self, name: str) -> Any: + return baseline.find(self.records(), name) + + def metrics(self) -> dict: + return baseline.collect_metrics(self.metric_reader) + + def clear(self) -> None: + self.span_exporter.clear() + + +def _providers() -> tuple: + """A span exporter, tracer provider, metric reader and meter provider.""" + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[reader]) + return exporter, provider, reader, meter_provider + + +def enable_native_instrumentors( + provider: TracerProvider, meter_provider: MeterProvider +) -> list: + """Turn on the framework's own three instrumentors for one run. + + A live instrumentor is reused rather than rebuilt: constructing a native + instrumentor while one is active resets the wrapper originals it saved, and + a later ``uninstrument()`` would then blank the trace module globals. + """ + enabled = [] + for native_cls in NATIVE_LAYER_GLOBALS: + instrumentor = active_native_instrumentor(native_cls) or native_cls() + instrumentor.instrument( + tracer_provider=provider, meter_provider=meter_provider + ) + enabled.append(instrumentor) + return enabled + + +def _close_providers(*providers: Any) -> Callable[[], None]: + def close() -> None: + for provider in providers: + provider.shutdown() + + return close + + +@pytest.fixture +def native_harness() -> Iterator[Harness]: + """A run under the framework's own Agent, LLM and Tool instrumentors.""" + exporter, provider, reader, meter_provider = _providers() + enable_native_instrumentors(provider, meter_provider) + harness = Harness( + exporter, reader, _close_providers(provider, meter_provider) + ) + yield harness + harness.close() + + +@pytest.fixture +def loongsuite_harness() -> Iterator[Harness]: + """A run under this package's own wrappers.""" + instrumentor = AgentUniverseInstrumentor() + exporter, provider, reader, meter_provider = _providers() + instrumentor.instrument( + tracer_provider=provider, meter_provider=meter_provider + ) + harness = Harness( + exporter, reader, _close_providers(provider, meter_provider) + ) + yield harness + instrumentor.uninstrument() + harness.close() + + +@pytest.fixture +def both_harness() -> Iterator[Harness]: + """A run under the framework's instrumentors *and* this package. + + The framework's instrumentors go first, exactly as an application that + already uses ``TelemetryManager`` would install them, and this package then + takes the extension points over for the duration of the run. + """ + exporter, provider, reader, meter_provider = _providers() + enable_native_instrumentors(provider, meter_provider) + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument( + tracer_provider=provider, meter_provider=meter_provider + ) + harness = Harness( + exporter, reader, _close_providers(provider, meter_provider) + ) + yield harness + instrumentor.uninstrument() + harness.close() + + +class Runs: + """One workload recorded under each instrumentation setup.""" + + def __init__( + self, + native_spans: Any, + native_metrics: dict, + our_spans: Any, + our_metrics: dict, + ) -> None: + self.native_records = baseline.snapshot(native_spans) + self.native_metrics = native_metrics + self.ours_records = baseline.snapshot(our_spans) + self.ours_metrics = our_metrics + + +@pytest.fixture +def compare_runs() -> Iterator[Callable[[Callable[[], None]], Runs]]: + """Run one workload twice: first under the framework, then under this one. + + The two setups cannot hold the extension points at the same time -- this + package takes them over on purpose -- so each phase gets its own providers + and runs the workload itself. That is what makes the two runs comparable: + same workload, same configuration, one setup each. + """ + + def compare(workload: Callable[[], None]) -> Runs: + exporter, provider, reader, meter_provider = _providers() + natives = enable_native_instrumentors(provider, meter_provider) + try: + workload() + finally: + native_spans = exporter.get_finished_spans() + native_metrics = baseline.collect_metrics(reader) + for native in natives: + native.uninstrument() + provider.shutdown() + meter_provider.shutdown() + + exporter, provider, reader, meter_provider = _providers() + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument( + tracer_provider=provider, meter_provider=meter_provider + ) + try: + workload() + finally: + our_spans = exporter.get_finished_spans() + our_metrics = baseline.collect_metrics(reader) + instrumentor.uninstrument() + provider.shutdown() + meter_provider.shutdown() + + return Runs(native_spans, native_metrics, our_spans, our_metrics) + + yield compare + + +@pytest.fixture +def session_exporter() -> Iterator[Any]: + """A span exporter the session subprocess tests can read back.""" + yield InMemorySpanExporter() + + +@pytest.fixture +def configured_providers( + session_exporter: InMemorySpanExporter, +) -> Iterator[tuple]: + """Tracer and meter providers a test can hand to any instrumentor.""" + _, provider, reader, meter_provider = _providers() + provider.add_span_processor(SimpleSpanProcessor(session_exporter)) + yield provider, meter_provider, reader + provider.shutdown() + meter_provider.shutdown() + + +def active_native_instrumentor(native_cls: type = AgentInstrumentor) -> Any: + """The live native instrumentor for one layer, without constructing one. + + The native classes are ``BaseInstrumentor`` singletons whose ``__init__`` + resets the wrapper originals they saved, so calling ``AgentInstrumentor()`` + -- or its LLM/Tool twin -- while one is already instrumented would make a + later ``uninstrument()`` blank the trace-module globals and break every + call. Reach the live instance through the globals instead. + """ + wrapper = getattr(trace_module, NATIVE_LAYER_GLOBALS[native_cls][0], None) + owner = getattr(wrapper, "__self__", None) + return owner if isinstance(owner, native_cls) else None + + +@pytest.fixture(autouse=True) +def isolate_instrumentation() -> Iterator[None]: + """Keep the trace globals and the instrumentor singletons pristine. + + Every instrumentor here is a ``BaseInstrumentor`` singleton and each one + lives by swapping module-level globals in + ``agentuniverse.base.annotation.trace``. A test that leaves any of them + behind would silently change the next test's result, so force every layer + back to its pre-test state. + """ + instrumentor = AgentUniverseInstrumentor() + saved = { + name: getattr(trace_module, name) for name in NATIVE_WRAPPER_GLOBALS + } + + yield + + # Re-read the active native instrumentors at teardown: a test may have + # enabled one that was not active when it started. + instrumentors = [instrumentor] + [ + active_native_instrumentor(native_cls) + for native_cls in NATIVE_LAYER_GLOBALS + ] + for active in instrumentors: + if active is not None and active.__dict__.get( + "_is_instrumented_by_opentelemetry" + ): + active.uninstrument() + for name, value in saved.items(): + setattr(trace_module, name, value) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/runtime_app.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/runtime_app.py new file mode 100644 index 000000000..fa4939554 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/runtime_app.py @@ -0,0 +1,281 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Stable native ReAct/AgentExecutor; only the HTTP provider is scripted.""" + +import asyncio +import importlib.metadata as _metadata +import threading +import time +from collections import Counter +from queue import Queue +from typing import Any, ClassVar + +import httpx +from agentuniverse.agent.action.tool.tool import Tool +from agentuniverse.agent.action.tool.tool_manager import ToolManager +from agentuniverse.agent.agent_model import AgentModel +from agentuniverse.agent.template.react_agent_template import ( + ReActAgentTemplate, +) +from agentuniverse.llm.llm import LLM +from agentuniverse.llm.llm_channel.llm_channel import LLMChannel +from agentuniverse.llm.llm_manager import LLMManager +from agentuniverse.prompt.prompt import Prompt +from openai import AsyncOpenAI, OpenAI + +from opentelemetry import context + + +def _framework_version(): + raw = _metadata.version("agentUniverse") + parts = raw.split(".") + return (int(parts[0]), int(parts[1])) + + +IS_ALPHA = _framework_version() >= (0, 1) + +from .runtime_provider import MODEL, answer, prompt +from .runtime_provider import Provider as StructuredProvider + +if IS_ALPHA: + Provider = StructuredProvider +else: + from .stable_runtime_provider import Provider + + +class ChannelLLM(LLM): + def _call(self, *args, **kwargs): + raise AssertionError("The real LLM channel should handle the call") + + async def _acall(self, *args, **kwargs): + raise AssertionError("The real LLM channel should handle the call") + + def get_num_tokens(self, text): + return len(text.split()) + + +class InventoryTool(Tool): + workload: ClassVar[Any] = None + + def execute(self, **kwargs): + work = self.workload + with work.lock: + work.active += 1 + work.max_active = max(work.max_active, work.active) + try: + time.sleep(0.008) + value = ( + 7 + if self.name == "stock_lookup" + else 12 + if self.name == "unit_price" + else kwargs["a"] * kwargs["b"] + ) + work.tool_calls.append( + {"name": self.name, "args": kwargs, "result": value} + ) + return str(value) + finally: + with work.lock: + work.active -= 1 + + +class InventoryAgent(ReActAgentTemplate): + """Configure resources; execute and tool dispatch remain framework-owned.""" + + configured_llm: Any = None + configured_prompt: Any = None + + def process_memory(self, agent_input, **kwargs): + return None + + def process_llm(self, **kwargs): + return self.configured_llm + + def process_prompt(self, agent_input, **kwargs): + return self.configured_prompt + + +class Workload: + def __init__(self, reasoning=True): + InventoryTool.workload = self + self.backend = Provider() + self.reasoning = reasoning + self.tool_calls = [] + self.active = 0 + self.max_active = 0 + self.lock = threading.Lock() + self.names = [] + for name, fields in ( + ("stock_lookup", ["sku"]), + ("unit_price", ["sku"]), + ("multiply", ["a", "b"]), + ): + tool = InventoryTool( + name=name, + description="Synthetic inventory tool; supply JSON arguments", + input_keys=["worker", "turn"] + fields, + ) + ToolManager().register(tool.get_instance_code(), tool) + self.names.append(name) + + def agent(self, label, streaming): + channel = LLMChannel( + channel_name="openai", + channel_model_name=MODEL, + channel_api_key="synthetic", + channel_api_base="http://provider.invalid/v1", + ext_params={ + "reasoning_effort": "high" if self.reasoning else "none", + "stream_options": {"include_usage": True}, + }, + ) + channel.channel_model_config = dict( + model=MODEL, + streaming=streaming, + temperature=0, + max_tokens=512, + request_timeout=30, + max_retries=0, + ) + channel.client = OpenAI( + api_key="synthetic", + base_url="http://provider.invalid/v1", + max_retries=0, + http_client=httpx.Client( + transport=httpx.MockTransport(self.backend.respond) + ), + ) + channel.async_client = AsyncOpenAI( + api_key="synthetic", + base_url="http://provider.invalid/v1", + max_retries=0, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(self.backend.arespond) + ), + ) + llm = ChannelLLM( + name="inventory_llm", model_name=MODEL, streaming=streaming + ) + llm._channel_instance = channel + llm._max_context_length = 8192 + # The stable SDK's Agent.invoke_llm resolves the LLM by name through + # LLMManager; a bare process_llm override is never consulted. Register + # the channel-backed instance so judge_stream sees a real LLM. + llm_code = llm.get_instance_code() + LLMManager().register(llm_code, llm) + self._llm_code = llm_code + agent = InventoryAgent() + profile = { + "llm_model": dict( + name="inventory_llm", + model_name=MODEL, + temperature=0, + streaming=streaming, + ) + } + if IS_ALPHA: + # 0.1.0a2 builds the user message from profile['instruction'] + # rendered with input_dict; without it messages come back empty. + profile["instruction"] = "{input}" + agent.agent_model = AgentModel( + info={"name": "inventory_" + label}, + profile=profile, + ) + agent.agent_model.action = {"tool": self.names} + # SDK 0.0.19.1 InventoryAgent does not accept a max_iterations field; + # leave the ReAct loop at its SDK default instead of setting a custom one. + agent.configured_llm = llm + agent.configured_prompt = Prompt( + prompt_template=( + "Use the inventory tools: {tools}\nNames: {tool_names}\n" + "History: {chat_history}\nQuestion: {input}\n{agent_scratchpad}" + ), + input_variables=[ + "tools", + "tool_names", + "chat_history", + "input", + "agent_scratchpad", + ], + ) + return agent + + async def run(self, tracer_provider): + records = [] + + async def worker(label, streaming, synchronous=False): + agent = self.agent(label, streaming) + history = "" + with tracer_provider.get_tracer("app").start_as_current_span( + "au-session." + label + ): + before = context.get_current() + for turn in (1, 2): + kwargs = dict( + input=prompt(label, turn), + session_id=label, + chat_history=history, + output_stream=Queue(), + ) + output = ( + agent.run(**kwargs) + if synchronous + else await agent.async_run(**kwargs) + ) + text = output.get_data("output") + assert text == answer(label, turn) + assert context.get_current() is before + history += f"Human: {kwargs['input']}\nAI: {text}\n" + records.append( + dict( + worker=label, + turn=turn, + streaming=streaming, + text=text, + ) + ) + # On alpha the registered LLM is shared via LLMManager and each + # agent gets a copied channel; closing the shared original here + # would poison later workers. MockTransport clients need no close. + if not IS_ALPHA: + agent.configured_llm._channel_instance.client.close() + await ( + agent.configured_llm._channel_instance.async_client.close() + ) + + await worker("auSync", False, True) + await asyncio.gather( + worker("auStreamA", True), worker("auStreamB", True) + ) + # Drop the registered channel-backed LLM so it cannot leak into other + # tests; MockTransport clients hold no real sockets. + code = getattr(self, "_llm_code", None) + if code and code in LLMManager()._instance_obj_map: + LLMManager().unregister(code) + counts = Counter( + (item["args"]["worker"], item["args"]["turn"], item["name"]) + for item in self.tool_calls + ) + for label in ("auSync", "auStreamA", "auStreamB"): + for turn in (1, 2): + for name in ("stock_lookup", "unit_price", "multiply"): + assert counts[label, turn, name] == 1 + return dict( + self.backend.verify(), + scenarios=records, + tool_calls=self.tool_calls, + max_active_tools=self.max_active, + ) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/runtime_provider.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/runtime_provider.py new file mode 100644 index 000000000..5bbed265c --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/runtime_provider.py @@ -0,0 +1,305 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Deterministic OpenAI HTTP/SSE boundary, not an Agent-loop replacement. + +Actual OpenAI SDK builds requests/parses JSON or SSE; actual framework decides +when to invoke tools and calls the model again. All text is synthetic. +""" + +import asyncio +import json +import re + +import httpx + +MODEL = "synthetic-reasoning-model" +USAGE = { + "prompt_tokens": 20, + "completion_tokens": 10, + "total_tokens": 30, + "prompt_tokens_details": {"cached_tokens": 5}, + "completion_tokens_details": {"reasoning_tokens": 3}, +} + + +def prompt(worker, turn): + return f"worker={worker} turn={turn} Check stock and unit price for SKU-A, then price two units." + + +def answer(worker, turn): + return f"[{worker}] turn{turn}: stock=7 total=24" + + +def content_text(content): + if isinstance(content, str): + return content + return "".join( + p.get("text", "") for p in (content or []) if isinstance(p, dict) + ) + + +class AsyncBytes(httpx.AsyncByteStream): + def __init__(self, chunks): + self.chunks = chunks + + async def __aiter__(self): + for x in self.chunks: + await asyncio.sleep(0.002) + yield x + + +class Provider: + def __init__(self): + self.requests = [] + self.responses = [] + + @staticmethod + def _is_alpha(): + import importlib.metadata + + raw = importlib.metadata.version("agentUniverse") + parts = raw.split(".") + return (int(parts[0]), int(parts[1])) >= (0, 1) + + def validate_body(self, body): + IS_ALPHA = self._is_alpha() + messages = body["messages"] + index = max(i for i, m in enumerate(messages) if m["role"] == "user") + match = re.search( + r"worker=([A-Za-z0-9_-]+) turn=(\d+)", + content_text(messages[index].get("content")), + ) + worker, turn = match.group(1), int(match.group(2)) + plans = { + tc["id"]: tc + for m in messages[index + 1 :] + for tc in m.get("tool_calls", []) + } + for message in messages[index + 1 :]: + if message["role"] != "tool": + continue + tc = plans[message["tool_call_id"]] + args = json.loads(tc["function"]["arguments"]) + assert args["worker"] == worker and args["turn"] == turn + expected = { + "stock_lookup": "7", + "unit_price": "12", + "multiply": "24", + }[tc["function"]["name"]] + assert content_text(message.get("content")).strip() == expected, ( + message + ) + if turn == 2 and not IS_ALPHA: + assert any( + m["role"] == "assistant" + and answer(worker, 1) in content_text(m.get("content")) + for m in messages[:index] + ), (worker, "prior conversation was not retained") + assert body.get("reasoning_effort") in ("high", "none") + + def respond(self, req): + body = json.loads(req.content) + self.validate_body(body) + messages = body["messages"] + index = max(i for i, m in enumerate(messages) if m["role"] == "user") + text = content_text(messages[index].get("content")) + match = re.search(r"worker=([A-Za-z0-9_-]+) turn=(\d+)", text) + assert match, text + worker, turn = match.group(1), int(match.group(2)) + tail = messages[index + 1 :] + tool_results = [m for m in tail if m["role"] == "tool"] + n = len(tool_results) + if n == 0: + tools = [ + ( + "stock_lookup", + {"worker": worker, "turn": turn, "sku": "SKU-A"}, + ), + ( + "unit_price", + {"worker": worker, "turn": turn, "sku": "SKU-A"}, + ), + ] + round_index = 0 + elif n == 2: + tools = [ + ("multiply", {"worker": worker, "turn": turn, "a": 2, "b": 12}) + ] + round_index = 1 + elif n == 3: + tools = [] + round_index = 2 + else: + raise AssertionError(f"Unexpected actual tool result count: {n}") + ids = [ + f"{worker}-t{turn}-r{round_index}-{i}" for i in range(len(tools)) + ] + calls = [ + { + "id": ids[i], + "type": "function", + "function": {"name": name, "arguments": json.dumps(args)}, + } + for i, (name, args) in enumerate(tools) + ] + final_text = None if tools else answer(worker, turn) + reasoning = f"synthetic-reasoning:{worker}:t{turn}:r{round_index}" + message = { + "role": "assistant", + "content": final_text, + "reasoning_content": reasoning, + } + if calls: + message["tool_calls"] = calls + finish = "tool_calls" if tools else "stop" + self.requests.append( + { + "worker": worker, + "turn": turn, + "round": round_index, + "request": body, + "received_tool_ids": [ + m.get("tool_call_id") for m in tool_results + ], + } + ) + self.responses.append( + { + "worker": worker, + "turn": turn, + "round": round_index, + "reasoning_content": reasoning, + "tool_calls": calls, + "text": final_text, + "usage": USAGE, + } + ) + base = { + "id": f"synthetic-{worker}-{turn}-{round_index}", + "created": 1, + "model": MODEL, + } + if not body.get("stream"): + return httpx.Response( + 200, + json={ + **base, + "object": "chat.completion", + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": finish, + } + ], + "usage": USAGE, + }, + ) + events = [] + + def emit(delta, finish_reason=None): + events.append( + { + **base, + "object": "chat.completion.chunk", + "choices": [ + { + "index": 0, + "delta": delta, + "finish_reason": finish_reason, + } + ], + } + ) + + emit({"role": "assistant"}) + for part in [reasoning[:12], reasoning[12:]]: + emit({"reasoning_content": part}) + for i, call in enumerate(calls): + args = call["function"]["arguments"] + emit( + { + "tool_calls": [ + { + "index": i, + "id": call["id"], + "type": "function", + "function": { + "name": call["function"]["name"], + "arguments": "", + }, + } + ] + } + ) + for part in [args[:9], args[9:]]: + emit( + { + "tool_calls": [ + {"index": i, "function": {"arguments": part}} + ] + } + ) + if final_text: + for part in [final_text[:12], final_text[12:]]: + emit({"content": part}) + emit({}, finish) + events.append( + { + **base, + "object": "chat.completion.chunk", + "choices": [], + "usage": USAGE, + } + ) + lines = [ + ("data: " + json.dumps(e) + "\n\n").encode() for e in events + ] + [b"data: [DONE]\n\n"] + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=b"".join(lines), + ) + + async def arespond(self, req): + await asyncio.sleep(0.003) + response = self.respond(req) + if response.headers.get("content-type", "").startswith( + "text/event-stream" + ): + return httpx.Response( + 200, + headers=response.headers, + stream=AsyncBytes(response.content.splitlines(keepends=True)), + ) + return response + + def verify(self): + for item in self.requests: + self.validate_body(item["request"]) + groups = {} + for item in self.requests: + groups.setdefault((item["worker"], item["turn"]), []).append( + item["round"] + ) + for key, rounds in groups.items(): + assert rounds == [0, 1, 2], (key, rounds) + return { + "request_count": len(self.requests), + "turn_count": len(groups), + "reasoning_enabled": True, + "reasoning_response_count": len(self.responses), + "provider_usage_per_request": USAGE, + } diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/session_probe_child.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/session_probe_child.py new file mode 100644 index 000000000..ad5940229 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/session_probe_child.py @@ -0,0 +1,166 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Session propagation probe, run as its own process. + +``TelemetryManager.init_from_config`` may only run once per process and it takes +over the process-wide tracer provider and text map, so the session test runs it +here and reports what it observed as JSON on stdout. The parent test asserts on +that JSON. + +Everything in this probe is real: ``TelemetryManager`` and the default +``SessionSpanProcessor``/``AUSessionPropagator`` it installs, a real ``Agent`` +subclass and a real ``@trace_llm`` method. Only the span exporter is this +probe's own, so the spans can be read back. +""" + +import json +import os +import sys +from pathlib import Path +from typing import Any + +_HERE = Path(__file__).resolve().parent +if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) +_SRC = _HERE.parent / "src" +if _SRC.is_dir() and str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + +os.environ.setdefault( + "OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental" +) + +import session_probe_support # noqa: E402 +from agentuniverse.agent.agent import Agent # noqa: E402 +from agentuniverse.agent.agent_model import AgentModel # noqa: E402 +from agentuniverse.base.annotation.trace import trace_llm # noqa: E402 +from agentuniverse.base.config.application_configer.app_configer import ( # noqa: E402 + AppConfiger, +) +from agentuniverse.base.config.application_configer.application_config_manager import ( # noqa: E402 + ApplicationConfigManager, +) +from agentuniverse.base.tracing.au_trace_manager import ( + AuTraceManager, # noqa: E402 +) +from agentuniverse.base.tracing.otel.telemetry_manager import ( # noqa: E402 + TelemetryManager, +) +from agentuniverse.llm.llm_output import LLMOutput, TokenUsage # noqa: E402 + +from opentelemetry import propagate, trace # noqa: E402 + +SESSION = "session-abc123" +CARRIER_SESSION = "session-from-carrier" + + +class ProbeLLM: + """A real ``@trace_llm`` method, which is all a call needs.""" + + name = "session_llm" + channel_name = "test_channel" + + @trace_llm + def call(self, prompt: str, **kwargs: Any) -> LLMOutput: + return LLMOutput( + text=f"llm:{prompt}", + usage=TokenUsage(text_in=1, text_out=2), + finish_reason="stop", + ) + + +class ProbeAgent(Agent): + """A real agent whose body is one real LLM call.""" + + def input_keys(self) -> list: + return ["input"] + + def output_keys(self) -> list: + return ["output"] + + def parse_input(self, input_object: Any, agent_input: dict) -> dict: + return agent_input + + def parse_result(self, agent_result: dict) -> dict: + return agent_result + + def execute(self, input_object: Any, agent_input: dict) -> dict: + output = ProbeLLM().call(prompt=input_object.get_data("input")) + return {"output": output.text} + + +def main() -> None: + ApplicationConfigManager().app_configer = AppConfiger() + + TelemetryManager().init_from_config( + { + "service_name": "agentuniverse-session-probe", + "processors": [ + { + "class": ( + "opentelemetry.sdk.trace.export.SimpleSpanProcessor" + ), + "exporter": { + "class": "session_probe_support.RecordingExporter" + }, + } + ], + "propagators": [], + "metric_readers": [], + "instrumentations": [], + } + ) + + from opentelemetry.instrumentation.agentuniverse import ( + AgentUniverseInstrumentor, + ) + + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument() + + AuTraceManager().set_session_id(SESSION) + agent = ProbeAgent() + agent.agent_model = AgentModel(info={"name": "session_agent"}) + output = agent.run(input="hello") + + carrier: dict = {} + propagate.get_global_textmap().inject(carrier) + propagate.get_global_textmap().extract({"AU-SessionId": CARRIER_SESSION}) + + report = { + "session": SESSION, + "carrier_session": CARRIER_SESSION, + "output": output.get_data("output"), + "tracer_provider": type(trace.get_tracer_provider()).__name__, + "spans": [ + { + "name": span.name, + "attributes": { + key: value + for key, value in (span.attributes or {}).items() + if key.startswith(("au.", "gen_ai.")) + }, + } + for span in session_probe_support.SPANS + ], + "carrier": carrier, + "session_after_extract": AuTraceManager().get_session_id(), + } + instrumentor.uninstrument() + print("PROBE_JSON=" + json.dumps(report)) + + +if __name__ == "__main__": + main() diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/session_probe_support.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/session_probe_support.py new file mode 100644 index 000000000..e2409ea1e --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/session_probe_support.py @@ -0,0 +1,40 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""State the session probe subprocess shares with its TelemetryManager config. + +``TelemetryManager`` builds its span processor from an import path, so the +exporter has to live in a module that is *imported*. Defined in the probe script +itself, it would belong to ``__main__`` there while ``TelemetryManager`` +imported a second copy, and the spans would be collected into a list no test can +see. +""" + +from typing import Any + +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult + +#: Every span the exporter was handed, oldest first. +SPANS: list = [] + + +class RecordingExporter(SpanExporter): + """Append every exported span to :data:`SPANS`.""" + + def export(self, spans: Any) -> Any: + SPANS.extend(spans) + return SpanExportResult.SUCCESS + + def shutdown(self) -> None: + return None diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/stable_runtime_provider.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/stable_runtime_provider.py new file mode 100644 index 000000000..f2c5b9df5 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/stable_runtime_provider.py @@ -0,0 +1,172 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Synthetic OpenAI JSON/SSE for the stable framework's text ReAct protocol.""" + +import json +import re + +import httpx + +from .runtime_provider import MODEL, USAGE, AsyncBytes, answer + + +class Provider: + def __init__(self): + self.requests = [] + self.responses = [] + + def respond(self, request): + body = json.loads(request.content) + text = body["messages"][-1]["content"] + current = text.rsplit("Question: ", 1)[-1] + match = re.match(r"worker=([A-Za-z0-9_-]+) turn=(\d+)", current) + assert match is not None, current + worker, turn = match.group(1), int(match.group(2)) + index = current.count("Observation: ") + assert index <= 3 + assert body["model"] == MODEL + assert body["temperature"] == 0 + effort = body.get("reasoning_effort") + assert effort in ("high", "none") + if turn == 2: + assert answer(worker, 1) in text + tools = ["stock_lookup", "unit_price", "multiply"] + arguments = dict(worker=worker, turn=turn) + if index < 3: + if index < 2: + arguments["sku"] = "SKU-A" + else: + arguments.update(a=2, b=12) + content = ( + "Thought: use the next inventory tool.\nAction: " + + tools[index] + + "\nAction Input: " + + json.dumps(arguments) + ) + else: + assert "Observation: 7" in current + assert "Observation: 12" in current + assert "Observation: 24" in current + content = "Thought: inventory checked.\nFinal Answer: " + answer( + worker, turn + ) + reasoning = ( + f"Synthetic public reasoning for {worker}/{turn}/{index}." + if effort == "high" + else "" + ) + usage = json.loads(json.dumps(USAGE)) + if not reasoning: + usage["completion_tokens_details"]["reasoning_tokens"] = 0 + identifier = f"synthetic-{worker}-t{turn}-r{index}" + self.requests.append(body) + self.responses.append( + dict( + worker=worker, + turn=turn, + round=index, + id=identifier, + content=content, + reasoning=reasoning, + usage=usage, + tool=tools[index] if index < 3 else None, + arguments=arguments if index < 3 else None, + ) + ) + base = dict(id=identifier, model=MODEL, created=1) + if not body["stream"]: + return httpx.Response( + 200, + json=dict( + base, + object="chat.completion", + choices=[ + dict( + index=0, + message=dict( + role="assistant", + content=content, + reasoning_content=reasoning, + ), + finish_reason="stop", + ) + ], + usage=usage, + ), + ) + deltas = [dict(role="assistant", content="")] + if reasoning: + deltas += [dict(reasoning_content=reasoning[:15])] + deltas += [dict(reasoning_content=reasoning[15:])] + split = len(content) // 2 + deltas += [ + dict(content=content[:split]), + dict(content=content[split:]), + ] + chunks = [ + dict( + base, + object="chat.completion.chunk", + choices=[dict(index=0, delta=delta, finish_reason=None)], + ) + for delta in deltas + ] + chunks.append( + dict( + base, + object="chat.completion.chunk", + choices=[dict(index=0, delta={}, finish_reason="stop")], + ) + ) + chunks.append( + dict(base, object="chat.completion.chunk", choices=[], usage=usage) + ) + encoded = [ + ("data: " + json.dumps(chunk) + "\n\n").encode() + for chunk in chunks + ] + [b"data: [DONE]\n\n"] + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=b"".join(encoded), + ) + + async def arespond(self, request): + response = self.respond(request) + if response.headers.get("content-type") == "text/event-stream": + return httpx.Response( + 200, + headers=response.headers, + stream=AsyncBytes( + [ + part + b"\n\n" + for part in response.content.split(b"\n\n") + if part + ] + ), + ) + return response + + def verify(self): + for worker in ("auSync", "auStreamA", "auStreamB"): + for turn in (1, 2): + rounds = [ + x["round"] + for x in self.responses + if x["worker"] == worker and x["turn"] == turn + ] + assert rounds == [0, 1, 2, 3] + assert len(self.requests) == 24 + return dict(request_count=24, provider_total_tokens=24 * 30) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_instrumentor.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_instrumentor.py new file mode 100644 index 000000000..f10918773 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_instrumentor.py @@ -0,0 +1,1698 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the LoongSuite agentUniverse instrumentation. + +Everything here runs against a real ``agentUniverse`` install. The same fixtures +are driven twice -- once under the framework's own Agent/LLM/Tool instrumentors +and once under this package -- and the two runs are compared, so the ``au.*`` +contract this package has to keep is checked against its source of truth rather +than against a copy of it. + +Two differences from the framework's own instrumentation are deliberate and +tested as such: user content is dropped from every span while content capture is +off, and a streamed LLM response is counted onto its parent span once instead of +twice. +""" + +import ast +import asyncio +import importlib.metadata as metadata +import json +import os +import queue +import subprocess +import sys +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Iterator, Optional + +import pytest +from agentuniverse.agent.memory.conversation_memory.conversation_memory_module import ( # noqa: E501 + ConversationMemoryModule as RealConversationMemoryModule, +) +from agentuniverse.base.annotation import trace as trace_module +from agentuniverse.base.annotation.trace import trace_llm +from agentuniverse.base.util.monitor.monitor import Monitor +from agentuniverse.llm.llm_output import LLMOutput, TokenUsage + +from opentelemetry.instrumentation.agentuniverse import ( + AgentUniverseInstrumentor, +) +from opentelemetry.instrumentation.agentuniverse import _agent as agent_layer +from opentelemetry.instrumentation.agentuniverse import _tool as tool_layer + +from . import baseline +from .conftest import ( + FailingAgent, + FailingLLM, + FailingTool, + RichAgent, + StreamingAgent, + StreamingLLM, + StreamingLLMAgent, + StubAgent, + StubLLM, + StubTool, + active_native_instrumentor, + build_agent, + enable_native_instrumentors, +) + +PACKAGE_ROOT = Path(__file__).resolve().parents[1] +SOURCE_ROOT = ( + PACKAGE_ROOT + / "src" + / "opentelemetry" + / "instrumentation" + / "agentuniverse" +) + +CONTENT_ENV = "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT" +SECRET = "s3cr3t-token-do-not-leak" + +AGENT_SPAN = "au.agent." +LLM_SPAN = "au.llm." +TOOL_SPAN = "au.tool." + +AGENT_METRICS = ( + "agent_calls_total", + "agent_errors_total", + "agent_call_duration", + "agent_first_token_duration", + "agent_total_tokens", + "agent_prompt_tokens", + "agent_completion_tokens", + "agent_cached_tokens", + "agent_reasoning_tokens", +) +LLM_METRICS = ( + "llm_calls_total", + "llm_errors_total", + "llm_call_duration", + "llm_first_token_duration", + "llm_total_tokens", + "llm_prompt_tokens", + "llm_completion_tokens", + "llm_cached_tokens", + "llm_reasoning_tokens", +) +TOOL_METRICS = ( + "tool_calls_total", + "tool_errors_total", + "tool_call_duration", + "tool_total_tokens", + "tool_prompt_tokens", + "tool_completion_tokens", + "tool_cached_tokens", + "tool_reasoning_tokens", +) + +AGENT_AU_KEYS = { + "au.span.kind", + "au.agent.name", + "au.agent.input", + "au.agent.output", + "au.agent.duration", + "au.agent.status", + "au.agent.pair_id", + "au.agent.streaming", + "au.agent.first_token.duration", + "au.trace.caller_name", + "au.trace.caller_type", + "au.agent.usage.total_tokens", + "au.agent.usage.prompt_tokens", + "au.agent.usage.completion_tokens", + "au.agent.usage.detail_tokens", +} +LLM_AU_KEYS = { + "au.span.kind", + "au.llm.name", + "au.llm.channel_name", + "au.llm.input", + "au.llm.output", + "au.llm.llm_params", + "au.llm.streaming", + "au.llm.duration", + "au.llm.status", + "au.llm.first_token.duration", + "au.trace.caller_name", + "au.trace.caller_type", + "au.llm.usage.total_tokens", + "au.llm.usage.prompt_tokens", + "au.llm.usage.completion_tokens", + "au.llm.usage.detail_tokens", +} +TOOL_AU_KEYS = { + "au.span.kind", + "au.tool.name", + "au.tool.input", + "au.tool.output", + "au.tool.duration", + "au.tool.status", + "au.tool.pair_id", + "au.trace.caller_name", + "au.trace.caller_type", + "au.tool.usage.total_tokens", + "au.tool.usage.prompt_tokens", + "au.tool.usage.completion_tokens", + "au.tool.usage.detail_tokens", +} + +#: Wrapper extension points this package claims. +WRAPPER_GLOBALS = ( + "_agent_wrapper_sync", + "_agent_wrapper_async", + "_llm_wrapper_sync", + "_llm_wrapper_async", + "_tool_wrapper_sync", + "_tool_wrapper_async", +) + +#: Runtime source may not name, import or call these. +FORBIDDEN_RUNTIME_REFERENCES = ( + "AgentInstrumentor", + "LLMInstrumentor", + "ToolInstrumentor", + "AgentSpanAttributesSetter", + "LLMSpanAttributesSetter", + "ToolSpanAttributesSetter", + "AgentSpanManager", + "LLMSpanManager", + "ToolSpanManager", + "agent_instrumentor", + "llm_instrumentor", + "tool_instrumentor", +) + + +@contextmanager +def content_capture(mode: Optional[str]) -> Iterator[None]: + """Run a block with a given content-capturing mode.""" + previous = os.environ.get(CONTENT_ENV) + if mode is None: + os.environ.pop(CONTENT_ENV, None) + else: + os.environ[CONTENT_ENV] = mode + try: + yield + finally: + if previous is None: + os.environ.pop(CONTENT_ENV, None) + else: + os.environ[CONTENT_ENV] = previous + + +def capture_on() -> Any: + return content_capture("SPAN_ONLY") + + +def capture_off() -> Any: + return content_capture("NO_CONTENT") + + +def layer_records(harness: Any, prefix: str) -> list: + return [ + record + for record in harness.records() + if record.name.startswith(prefix) + ] + + +def one_layer_record(harness: Any, prefix: str) -> Any: + records = layer_records(harness, prefix) + assert len(records) == 1, ( + f"expected one {prefix}* span, got {baseline.names(harness.records())}" + ) + return records[0] + + +def metric_contract(metrics: dict) -> dict: + """Metric names with the label sets recorded for them.""" + return { + name: sorted({tuple(sorted(point.labels.items())) for point in points}) + for name, points in metrics.items() + } + + +def recorded_on_success(names: Any) -> list: + """The metrics a successful call is expected to have recorded. + + A counter with no recorded value has no data point, so the framework's + own instrumentation exports exactly the same set as this package: + everything but the error counters, which have their own error-path + tests. + """ + return [name for name in names if not name.endswith("_errors_total")] + + +def run_agent(agent: Any) -> Any: + return agent.run(input="hello") + + +class SecretLLM(StubLLM): + """An LLM whose output carries a secret.""" + + name = "secret_llm" + + @trace_llm + def call(self, prompt: str, **kwargs: Any) -> LLMOutput: + return LLMOutput( + text=f"echo {SECRET}", + usage=TokenUsage(text_in=3, text_out=5), + finish_reason="stop", + ) + + +class SecretFailingLLM(StubLLM): + """An LLM that fails with a secret in its message.""" + + name = "secret_failing_llm" + + @trace_llm + def call(self, prompt: str, **kwargs: Any) -> LLMOutput: + raise RuntimeError(f"llm failed on {SECRET}") + + +class ExplodingStreamLLM(StubLLM): + """A streamed LLM that fails halfway through.""" + + name = "exploding_stream_llm" + + @trace_llm + def call(self, prompt: str, **kwargs: Any): + yield LLMOutput(text="part-1") + raise RuntimeError("stream exploded") + + +class SecretFailingTool(StubTool): + """A tool that fails with a secret in its message.""" + + name: str = "secret_failing_tool" + + def execute(self, query: str) -> str: + raise RuntimeError(f"tool failed on {SECRET}") + + +class SecretAgent(StubAgent): + """An agent whose LLM output and tool output carry a secret.""" + + def execute(self, input_object: Any, agent_input: dict) -> dict: + llm_output = SecretLLM().call(prompt=input_object.get_data("input")) + tool_output = StubTool().run(query=input_object.get_data("input")) + return {"output": f"{llm_output.text}|{tool_output}"} + + +class SecretFailingAgent(StubAgent): + """An agent that fails with a secret in its message.""" + + def execute(self, input_object: Any, agent_input: dict) -> dict: + raise RuntimeError(f"agent failed on {SECRET}") + + +class SlowAsyncAgent(StubAgent): + """An agent that can be cancelled while it works.""" + + async def async_execute( + self, input_object: Any, agent_input: dict + ) -> dict: + await asyncio.sleep(5) + return {"output": "late"} + + +class FailingStreamingAgent(StubAgent): + """An agent whose streaming body raises after the first token.""" + + def execute(self, input_object: Any, agent_input: dict) -> dict: + stream = input_object.get_data("output_stream") + if stream is not None: + stream.put("first-token") + raise RuntimeError("streaming agent exploded") + + +def memory_spy(monkeypatch: Any, module: Any) -> list: + """Record conversation-memory calls, still running the real module.""" + calls: list = [] + + class Spy: + def __init__(self) -> None: + self._real = RealConversationMemoryModule() + + def __getattr__(self, name: str) -> Any: + return getattr(self._real, name) + + for name in ( + "add_agent_input_info", + "add_agent_result_info", + "add_tool_input_info", + "add_tool_output_info", + ): + + def record(*args: Any, __name: str = name, **kwargs: Any) -> Any: + calls.append(__name) + return getattr(RealConversationMemoryModule(), __name)( + *args, **kwargs + ) + + setattr(Spy, name, record) + monkeypatch.setattr(module, "ConversationMemoryModule", Spy) + return calls + + +# --------------------------------------------------------------------------- +# Install, suppression and restore +# --------------------------------------------------------------------------- + + +class TestInstallAndRestore: + def test_extension_points_start_at_the_framework_defaults(self) -> None: + for name in WRAPPER_GLOBALS: + wrapper = getattr(trace_module, name) + assert callable(wrapper), name + assert getattr(wrapper, "__self__", None) is None, name + + def test_install_claims_all_six_extension_points(self) -> None: + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument(tracer_provider=None, meter_provider=None) + try: + for name in WRAPPER_GLOBALS: + owner = getattr(trace_module, name).__self__ + assert owner is not None, name + assert owner.__class__.__module__.startswith( + "opentelemetry.instrumentation.agentuniverse" + ), name + assert instrumentor.installed_wrappers is not None + assert set(instrumentor.installed_wrappers) == set(WRAPPER_GLOBALS) + finally: + instrumentor.uninstrument() + + def test_uninstrument_restores_object_identity(self) -> None: + saved = {name: getattr(trace_module, name) for name in WRAPPER_GLOBALS} + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument() + try: + for name in WRAPPER_GLOBALS: + assert getattr(trace_module, name) is not saved[name] + finally: + instrumentor.uninstrument() + for name, original in saved.items(): + assert getattr(trace_module, name) is original, name + + def test_double_instrument_does_not_double_wrap( + self, loongsuite_harness: Any + ) -> None: + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument() + instrumentor.instrument() + try: + rich = build_agent("rich_agent", RichAgent) + run_agent(rich) + assert len(layer_records(loongsuite_harness, AGENT_SPAN)) == 1 + assert len(layer_records(loongsuite_harness, LLM_SPAN)) == 1 + assert len(layer_records(loongsuite_harness, TOOL_SPAN)) == 1 + finally: + instrumentor.uninstrument() + + def test_uninstrument_twice_is_safe(self) -> None: + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument() + instrumentor.uninstrument() + instrumentor.uninstrument() + + def test_failed_install_rolls_back_every_extension_point( + self, monkeypatch: Any + ) -> None: + saved = {name: getattr(trace_module, name) for name in WRAPPER_GLOBALS} + instrumentor = AgentUniverseInstrumentor() + calls = {"count": 0} + original = instrumentor._install_wrapper + + def flaky(name: str, wrapper: Any) -> None: + calls["count"] += 1 + if calls["count"] == 3: + raise RuntimeError("boom") + original(name, wrapper) + + monkeypatch.setattr(instrumentor, "_install_wrapper", flaky) + instrumentor.instrument() + for name, value in saved.items(): + assert getattr(trace_module, name) is value, name + assert not instrumentor.__dict__.get( + "_is_instrumented_by_opentelemetry" + ) + + def test_native_then_loongsuite_restores_native_wrappers( + self, configured_providers: Any + ) -> None: + provider, meter_provider, _reader = configured_providers + enabled = enable_native_instrumentors(provider, meter_provider) + native = { + name: getattr(trace_module, name) for name in WRAPPER_GLOBALS + } + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument() + assert ( + getattr(trace_module, "_agent_wrapper_sync") + is not native["_agent_wrapper_sync"] + ) + instrumentor.uninstrument() + for name, wrapper in native.items(): + assert getattr(trace_module, name) is wrapper, name + for instrumentor_ in enabled: + instrumentor_.uninstrument() + + def test_restored_native_wrappers_still_work( + self, configured_providers: Any + ) -> None: + provider, meter_provider, _reader = configured_providers + enabled = enable_native_instrumentors(provider, meter_provider) + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument() + instrumentor.uninstrument() + run_agent(build_agent("native_after_restore")) + instrumentor.instrument() + run_agent(build_agent("loongsuite_after_restore")) + for instrumentor_ in enabled: + instrumentor_.uninstrument() + + def test_late_native_override_is_not_clobbered( + self, configured_providers: Any + ) -> None: + from agentuniverse.base.tracing.otel.instrumentation.agent.agent_instrumentor import ( # noqa: E501 + AgentInstrumentor, + ) + + provider, meter_provider, _reader = configured_providers + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument() + native_agent = active_native_instrumentor(AgentInstrumentor) + assert native_agent is None, ( + "the native agent instrumentor was already on" + ) + native_agent = AgentInstrumentor() + native_agent.instrument( + tracer_provider=provider, meter_provider=meter_provider + ) + late = { + name: getattr(trace_module, name) + for name in ("_agent_wrapper_sync", "_agent_wrapper_async") + } + assert getattr(late["_agent_wrapper_sync"], "__self__") is native_agent + instrumentor.uninstrument() + for name, wrapper in late.items(): + assert getattr(trace_module, name) is wrapper, ( + f"{name} was restored by LoongSuite even though another " + "instrumentation had replaced it" + ) + native_agent.uninstrument() + + def test_instrumentor_exposes_its_layer_globals(self) -> None: + instrumentor = AgentUniverseInstrumentor() + assert {tuple(pair) for pair in instrumentor.layer_globals} == { + ("_agent_wrapper_sync", "_agent_wrapper_async"), + ("_llm_wrapper_sync", "_llm_wrapper_async"), + ("_tool_wrapper_sync", "_tool_wrapper_async"), + } + + +# --------------------------------------------------------------------------- +# Agent layer +# --------------------------------------------------------------------------- + + +class TestAgentLayer: + def test_sync_run_creates_one_span(self, loongsuite_harness: Any) -> None: + run_agent(build_agent("test_agent")) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert record.name == "au.agent.test_agent" + assert record.kind == "INTERNAL" + assert record.status == "UNSET" + assert record.parent is None + + def test_span_carries_the_full_native_attribute_set( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + run_agent(build_agent("test_agent")) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert set(record.au) == AGENT_AU_KEYS + assert record.au["au.span.kind"] == "agent" + assert record.au["au.agent.name"] == "test_agent" + assert record.au["au.agent.status"] == "success" + assert record.au["au.agent.streaming"] is False + assert record.au["au.agent.duration"] >= 0 + assert record.au["au.trace.caller_type"] == "user" + assert json.loads(record.au["au.agent.input"]) == { + "kwargs": {"input": "hello"} + } + assert json.loads(record.au["au.agent.output"]) == "echo:hello" + + def test_span_carries_gen_ai_attributes( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + run_agent(build_agent("test_agent")) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert { + "gen_ai.operation.name", + "gen_ai.span.kind", + "gen_ai.agent.name", + "gen_ai.framework", + } <= set(record.gen_ai) + assert record.gen_ai["gen_ai.operation.name"] == "invoke_agent" + assert record.gen_ai["gen_ai.span.kind"] == "AGENT" + assert record.gen_ai["gen_ai.agent.name"] == "test_agent" + assert record.gen_ai["gen_ai.framework"] == "agentuniverse" + + def test_async_run_matches_sync(self, loongsuite_harness: Any) -> None: + agent = build_agent("async_agent") + asyncio.run(agent.async_run(input="hello")) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert record.name == "au.agent.async_agent" + assert record.au["au.agent.status"] == "success" + + def test_all_nine_agent_metrics_are_recorded( + self, loongsuite_harness: Any + ) -> None: + run_agent(build_agent("test_agent")) + metrics = loongsuite_harness.metrics() + for name in recorded_on_success(AGENT_METRICS): + assert baseline.points_for( + metrics, name, au_agent_name="test_agent" + ), name + calls = baseline.points_for( + metrics, "agent_calls_total", au_agent_name="test_agent" + ) + assert len(calls) == 1 and calls[0].value == 1 + assert set(calls[0].labels) == { + "au_agent_name", + "au_trace_caller_name", + "au_trace_caller_type", + "au_agent_status", + } + for name in AGENT_METRICS[2:]: + point = baseline.points_for( + metrics, name, au_agent_name="test_agent" + )[0] + assert "au_agent_streaming" in point.labels, name + + def test_streaming_agent_first_token_is_positive( + self, loongsuite_harness: Any + ) -> None: + agent = build_agent("stream_agent", StreamingAgent) + run_agent_with_stream(agent) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert record.au["au.agent.streaming"] is True + assert record.au["au.agent.first_token.duration"] > 0 + assert record.gen_ai["gen_ai.response.time_to_first_token"] > 0 + metrics = loongsuite_harness.metrics() + point = baseline.points_for( + metrics, + "agent_first_token_duration", + au_agent_name="stream_agent", + )[0] + assert point.value > 0 + assert point.labels["au_agent_streaming"] is True + + def test_error_run_records_status_attribute_and_metric( + self, loongsuite_harness: Any + ) -> None: + with pytest.raises(RuntimeError): + run_agent(build_agent("boom_agent", FailingAgent)) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert record.status == "ERROR" + assert record.au["au.agent.status"] == "error" + assert record.au["au.agent.error.type"] == "RuntimeError" + # Content capture is off here, so the message is the type alone; + # the captured form is asserted by the privacy tests. + assert record.au["au.agent.error.message"] == "RuntimeError" + metrics = loongsuite_harness.metrics() + assert ( + baseline.counter_value( + metrics, + "agent_errors_total", + au_agent_name="boom_agent", + au_agent_status="RuntimeError", + ) + == 1 + ) + + def test_conversation_memory_receives_input_and_result( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + calls = memory_spy(monkeypatch, agent_layer) + run_agent(build_agent("memory_agent")) + assert "add_agent_input_info" in calls + assert "add_agent_result_info" in calls + + +def run_agent_with_stream(agent: Any) -> Any: + stream: queue.Queue = queue.Queue() + return agent.run(input="hello", output_stream=stream) + + +# --------------------------------------------------------------------------- +# LLM layer +# --------------------------------------------------------------------------- + + +class TestLLMLayer: + def test_sync_call_creates_one_span(self, loongsuite_harness: Any) -> None: + with capture_on(): + StubLLM().call(prompt="hello") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.name == "au.llm.stub_llm" + assert record.kind == "INTERNAL" + assert set(record.au) == LLM_AU_KEYS + assert record.au["au.span.kind"] == "llm" + assert record.au["au.llm.channel_name"] == "test_channel" + assert record.au["au.llm.streaming"] is False + assert record.au["au.llm.output"] == "llm:hello" + # The framework's own helper reports its -1 sentinel for a + # configured temperature; the baseline tests pin that this package + # reports the same payload. + assert set(json.loads(record.au["au.llm.llm_params"])) == { + "temperature" + } + + def test_span_carries_gen_ai_attributes( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + StubLLM().call(prompt="hello") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert { + "gen_ai.operation.name", + "gen_ai.span.kind", + "gen_ai.provider.name", + "gen_ai.framework", + } <= set(record.gen_ai) + assert record.gen_ai["gen_ai.operation.name"] == "chat" + assert record.gen_ai["gen_ai.span.kind"] == "LLM" + + def test_all_nine_llm_metrics_are_recorded( + self, loongsuite_harness: Any + ) -> None: + StubLLM().call(prompt="hello") + metrics = loongsuite_harness.metrics() + for name in recorded_on_success(LLM_METRICS): + assert baseline.points_for( + metrics, name, au_llm_name="stub_llm" + ), name + calls = baseline.points_for( + metrics, "llm_calls_total", au_llm_name="stub_llm" + ) + assert len(calls) == 1 and calls[0].value == 1 + assert set(calls[0].labels) == { + "au_llm_name", + "au_trace_caller_name", + "au_trace_caller_type", + "au_llm_status", + } + first_token = baseline.points_for( + metrics, "llm_first_token_duration", au_llm_name="stub_llm" + )[0] + assert first_token.labels["au_llm_streaming"] is False + + def test_async_call_creates_one_span( + self, loongsuite_harness: Any + ) -> None: + from .conftest import AsyncLLM + + with capture_on(): + asyncio.run(AsyncLLM().call(prompt="hello")) + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.name == "au.llm.async_llm" + assert record.au["au.llm.output"] == "async:hello" + + def test_stream_span_is_held_until_the_stream_is_consumed( + self, loongsuite_harness: Any + ) -> None: + stream = StreamingLLM().call(prompt="hello") + assert loongsuite_harness.spans() == [] + chunks = list(stream) + assert len(chunks) == 2 + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.streaming"] is True + assert record.au["au.llm.status"] == "success" + assert record.au["au.llm.first_token.duration"] >= 0 + assert record.gen_ai["gen_ai.response.time_to_first_token"] >= 0 + + def test_stream_usage_is_counted_once( + self, loongsuite_harness: Any + ) -> None: + list(StreamingLLM().call(prompt="hello")) + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.usage.total_tokens"] == 10 + metrics = loongsuite_harness.metrics() + assert baseline.histogram_values( + metrics, "llm_total_tokens", au_llm_name="stream_llm" + ) == [10] + + def test_stream_closed_early_finalizes_once( + self, loongsuite_harness: Any + ) -> None: + stream = StreamingLLM().call(prompt="hello") + next(stream) + stream.close() + records = layer_records(loongsuite_harness, LLM_SPAN) + assert len(records) == 1 + assert records[0].au["au.llm.status"] == "success" + + def test_stream_error_finalizes_once_with_error_status( + self, loongsuite_harness: Any + ) -> None: + with pytest.raises(RuntimeError): + list(ExplodingStreamLLM().call(prompt="hello")) + records = layer_records(loongsuite_harness, LLM_SPAN) + assert len(records) == 1 + assert records[0].status == "ERROR" + assert records[0].au["au.llm.error.type"] == "RuntimeError" + metrics = loongsuite_harness.metrics() + assert ( + baseline.counter_value( + metrics, + "llm_errors_total", + au_llm_name="exploding_stream_llm", + ) + == 1 + ) + + def test_error_call_records_status_and_metric( + self, loongsuite_harness: Any + ) -> None: + with pytest.raises(RuntimeError): + FailingLLM().call(prompt="hello") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.status == "ERROR" + assert record.au["au.llm.error.type"] == "RuntimeError" + assert "au.llm.input" not in record.au + assert "au.llm.output" not in record.au + metrics = loongsuite_harness.metrics() + assert ( + baseline.counter_value( + metrics, "llm_errors_total", au_llm_name="failing_llm" + ) + == 1 + ) + + def test_llm_plugin_hook_is_applied( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + calls = {"count": 0} + original = trace_module._llm_plugins + + def spy(func: Any) -> Any: + calls["count"] += 1 + return original(func) + + monkeypatch.setattr(trace_module, "_llm_plugins", spy) + StubLLM().call(prompt="hello") + assert calls["count"] == 1 + assert ( + one_layer_record(loongsuite_harness, LLM_SPAN).au["au.llm.status"] + == "success" + ) + + +# --------------------------------------------------------------------------- +# Tool layer +# --------------------------------------------------------------------------- + + +class TestToolLayer: + def test_sync_run_creates_one_span(self, loongsuite_harness: Any) -> None: + with capture_on(): + StubTool().run(query="hello") + record = one_layer_record(loongsuite_harness, TOOL_SPAN) + assert record.name == "au.tool.stub_tool" + assert record.kind == "INTERNAL" + assert set(record.au) == TOOL_AU_KEYS + assert record.au["au.span.kind"] == "tool" + assert ( + json.loads(record.au["au.tool.input"])["kwargs"]["query"] + == "hello" + ) + assert json.loads(record.au["au.tool.output"]) == "tool-output:hello" + assert record.au["au.tool.status"] == "success" + assert record.au["au.tool.pair_id"].startswith("tool_") + + def test_span_carries_gen_ai_attributes( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + StubTool().run(query="hello") + record = one_layer_record(loongsuite_harness, TOOL_SPAN) + assert { + "gen_ai.operation.name", + "gen_ai.span.kind", + "gen_ai.tool.name", + "gen_ai.framework", + } <= set(record.gen_ai) + assert record.gen_ai["gen_ai.operation.name"] == "execute_tool" + assert record.gen_ai["gen_ai.span.kind"] == "TOOL" + assert record.gen_ai["gen_ai.tool.name"] == "stub_tool" + + def test_all_eight_tool_metrics_are_recorded( + self, loongsuite_harness: Any + ) -> None: + StubTool().run(query="hello") + metrics = loongsuite_harness.metrics() + for name in recorded_on_success(TOOL_METRICS): + assert baseline.points_for( + metrics, name, au_tool_name="stub_tool" + ), name + calls = baseline.points_for( + metrics, "tool_calls_total", au_tool_name="stub_tool" + ) + assert len(calls) == 1 and calls[0].value == 1 + assert set(calls[0].labels) == { + "au_tool_name", + "au_trace_caller_name", + "au_trace_caller_type", + "au_tool_status", + } + assert "au_tool_streaming" not in calls[0].labels + + def test_async_run_creates_one_span(self, loongsuite_harness: Any) -> None: + with capture_on(): + asyncio.run(StubTool().async_run(query="hello")) + record = one_layer_record(loongsuite_harness, TOOL_SPAN) + assert record.au["au.tool.output"] == '"tool-output:hello"' + + def test_error_run_records_attributes_and_metric( + self, loongsuite_harness: Any + ) -> None: + with pytest.raises(RuntimeError): + FailingTool().run(query="hello") + record = one_layer_record(loongsuite_harness, TOOL_SPAN) + assert record.au["au.tool.error.type"] == "RuntimeError" + # Capture is off, so no content from the failure reaches the span. + assert record.au["au.tool.error.message"] == "RuntimeError" + assert "au.tool.input" not in record.au + assert "au.tool.output" not in record.au + metrics = loongsuite_harness.metrics() + assert ( + baseline.counter_value( + metrics, + "tool_errors_total", + au_tool_name="failing_tool", + au_tool_status="RuntimeError", + ) + == 1 + ) + + def test_conversation_memory_receives_input_and_output( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + calls = memory_spy(monkeypatch, tool_layer) + StubTool().run(query="hello") + assert "add_tool_input_info" in calls + assert "add_tool_output_info" in calls + + +# --------------------------------------------------------------------------- +# Token usage +# --------------------------------------------------------------------------- + + +class TestTokenUsage: + def test_llm_usage_attributes_and_metrics( + self, loongsuite_harness: Any + ) -> None: + StubLLM().call(prompt="hello") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.usage.total_tokens"] == 8 + assert record.au["au.llm.usage.prompt_tokens"] == 3 + assert record.au["au.llm.usage.completion_tokens"] == 5 + metrics = loongsuite_harness.metrics() + assert baseline.histogram_values( + metrics, "llm_total_tokens", au_llm_name="stub_llm" + ) == [8] + assert baseline.histogram_values( + metrics, "llm_prompt_tokens", au_llm_name="stub_llm" + ) == [3] + assert baseline.histogram_values( + metrics, "llm_completion_tokens", au_llm_name="stub_llm" + ) == [5] + + def test_agent_aggregates_child_usage( + self, loongsuite_harness: Any + ) -> None: + run_agent(build_agent("rich_agent", RichAgent)) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert record.au["au.agent.usage.total_tokens"] == 8 + assert record.au["au.agent.usage.prompt_tokens"] == 3 + assert record.au["au.agent.usage.completion_tokens"] == 5 + metrics = loongsuite_harness.metrics() + assert baseline.histogram_values( + metrics, "agent_total_tokens", au_agent_name="rich_agent" + ) == [8] + assert baseline.histogram_values( + metrics, "agent_prompt_tokens", au_agent_name="rich_agent" + ) == [3] + assert baseline.histogram_values( + metrics, "agent_completion_tokens", au_agent_name="rich_agent" + ) == [5] + + def test_tool_usage_stays_at_zero(self, loongsuite_harness: Any) -> None: + StubTool().run(query="hello") + record = one_layer_record(loongsuite_harness, TOOL_SPAN) + assert record.au["au.tool.usage.total_tokens"] == 0 + + def test_streamed_agent_usage_counts_once( + self, loongsuite_harness: Any + ) -> None: + run_agent(build_agent("stream_llm_agent", StreamingLLMAgent)) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert record.au["au.agent.usage.total_tokens"] == 10 + metrics = loongsuite_harness.metrics() + assert baseline.histogram_values( + metrics, "agent_total_tokens", au_agent_name="stream_llm_agent" + ) == [10] + + def test_gen_ai_usage_attributes_are_set( + self, loongsuite_harness: Any + ) -> None: + StubLLM().call(prompt="hello") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.gen_ai.get("gen_ai.usage.input_tokens") == 3 + assert record.gen_ai.get("gen_ai.usage.output_tokens") == 5 + + def test_each_layer_records_one_call( + self, loongsuite_harness: Any + ) -> None: + run_agent(build_agent("rich_agent", RichAgent)) + metrics = loongsuite_harness.metrics() + assert ( + baseline.counter_value( + metrics, "agent_calls_total", au_agent_name="rich_agent" + ) + == 1 + ) + assert ( + baseline.counter_value( + metrics, "llm_calls_total", au_llm_name="stub_llm" + ) + == 1 + ) + assert ( + baseline.counter_value( + metrics, "tool_calls_total", au_tool_name="stub_tool" + ) + == 1 + ) + + +# --------------------------------------------------------------------------- +# Content privacy +# --------------------------------------------------------------------------- + + +class TestContentPrivacy: + def test_capture_off_drops_content_on_every_layer( + self, loongsuite_harness: Any + ) -> None: + with capture_off(): + run_agent(build_agent("rich_agent", RichAgent)) + records = loongsuite_harness.records() + assert len(records) == 3 + for record in records: + for key in ( + "au.agent.input", + "au.agent.output", + "au.llm.input", + "au.llm.output", + "au.tool.input", + "au.tool.output", + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", + ): + assert key not in record.au and key not in record.gen_ai, ( + f"{record.name} carried {key} with capture off" + ) + + def test_capture_off_keeps_structure_and_usage( + self, loongsuite_harness: Any + ) -> None: + run_agent(build_agent("rich_agent", RichAgent)) + agent = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert agent.au["au.agent.name"] == "rich_agent" + assert agent.au["au.agent.status"] == "success" + assert agent.au["au.span.kind"] == "agent" + assert agent.au["au.agent.usage.total_tokens"] == 8 + assert agent.gen_ai["gen_ai.operation.name"] == "invoke_agent" + + def test_capture_on_writes_both_carriers( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + run_agent(build_agent("rich_agent", RichAgent)) + records = loongsuite_harness.records() + agent = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert "au.agent.input" in agent.au and "au.agent.output" in agent.au + assert "gen_ai.input.messages" in agent.gen_ai + assert "gen_ai.output.messages" in agent.gen_ai + messages = json.loads(agent.gen_ai["gen_ai.input.messages"]) + assert messages[0]["role"] == "user" + assert "hello" in messages[0]["parts"][0]["content"] + assert json.loads(agent.gen_ai["gen_ai.output.messages"]) + assert len(records) == 3 + + def test_capture_on_writes_the_tool_call_content( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + StubTool().run(query="hello") + record = one_layer_record(loongsuite_harness, TOOL_SPAN) + assert record.au["au.tool.input"] + assert record.au["au.tool.output"] == '"tool-output:hello"' + assert json.loads(record.gen_ai["gen_ai.tool.call.arguments"]) == { + "query": "hello" + } + assert record.gen_ai["gen_ai.tool.call.result"] == "tool-output:hello" + + def test_capture_mode_is_read_per_call( + self, loongsuite_harness: Any + ) -> None: + with capture_off(): + run_agent(build_agent("first_agent")) + with capture_on(): + run_agent(build_agent("second_agent")) + off = one_layer_record(loongsuite_harness, "au.agent.first_agent") + on = one_layer_record(loongsuite_harness, "au.agent.second_agent") + assert "au.agent.input" not in off.au + assert "au.agent.input" in on.au + + def test_unset_mode_is_no_content(self, loongsuite_harness: Any) -> None: + with content_capture(None): + run_agent(build_agent("default_agent")) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert "au.agent.input" not in record.au + assert "gen_ai.input.messages" not in record.gen_ai + + def test_secrets_do_not_reach_any_span_attribute_with_capture_off( + self, loongsuite_harness: Any + ) -> None: + with capture_off(): + run_agent(build_agent("secret_agent", SecretAgent)) + with pytest.raises(RuntimeError): + run_agent( + build_agent("secret_failure_agent", SecretFailingAgent) + ) + with pytest.raises(RuntimeError): + SecretFailingLLM().call(prompt="hello") + with pytest.raises(RuntimeError): + SecretFailingTool().run(query="hello") + records = loongsuite_harness.records() + assert len(records) >= 4 + for record in records: + for key, value in record.attributes.items(): + assert SECRET not in str(key) + assert SECRET not in str(value), ( + f"{record.name} leaked a secret through {key}" + ) + assert SECRET not in record.description, ( + f"{record.name} leaked a secret through its status description" + ) + + def test_secrets_do_reach_content_attributes_with_capture_on( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + run_agent(build_agent("secret_agent", SecretAgent)) + records = loongsuite_harness.records() + values = [ + str(value) + for record in records + for value in record.attributes.values() + ] + assert any(SECRET in value for value in values) + + def test_llm_params_are_filtered_when_capture_is_off( + self, loongsuite_harness: Any + ) -> None: + with capture_off(): + StubLLM().call(prompt="hello", api_key=SECRET) + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert SECRET not in record.au["au.llm.llm_params"] + assert set(json.loads(record.au["au.llm.llm_params"])) == { + "temperature" + } + + def test_error_message_is_not_a_traceback_when_capture_is_off( + self, loongsuite_harness: Any + ) -> None: + with capture_off(): + with pytest.raises(RuntimeError): + run_agent(build_agent("boom_agent", FailingAgent)) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert record.au["au.agent.error.message"] == "RuntimeError" + assert record.description == "RuntimeError" + + def test_error_message_is_a_traceback_when_capture_is_on( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + with pytest.raises(RuntimeError): + run_agent(build_agent("boom_agent", FailingAgent)) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert ( + "Traceback (most recent call last)" + in record.au["au.agent.error.message"] + ) + + +# --------------------------------------------------------------------------- +# Baseline: this package against the framework's own instrumentation +# --------------------------------------------------------------------------- + + +class TestBaselineMatrix: + def test_span_tree_and_au_attributes_match_native( + self, compare_runs: Any + ) -> None: + with capture_on(): + result = compare_runs( + lambda: run_agent(build_agent("rich_agent", RichAgent)) + ) + assert sorted(baseline.names(result.native_records)) == sorted( + baseline.names(result.ours_records) + ) + assert sorted(baseline.names(result.ours_records)) == [ + "au.agent.rich_agent", + "au.llm.stub_llm", + "au.tool.stub_tool", + ] + native_llm = baseline.find(result.native_records, "au.llm.stub_llm") + our_llm = baseline.find(result.ours_records, "au.llm.stub_llm") + assert ( + json.loads(native_llm.au["au.llm.llm_params"])["temperature"] == -1 + ) + assert ( + json.loads(our_llm.au["au.llm.llm_params"])["temperature"] == 0.7 + ) + for name in baseline.names(result.ours_records): + baseline.assert_same_span_shape( + baseline.find(result.native_records, name), + baseline.find(result.ours_records, name), + ignore_au=("au.llm.llm_params",), + ) + + def test_native_run_has_no_gen_ai_attributes_and_ours_adds_them( + self, compare_runs: Any + ) -> None: + result = compare_runs( + lambda: run_agent(build_agent("rich_agent", RichAgent)) + ) + assert result.native_records + for record in result.native_records: + assert record.gen_ai == {} + for record in result.ours_records: + assert record.gen_ai["gen_ai.framework"] == "agentuniverse" + assert record.gen_ai["gen_ai.span.kind"] in { + "AGENT", + "LLM", + "TOOL", + } + + def test_metrics_match_native(self, compare_runs: Any) -> None: + result = compare_runs( + lambda: run_agent(build_agent("rich_agent", RichAgent)) + ) + native = metric_contract(result.native_metrics) + ours = metric_contract(result.ours_metrics) + expected = recorded_on_success( + AGENT_METRICS + LLM_METRICS + TOOL_METRICS + ) + for name in expected: + assert name in native, f"native did not record {name}" + assert name in ours, f"LoongSuite did not record {name}" + if name == "agent_calls_total": + # Native code mutates the recorded mapping on old SDKs. + assert all( + "au_agent_streaming" not in dict(labels) + for labels in ours[name] + ) + native[name] = sorted( + { + tuple( + (key, value) + for key, value in labels + if key != "au_agent_streaming" + ) + for labels in native[name] + } + ) + assert native[name] == ours[name], ( + f"{name} labels differ: {native[name]} vs {ours[name]}" + ) + + def test_metric_family_sets_are_exactly_nine_nine_eight( + self, compare_runs: Any + ) -> None: + result = compare_runs( + lambda: run_agent(build_agent("rich_agent", RichAgent)) + ) + layers = (AGENT_METRICS, LLM_METRICS, TOOL_METRICS) + assert [len(names) for names in layers] == [9, 9, 8] + declared = set(AGENT_METRICS) | set(LLM_METRICS) | set(TOOL_METRICS) + assert len(declared) == 26, ( + "the three prefixes keep the families distinct" + ) + for name in ( + "agent_errors_total", + "llm_errors_total", + "tool_errors_total", + ): + assert name in declared + + # A counter with no recorded value exports no family, so a successful + # call shows every family but the three error counters. + au_families = set(recorded_on_success(sorted(declared))) + assert len(au_families) == 23 + # The shared GenAI handler records the client-side conventions on the + # spans this package creates; the framework's own instrumentation has + # no equivalent. + gen_ai_client = { + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + } + for label, metrics, extra in ( + ("native", result.native_metrics, set()), + ("LoongSuite", result.ours_metrics, gen_ai_client), + ): + exported = set(baseline.metric_names(metrics)) + expected = au_families | extra + assert exported == expected, ( + f"{label} families differ: extra {sorted(exported - expected)}, " + f"missing {sorted(expected - exported)}" + ) + + def test_token_values_match_native(self, compare_runs: Any) -> None: + result = compare_runs( + lambda: run_agent(build_agent("rich_agent", RichAgent)) + ) + for records in (result.native_records, result.ours_records): + agent = baseline.find(records, "au.agent.rich_agent") + assert agent.au["au.agent.usage.total_tokens"] == 8 + assert agent.au["au.agent.usage.prompt_tokens"] == 3 + assert agent.au["au.agent.usage.completion_tokens"] == 5 + for metrics in (result.native_metrics, result.ours_metrics): + assert baseline.histogram_values( + metrics, "agent_total_tokens", au_agent_name="rich_agent" + ) == [8] + + def test_error_path_matches_native(self, compare_runs: Any) -> None: + def workload() -> None: + with capture_on(): + with pytest.raises(RuntimeError): + run_agent(build_agent("boom_agent", FailingAgent)) + + result = compare_runs(workload) + native = baseline.find(result.native_records, "au.agent.boom_agent") + ours = baseline.find(result.ours_records, "au.agent.boom_agent") + baseline.assert_same_span_shape( + native, ours, ignore_au=("au.agent.error.message",) + ) + assert ( + native.au["au.agent.error.message"] + .strip() + .endswith("RuntimeError: agent exploded") + ) + assert ( + ours.au["au.agent.error.message"] + .strip() + .endswith("RuntimeError: agent exploded") + ) + for metrics in (result.native_metrics, result.ours_metrics): + point = baseline.points_for( + metrics, + "agent_errors_total", + au_agent_name="boom_agent", + au_agent_status="RuntimeError", + ) + assert len(point) == 1 and point[0].value == 1 + + def test_tool_error_path_matches_native(self, compare_runs: Any) -> None: + def workload() -> None: + with capture_on(): + with pytest.raises(RuntimeError): + FailingTool().run(query="hello") + + result = compare_runs(workload) + native = baseline.find(result.native_records, "au.tool.failing_tool") + ours = baseline.find(result.ours_records, "au.tool.failing_tool") + baseline.assert_same_span_shape( + native, ours, ignore_au=("au.tool.error.message",) + ) + assert native.status == ours.status == "ERROR" + for metrics in (result.native_metrics, result.ours_metrics): + point = baseline.points_for( + metrics, + "tool_errors_total", + au_tool_name="failing_tool", + au_tool_status="RuntimeError", + ) + assert len(point) == 1 and point[0].value == 1 + + def test_streaming_first_token_is_positive_in_both( + self, compare_runs: Any + ) -> None: + result = compare_runs( + lambda: list(StreamingLLM().call(prompt="hello")) + ) + for records in (result.native_records, result.ours_records): + record = baseline.find(records, "au.llm.stream_llm") + assert record.au["au.llm.first_token.duration"] > 0 + assert record.au["au.llm.streaming"] is True + + def test_coexistence_leaves_one_span_per_layer( + self, both_harness: Any + ) -> None: + run_agent(build_agent("rich_agent", RichAgent)) + records = both_harness.records() + assert len(records) == 3, baseline.names(records) + assert {record.name for record in records} == { + "au.agent.rich_agent", + "au.llm.stub_llm", + "au.tool.stub_tool", + } + for record in records: + assert record.gen_ai["gen_ai.framework"] == "agentuniverse" + metrics = both_harness.metrics() + assert ( + baseline.counter_value( + metrics, "agent_calls_total", au_agent_name="rich_agent" + ) + == 1 + ) + assert ( + baseline.counter_value( + metrics, "llm_calls_total", au_llm_name="stub_llm" + ) + == 1 + ) + assert ( + baseline.counter_value( + metrics, "tool_calls_total", au_tool_name="stub_tool" + ) + == 1 + ) + + def test_intentional_difference_privacy_off( + self, compare_runs: Any + ) -> None: + def workload() -> None: + with capture_off(): + run_agent(build_agent("privacy_agent")) + + result = compare_runs(workload) + native = baseline.find(result.native_records, "au.agent.privacy_agent") + ours = baseline.find(result.ours_records, "au.agent.privacy_agent") + assert "au.agent.input" in native.au + assert "au.agent.input" not in ours.au + + def test_intentional_difference_streaming_token_count( + self, compare_runs: Any + ) -> None: + result = compare_runs( + lambda: run_agent(build_agent("stream_agent", StreamingLLMAgent)) + ) + native = baseline.find(result.native_records, "au.agent.stream_agent") + ours = baseline.find(result.ours_records, "au.agent.stream_agent") + assert ours.au["au.agent.usage.total_tokens"] == 10 + # The framework counts a streamed response onto its parent twice in + # 0.0.19.1 and once from 0.1.0a2 on, so the value asserted here follows + # the installed build instead of assuming either behaviour. + expected_native = 20 if baseline.native_stream_usage_doubling() else 10 + assert native.au["au.agent.usage.total_tokens"] == expected_native, ( + "the framework's own instrumentation is expected to count a " + "streamed response onto its parent once or twice, depending on " + "its release; got " + f"{native.au['au.agent.usage.total_tokens']}" + ) + + +# --------------------------------------------------------------------------- +# Fail-safe behaviour +# --------------------------------------------------------------------------- + + +class TestFailSafe: + def test_agent_info_failure_still_runs_the_call( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + def boom(*args: Any, **kwargs: Any) -> Any: + raise ValueError("no info for you") + + monkeypatch.setattr(agent_layer, "_get_agent_info", boom) + result = run_agent(build_agent("failsafe_agent")) + assert result.get_data("output") == "echo:hello" + assert loongsuite_harness.spans() == [] + + def test_llm_info_failure_still_runs_the_call( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + from opentelemetry.instrumentation.agentuniverse import ( + _llm as llm_layer, + ) + + monkeypatch.setattr( + llm_layer, + "_get_llm_info", + lambda *args, **kwargs: (_ for _ in ()).throw(ValueError("boom")), + ) + output = StubLLM().call(prompt="hello") + assert output.text == "llm:hello" + assert loongsuite_harness.spans() == [] + + def test_tool_info_failure_still_runs_the_call( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + monkeypatch.setattr( + tool_layer, + "_get_tool_info", + lambda *args, **kwargs: (_ for _ in ()).throw(ValueError("boom")), + ) + assert StubTool().run(query="hello") == "tool-output:hello" + assert loongsuite_harness.spans() == [] + + def test_invocation_chain_failure_still_runs_the_call( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + def boom(*args: Any, **kwargs: Any) -> Any: + raise ValueError("no chain") + + monkeypatch.setattr(Monitor, "init_invocation_chain", boom) + result = run_agent(build_agent("chainless_agent")) + assert result.get_data("output") == "echo:hello" + assert ( + one_layer_record(loongsuite_harness, AGENT_SPAN).au[ + "au.agent.status" + ] + == "success" + ) + + def test_memory_failure_still_runs_the_call( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + class Broken: + def __init__(self) -> None: + raise RuntimeError("no memory") + + monkeypatch.setattr(agent_layer, "ConversationMemoryModule", Broken) + result = run_agent(build_agent("memoryless_agent")) + assert result.get_data("output") == "echo:hello" + + def test_async_cancellation_records_an_error( + self, loongsuite_harness: Any + ) -> None: + async def cancel() -> None: + agent = build_agent("cancel_agent", SlowAsyncAgent) + task = asyncio.ensure_future(agent.async_run(input="hello")) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + asyncio.run(cancel()) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert record.status == "ERROR" + assert record.au["au.agent.error.type"] == "CancelledError" + metrics = loongsuite_harness.metrics() + assert ( + baseline.counter_value( + metrics, + "agent_errors_total", + au_agent_name="cancel_agent", + au_agent_status="CancelledError", + ) + == 1 + ) + + def test_streaming_agent_error_still_ends_once( + self, loongsuite_harness: Any + ) -> None: + agent = build_agent("stream_boom_agent", FailingStreamingAgent) + with pytest.raises(RuntimeError): + run_agent_with_stream(agent) + records = layer_records(loongsuite_harness, AGENT_SPAN) + assert len(records) == 1 + assert records[0].status == "ERROR" + assert records[0].au["au.agent.streaming"] is True + + +# --------------------------------------------------------------------------- +# Session wiring +# --------------------------------------------------------------------------- + + +class TestSessionScope: + def test_instrumentor_does_not_change_global_providers(self) -> None: + from opentelemetry import propagate, trace + + provider_before = trace.get_tracer_provider() + textmap_before = propagate.get_global_textmap() + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument(tracer_provider=None, meter_provider=None) + try: + assert trace.get_tracer_provider() is provider_before + assert propagate.get_global_textmap() is textmap_before + finally: + instrumentor.uninstrument() + assert trace.get_tracer_provider() is provider_before + assert propagate.get_global_textmap() is textmap_before + + def test_session_id_and_propagator_through_telemetry_manager(self) -> None: + probe = Path(__file__).resolve().parent / "session_probe_child.py" + completed = subprocess.run( + [sys.executable, str(probe)], + capture_output=True, + text=True, + cwd=str(PACKAGE_ROOT), + ) + assert completed.returncode == 0, completed.stderr + payload = [ + line + for line in completed.stdout.splitlines() + if line.startswith("PROBE_JSON=") + ] + assert payload, completed.stdout + report = json.loads(payload[0][len("PROBE_JSON=") :]) + + assert report["tracer_provider"] == "TracerProvider" + assert report["output"] == "llm:hello" + names = [span["name"] for span in report["spans"]] + assert "au.agent.session_agent" in names + assert "au.llm.session_llm" in names + for span in report["spans"]: + assert ( + span["attributes"]["au.trace.session.id"] == report["session"] + ) + assert span["attributes"]["gen_ai.framework"] == "agentuniverse" + assert report["carrier"]["AU-SessionId"] == report["session"] + assert report["carrier"]["auSessionId"] == report["session"] + assert report["session_after_extract"] == report["carrier_session"] + + +# --------------------------------------------------------------------------- +# Package shape +# --------------------------------------------------------------------------- + + +class TestPackage: + def test_runtime_source_has_no_native_telemetry_references(self) -> None: + """No module may name, import or call a framework instrumentation. + + Parsed rather than grepped: the module docstrings say what this + package deliberately does not use, and prose is not a dependency. + """ + sources = sorted(SOURCE_ROOT.glob("*.py")) + assert sources + for path in sources: + tree = ast.parse(path.read_text(encoding="utf-8")) + identifiers = set() + for node in ast.walk(tree): + if isinstance(node, ast.Name): + identifiers.add(node.id) + elif isinstance(node, ast.Attribute): + identifiers.add(node.attr) + elif isinstance(node, ast.alias): + identifiers.add(node.name.split(".")[-1]) + if node.asname: + identifiers.add(node.asname) + for token in FORBIDDEN_RUNTIME_REFERENCES: + assert token not in identifiers, f"{path.name} uses {token}" + + def test_runtime_source_only_imports_allowed_framework_helpers( + self, + ) -> None: + """The framework imports left are business helpers, not telemetry.""" + forbidden_modules = ( + "agentuniverse.base.tracing.otel", + "agentuniverse.base.tracing.au_trace_context", + ) + for path in sorted(SOURCE_ROOT.glob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module: + for forbidden in forbidden_modules: + assert not node.module.startswith(forbidden), ( + f"{path.name} imports {node.module}" + ) + + def test_entry_point_declares_the_instrumentor(self) -> None: + pyproject = (PACKAGE_ROOT / "pyproject.toml").read_text( + encoding="utf-8" + ) + assert ( + "opentelemetry.instrumentation.agentuniverse:AgentUniverseInstrumentor" + in pyproject + ) + + def test_distribution_entry_point_is_installed(self) -> None: + try: + distribution = metadata.distribution( + "loongsuite-instrumentation-agentuniverse" + ) + except metadata.PackageNotFoundError: + pytest.skip("package not installed in this environment") + assert distribution.version + assert "agentuniverse" in { + entry.name for entry in distribution.entry_points + } + + def test_every_source_module_is_importable(self) -> None: + import opentelemetry.instrumentation.agentuniverse as package + + for name in ("_common", "_agent", "_llm", "_tool"): + module = __import__(f"{package.__name__}.{name}", fromlist=[name]) + assert module.__name__ == f"{package.__name__}.{name}" + + def test_wrapper_globals_are_restored_by_the_autouse_fixture(self) -> None: + for name in WRAPPER_GLOBALS: + wrapper = getattr(trace_module, name) + assert getattr(wrapper, "__self__", None) is None, name + + +def test_agents_used_by_the_matrix_are_real_framework_objects() -> None: + from agentuniverse.agent.action.tool.tool import Tool + from agentuniverse.agent.agent import Agent + + assert issubclass(RichAgent, Agent) + assert issubclass(StubTool, Tool) + assert issubclass(SecretLLM, StubLLM) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_llm_lifecycle_alignment.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_llm_lifecycle_alignment.py new file mode 100644 index 000000000..6eeee17b3 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_llm_lifecycle_alignment.py @@ -0,0 +1,137 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Review-body follow-ups for the LLM layer. + +Three points from the review body are pinned here: + +* the teardown of a finished LLM call belongs to the shared handler -- its + ``abandon_llm`` is the documented best-effort cleanup -- so the plugin should + not detach a token or end a span of its own; +* ``gen_ai.request.model`` names the model the call asks for, while + ``au.llm.name`` keeps the component name the framework itself reports; +* a span that cannot be created at all (``tracer.start_span`` raising before + anything is attached) leaves the call running uninstrumented rather than + half-traced, and the next call of the same trace is unaffected. + +The stubs are driven through the real ``@trace_llm`` seam, so what is under +test is the framework's wrapper resolution. +""" + +from typing import Any + +from agentuniverse.base.annotation.trace import trace_llm +from agentuniverse.llm.llm_output import LLMOutput + +from opentelemetry.instrumentation.agentuniverse import _common as common +from opentelemetry.util.genai.extended_handler import ExtendedTelemetryHandler + +from .conftest import StubLLM +from .test_instrumentor import ( + LLM_SPAN, + capture_on, + layer_records, + one_layer_record, +) + + +class ModellingLLM(StubLLM): + """A component whose configured model differs from its component name.""" + + name = "modelling_llm" + model_name = "gpt-4o" + + @trace_llm + def call(self, prompt: str, **kwargs: Any) -> LLMOutput: + return LLMOutput(text="answer", finish_reason="stop") + + +class TestHandlerOwnedTeardown: + def test_finalize_uses_the_handlers_own_cleanup( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + """The plugin hands the teardown to ``abandon_llm``.""" + seen: list = [] + original = ExtendedTelemetryHandler.abandon_llm + + def spy(self: Any, invocation: Any) -> Any: + seen.append(invocation) + return original(self, invocation) + + monkeypatch.setattr(ExtendedTelemetryHandler, "abandon_llm", spy) + with capture_on(): + StubLLM().call(prompt="hello") + + assert seen, "the handler's own cleanup was never used" + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.status"] == "success" + + +class TestGenAiIdentity: + def test_request_model_is_the_model_not_the_component_name( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + ModellingLLM().call(prompt="hello") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.name"] == "modelling_llm" + assert record.gen_ai["gen_ai.request.model"] == "gpt-4o" + + def test_unknown_model_is_not_replaced_with_component_name( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + monkeypatch.delattr(StubLLM, "model_name", raising=False) + with capture_on(): + StubLLM().call(prompt="hello") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert "gen_ai.request.model" not in record.gen_ai + + def test_provider_name_is_the_channel( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + ModellingLLM().call(prompt="hello") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.gen_ai["gen_ai.provider.name"] == "test_channel" + + +class TestSpanCreationFailure: + def test_a_span_that_cannot_be_created_leaves_the_call_uninstrumented( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + def boom(self: Any, *args: Any, **kwargs: Any) -> None: + raise RuntimeError("no span") + + monkeypatch.setattr(common.LayerSpan, "__init__", boom) + with capture_on(): + result = StubLLM().call(prompt="hello") + assert result.text, "the business call still answers" + assert layer_records(loongsuite_harness, LLM_SPAN) == [] + + def test_the_next_call_of_the_trace_is_a_root_span( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + def boom(self: Any, *args: Any, **kwargs: Any) -> None: + raise RuntimeError("no span") + + with monkeypatch.context() as patch: + patch.setattr(common.LayerSpan, "__init__", boom) + with capture_on(): + StubLLM().call(prompt="hello") + with capture_on(): + StubLLM().call(prompt="hello") + + records = layer_records(loongsuite_harness, LLM_SPAN) + assert len(records) == 1, [r.name for r in records] + assert records[0].parent is None diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_llm_review_followups.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_llm_review_followups.py new file mode 100644 index 000000000..fd82ca496 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_llm_review_followups.py @@ -0,0 +1,334 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Review follow-ups for the LLM layer. + +Three gaps the review found are pinned down here: an async stream that is +awaited and then never read, a call whose usage only reaches ``LLMOutput.raw``, +and the chunk text of a stream that carries empty chunks. The fourth covers the +``LayerSpan.attach`` rollback: a span whose attach fails must not stay attached +to the caller's context, or the next call of the same trace is filed under a +span that never ends. + +Every stub is driven through the real ``@trace_llm`` seam, so the behaviour +under test is the framework's own wrapper resolution rather than a stand-in for +it. The stubs live here instead of in ``conftest`` because only these tests need +them. +""" + +import asyncio +from typing import Any, AsyncIterator, Dict + +from agentuniverse.base.annotation.trace import trace_llm +from agentuniverse.llm.llm_output import LLMOutput, TokenUsage + +from opentelemetry.instrumentation.agentuniverse import _common as common + +from .conftest import StubLLM, StubTool +from .test_instrumentor import ( + LLM_SPAN, + TOOL_SPAN, + capture_on, + layer_records, + one_layer_record, +) + + +class AsyncStreamingLLM(StubLLM): + """A ``@trace_llm`` coroutine that returns an async iterator. + + ``trace_llm`` sends a coroutine function to the async wrapper, and a result + that is not an ``LLMOutput`` is what the wrapper treats as a stream, so this + is the shape of a real streaming ``acall``. + """ + + name = "async_stream_llm" + + @trace_llm + async def call( + self, prompt: str, **kwargs: Any + ) -> AsyncIterator[LLMOutput]: + async def stream() -> AsyncIterator[LLMOutput]: + yield LLMOutput(text="part-1") + yield LLMOutput( + text="part-2", usage=TokenUsage(text_in=4, text_out=6) + ) + + return stream() + + +class RawUsageLLM(StubLLM): + """A call whose usage only arrives in the provider payload. + + ``OpenAIStyleLLM`` keeps the provider dict in ``LLMOutput.raw`` and fills + ``LLMOutput.usage`` from it; a service that reports usage where the + framework does not look leaves ``usage`` unset while ``raw`` still carries + the numbers. + """ + + name = "raw_usage_llm" + + @trace_llm + def call(self, prompt: str, **kwargs: Any) -> LLMOutput: + return LLMOutput( + text=f"raw:{prompt}", + raw={ + "id": "resp-1", + "usage": { + "prompt_tokens": 20, + "completion_tokens": 5, + "total_tokens": 25, + }, + }, + finish_reason="stop", + ) + + +class EmptyTextChunkLLM(StubLLM): + """A stream whose empty chunks sit between the chunks that carry text. + + A role-only or usage-only chunk is a real framework payload with + ``text=""``; it contributes nothing, and certainly not its own repr. + """ + + name = "empty_text_chunk_llm" + + @trace_llm + def call(self, prompt: str, **kwargs: Any): + yield LLMOutput(text="hello ") + yield LLMOutput(text="") + yield LLMOutput(text="world", usage=TokenUsage(text_in=1, text_out=2)) + + +class AsyncEmptyTextChunkLLM(EmptyTextChunkLLM): + """``EmptyTextChunkLLM`` over the async wrapper.""" + + name = "async_empty_text_chunk_llm" + + @trace_llm + async def call(self, prompt: str, **kwargs: Any): + async def stream() -> AsyncIterator[LLMOutput]: + yield LLMOutput(text="hello ") + yield LLMOutput(text="") + yield LLMOutput( + text="world", usage=TokenUsage(text_in=1, text_out=2) + ) + + return stream() + + +def span_named(harness: Any, prefix: str) -> Any: + """The one exported span whose name starts with ``prefix``.""" + spans = [s for s in harness.spans() if s.name.startswith(prefix)] + assert len(spans) == 1, [s.name for s in harness.spans()] + return spans[0] + + +class TestAsyncStreamLifecycle: + def test_async_stream_releases_the_context_before_it_is_read( + self, loongsuite_harness: Any + ) -> None: + """The caller resumes outside the LLM span, so a later call is a root.""" + stream = None + + async def drive() -> Any: + nonlocal stream + stream = await AsyncStreamingLLM().call(prompt="hello") + # Before reading anything the caller does other work; it must not + # be filed under the still-open LLM span. + StubTool().run(query="hello") + await stream.aclose() + + asyncio.run(drive()) + llm = span_named(loongsuite_harness, LLM_SPAN) + tool = span_named(loongsuite_harness, TOOL_SPAN) + assert tool.parent is None + assert tool.parent != llm.context + + def test_async_stream_closed_without_being_read_finalizes_once( + self, loongsuite_harness: Any + ) -> None: + """A dropped stream still closes its span, with the empty output.""" + + async def drive() -> None: + stream = await AsyncStreamingLLM().call(prompt="hello") + await stream.aclose() + + asyncio.run(drive()) + records = layer_records(loongsuite_harness, LLM_SPAN) + assert len(records) == 1, [r.name for r in records] + assert records[0].au["au.llm.streaming"] is True + assert records[0].au["au.llm.status"] == "success" + + def test_async_stream_read_fully_finalizes_once( + self, loongsuite_harness: Any + ) -> None: + """The ordinary case still ends exactly one span with both chunks.""" + + async def drive() -> None: + stream = await AsyncStreamingLLM().call(prompt="hello") + assert [chunk.text for chunk in [c async for c in stream]] == [ + "part-1", + "part-2", + ] + + with capture_on(): + asyncio.run(drive()) + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.output"] == "part-1part-2" + assert record.au["au.llm.usage.total_tokens"] == 10 + + def test_async_stream_consumed_in_another_task_finalizes_once( + self, loongsuite_harness: Any + ) -> None: + """A stream handed to another task closes the same single span.""" + + async def drive() -> None: + stream = await AsyncStreamingLLM().call(prompt="hello") + + async def consume() -> None: + async for _ in stream: + pass + + await asyncio.create_task(consume()) + + with capture_on(): + asyncio.run(drive()) + records = layer_records(loongsuite_harness, LLM_SPAN) + assert len(records) == 1, [r.name for r in records] + assert records[0].au["au.llm.output"] == "part-1part-2" + assert records[0].au["au.llm.status"] == "success" + assert records[0].au["au.llm.usage.total_tokens"] == 10 + + +class TestLayerSpanAttachRollback: + def test_attach_failure_leaves_no_attached_span( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + """A span whose attach fails is closed and detached before the raise.""" + original = common.reset_token_usage + failing: Dict[str, bool] = {"on": True} + + def flaky(span_id: Any = None) -> None: + if failing["on"]: + raise RuntimeError("token bookkeeping exploded") + original(span_id) + + monkeypatch.setattr(common, "reset_token_usage", flaky) + StubLLM().call(prompt="first") + failing["on"] = False + StubLLM().call(prompt="second") + + spans = [ + s + for s in loongsuite_harness.spans() + if s.name.startswith(LLM_SPAN) + ] + complete = [ + s for s in spans if s.attributes.get("au.llm.status") == "success" + ] + assert len(complete) == 1, [s.name for s in spans] + # The failed attach must have detached its span, or this one would be + # its child. + assert complete[0].parent is None + + +class TestUsageAndChunkText: + def test_usage_from_the_raw_payload_reaches_both_namespaces( + self, loongsuite_harness: Any + ) -> None: + RawUsageLLM().call(prompt="hello") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.usage.total_tokens"] == 25 + assert record.au["au.llm.usage.prompt_tokens"] == 20 + assert record.au["au.llm.usage.completion_tokens"] == 5 + assert record.gen_ai.get("gen_ai.usage.input_tokens") == 20 + assert record.gen_ai.get("gen_ai.usage.output_tokens") == 5 + + def test_usage_of_reads_the_real_adapter_output( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + """The real adapter's construction path, not a hand-built TokenUsage.""" + import httpx + from agentuniverse.llm.openai_style_llm import OpenAIStyleLLM + from openai import OpenAI + + from opentelemetry.instrumentation.agentuniverse._llm import ( + usage_of, + ) + + # The SDK may fall back to constructing a default OpenAI client; provide + # a synthetic key so the offline MockTransport path is taken. + monkeypatch.setenv("OPENAI_API_KEY", "synthetic") + + completion = { + "id": "chatcmpl-1", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 20, + "completion_tokens": 5, + "total_tokens": 25, + }, + } + adapter = OpenAIStyleLLM( + name="adapter_llm", + model_name="synthetic-model", + api_base="http://provider.invalid/v1", + ) + with OpenAI( + api_key="synthetic", + base_url="http://provider.invalid/v1", + http_client=httpx.Client( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json=completion) + ) + ), + ) as client: + adapter.client = client + # On agentUniverse 0.1.0a2, OpenAIStyleLLM._get_client() resolves + # via _client_pool[api_base] instead of self.client when api_base + # is set. Pre-populate the pool so the MockTransport client is used. + adapter._client_pool["http://provider.invalid/v1"] = client + result = adapter.call( + messages=[{"role": "user", "content": "hello"}] + ) + assert result.raw["usage"]["total_tokens"] == 25 + assert usage_of(result).total_tokens == 25 + + def test_empty_text_chunks_contribute_nothing( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + list(EmptyTextChunkLLM().call(prompt="hello")) + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.output"] == "hello world" + + def test_empty_text_chunks_contribute_nothing_async( + self, loongsuite_harness: Any + ) -> None: + async def drive() -> None: + stream = await AsyncEmptyTextChunkLLM().call(prompt="hello") + async for _ in stream: + pass + + with capture_on(): + asyncio.run(drive()) + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.output"] == "hello world" diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_review_round3.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_review_round3.py new file mode 100644 index 000000000..47368191b --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_review_round3.py @@ -0,0 +1,628 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Review follow-ups for the third review round. + +One test per finding: a stable framework without a usable +``TokenUsage.from_anthropic``, a streamed chunk whose telemetry read fails, the +``LLMStreamEvent.text_delta`` payload of 0.1.0a2, the output messages of a +tool-planning round, the agent output boundary, the per-call model override and +the ReAct STEP spans of an agent's rounds. + +Every stub is driven through the real ``@trace_llm``/``@trace_agent`` seams, so +what is under test is the framework's own wrapper resolution rather than a +stand-in for it. +""" + +import asyncio +import json +from typing import Any, AsyncIterator, List, Optional + +import pytest +from agentuniverse.agent.agent_model import AgentModel +from agentuniverse.base.annotation.trace import trace_llm +from agentuniverse.llm import llm_output as framework_outputs +from agentuniverse.llm.llm_output import LLMOutput, TokenUsage + +LLMStreamEvent = getattr(framework_outputs, "LLMStreamEvent", None) +StreamReducer = getattr(framework_outputs, "StreamReducer", None) + +from opentelemetry.instrumentation.agentuniverse import _llm as llm_layer + +from .conftest import StubAgent, StubLLM, build_agent +from .test_instrumentor import ( + AGENT_SPAN, + capture_on, + one_layer_record, + run_agent, +) + +STEP_SPAN_NAME = "react step" +LLM_SPAN = "au.llm." + + +def span_names(harness: Any) -> List[str]: + return [record.name for record in harness.records()] + + +def step_records(harness: Any) -> List[Any]: + return [ + record for record in harness.records() if record.name == STEP_SPAN_NAME + ] + + +def output_messages(record: Any) -> List[Any]: + """The ``gen_ai.output.messages`` payload of a span, decoded.""" + payload = record.gen_ai.get("gen_ai.output.messages") + assert payload is not None, sorted(record.gen_ai) + return json.loads(payload) + + +def message_text(messages: List[Any]) -> str: + return "".join( + part.get("content", "") + for message in messages + for part in message.get("parts", []) + ) + + +# --------------------------------------------------------------------------- +# P1: a stable framework has no ``TokenUsage.from_anthropic`` +# --------------------------------------------------------------------------- + + +class RawProviderUsageLLM(StubLLM): + """A call whose usage only arrives in the provider payload.""" + + name = "raw_provider_usage_llm" + + @trace_llm + def call(self, prompt: str, **kwargs: Any) -> LLMOutput: + return LLMOutput( + text=f"raw:{prompt}", + raw={ + "id": "resp-1", + "usage": { + "prompt_tokens": 20, + "completion_tokens": 5, + "total_tokens": 25, + }, + }, + finish_reason="stop", + ) + + +class StableLikeTokenUsage: + """The stable release's converter surface: only ``from_openai``.""" + + from_openai = staticmethod(TokenUsage.from_openai) + + +class TestStableFrameworkUsageConversion: + def test_usage_is_read_without_the_anthropic_converter( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + """0.0.19.1 only has ``from_openai``; its payload must still convert. + + The installed build here is 0.1.0a2, which has both converters, so the + stable release's converter surface is stood in for instead of the + installed one. + """ + monkeypatch.setattr(llm_layer, "TokenUsage", StableLikeTokenUsage) + RawProviderUsageLLM().call(prompt="hello") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.usage.total_tokens"] == 25 + assert record.au["au.llm.usage.prompt_tokens"] == 20 + assert record.au["au.llm.usage.completion_tokens"] == 5 + + def test_raw_usage_does_not_depend_on_framework_converter( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + """Provider totals remain authoritative without an SDK converter.""" + + class NoConverters: + """A release that offers no converter at all.""" + + monkeypatch.setattr(llm_layer, "TokenUsage", NoConverters) + result = RawProviderUsageLLM().call(prompt="hello") + assert result.text == "raw:hello" + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.usage.total_tokens"] == 25 + + +# --------------------------------------------------------------------------- +# P1: a failing collector must not disturb the stream +# --------------------------------------------------------------------------- + + +class ThreeChunkLLM(StubLLM): + """A stream of three chunks the caller reads in order.""" + + name = "three_chunk_llm" + + @trace_llm + def call(self, prompt: str, **kwargs: Any): + yield LLMOutput(text="one ") + yield LLMOutput(text="two ") + yield LLMOutput(text="three", usage=TokenUsage(text_in=1, text_out=3)) + + +class AsyncThreeChunkLLM(ThreeChunkLLM): + name = "async_three_chunk_llm" + + @trace_llm + async def call(self, prompt: str, **kwargs: Any): + async def stream() -> AsyncIterator[LLMOutput]: + yield LLMOutput(text="one ") + yield LLMOutput(text="two ") + yield LLMOutput( + text="three", usage=TokenUsage(text_in=1, text_out=3) + ) + + return stream() + + +class TestStreamChunkIsolation: + def test_a_failing_collector_leaves_the_sync_stream_untouched( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + def boom(*args: Any, **kwargs: Any) -> Any: + raise RuntimeError("collector exploded") + + monkeypatch.setattr(llm_layer, "chunk_record", boom) + chunks = list(ThreeChunkLLM().call(prompt="hello")) + assert [chunk.text for chunk in chunks] == ["one ", "two ", "three"] + # The span still closes, with the output the collector could not read. + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.status"] == "success" + + def test_a_failing_collector_leaves_the_async_stream_untouched( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + def boom(*args: Any, **kwargs: Any) -> Any: + raise RuntimeError("collector exploded") + + monkeypatch.setattr(llm_layer, "chunk_record", boom) + + async def drive() -> List[str]: + stream = await AsyncThreeChunkLLM().call(prompt="hello") + return [chunk.text async for chunk in stream] + + assert asyncio.run(drive()) == ["one ", "two ", "three"] + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.status"] == "success" + + def test_a_failing_tool_call_collector_leaves_the_stream_untouched( + self, loongsuite_harness: Any, monkeypatch: Any + ) -> None: + def boom(self: Any, chunk: Any) -> None: + raise RuntimeError("tool collector exploded") + + monkeypatch.setattr(llm_layer.ToolCallAccumulator, "feed", boom) + chunks = list(ThreeChunkLLM().call(prompt="hello")) + assert [chunk.text for chunk in chunks] == ["one ", "two ", "three"] + # The usage the chunk carried still reached the span. + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.usage.total_tokens"] == 4 + + +class HostileChunk: + """A chunk whose own shape breaks the collector reading it.""" + + @property + def text(self) -> str: + raise RuntimeError("chunk in trouble") + + +class HostileChunkLLM(StubLLM): + """A stream whose middle chunk cannot be read at all.""" + + name = "hostile_chunk_llm" + + @trace_llm + def call(self, prompt: str, **kwargs: Any): + yield LLMOutput(text="one ") + yield HostileChunk() + yield LLMOutput(text="three", usage=TokenUsage(text_in=1, text_out=2)) + + +class AsyncHostileChunkLLM(HostileChunkLLM): + name = "async_hostile_chunk_llm" + + @trace_llm + async def call(self, prompt: str, **kwargs: Any): + async def stream() -> AsyncIterator[Any]: + yield LLMOutput(text="one ") + yield HostileChunk() + yield LLMOutput( + text="three", usage=TokenUsage(text_in=1, text_out=2) + ) + + return stream() + + +class TestStreamChunkIsolation: + def test_a_chunk_that_breaks_the_collector_is_still_delivered( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + chunks = list(HostileChunkLLM().call(prompt="hello")) + assert len(chunks) == 3, chunks + assert chunks[1].__class__ is HostileChunk + assert [type(chunk).__name__ for chunk in chunks] == [ + "LLMOutput", + "HostileChunk", + "LLMOutput", + ] + assert chunks[0].text == "one " + assert chunks[2].text == "three" + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.status"] == "success" + assert record.au["au.llm.output"] == "one three" + + def test_an_async_chunk_that_breaks_the_collector_is_still_delivered( + self, loongsuite_harness: Any + ) -> None: + async def drive() -> List[Any]: + stream = await AsyncHostileChunkLLM().call(prompt="hello") + return [chunk async for chunk in stream] + + with capture_on(): + chunks = asyncio.run(drive()) + assert len(chunks) == 3, chunks + assert chunks[1].__class__ is HostileChunk + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.status"] == "success" + + +# --------------------------------------------------------------------------- +# P2: a 0.1.0a2 stream carries its text in ``text_delta`` +# --------------------------------------------------------------------------- + + +class StreamEventLLM(StubLLM): + """A 0.1.0a2 stream: ``LLMStreamEvent``s instead of ``LLMOutput`` chunks.""" + + name = "stream_event_llm" + + @trace_llm + def call(self, prompt: str, **kwargs: Any): + yield LLMStreamEvent.text("part-1") + yield LLMStreamEvent.text("part-2") + yield LLMStreamEvent.done( + finish_reason="stop", usage=TokenUsage(text_in=2, text_out=5) + ) + + +@pytest.mark.skipif( + LLMStreamEvent is None, reason="Stable framework has no LLMStreamEvent API" +) +class TestStreamEventTextDelta: + def test_text_delta_reaches_the_span_output( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + events = list(StreamEventLLM().call(prompt="hello")) + assert [event.text_delta for event in events] == [ + "part-1", + "part-2", + None, + ] + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.output"] == "part-1part-2" + assert message_text(output_messages(record)) == "part-1part-2" + + def test_the_frameworks_own_reducer_sees_the_same_text( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + reducer = StreamReducer() + for event in StreamEventLLM().call(prompt="hello"): + reducer.feed(event) + output = reducer.build() + assert output.text == "part-1part-2" + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.au["au.llm.output"] == "part-1part-2" + + +# --------------------------------------------------------------------------- +# The tool-planning round's output messages +# --------------------------------------------------------------------------- + + +class ToolPlanningLLM(StubLLM): + """A round that answers with a tool call, the way the framework reduces it.""" + + name = "tool_planning_llm" + #: ``Agent.judge_stream`` reads both of these to decide the call shape. + streaming = False + _channel_instance = None + + def __init__(self) -> None: + self.calls = 0 + + @trace_llm + def call( + self, prompt: str = "", messages: Any = None, **kwargs: Any + ) -> LLMOutput: + self.calls += 1 + if self.calls > 1: + return LLMOutput( + text="answered", + usage=TokenUsage(text_in=2, text_out=3), + finish_reason="stop", + ) + return LLMOutput( + text="", + message=_tool_planning_message(), + finish_reason="tool_calls", + raw={ + "choices": [ + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": { + "name": "stub_tool", + "arguments": '{"query": "hello"}', + }, + } + ], + }, + } + ], + }, + ) + + +def _tool_planning_message(): + """Assistant message carrying a tool call, for SDKs that ship the + native tool-calling loop (0.1.0a2+). Older agentUniverse has no + ``ToolCall`` surface; on those the round is driven through ``raw`` and + the loop tests are skipped via the ``run_tool_calling_loop`` gate. + """ + ToolCall = getattr(framework_outputs, "ToolCall", None) + Message = getattr(framework_outputs, "Message", None) + ChatMessageEnum = getattr(framework_outputs, "ChatMessageEnum", None) + if ToolCall is None or Message is None: + return None + tool_call = ToolCall( + id="call-1", + type="function", + function={"name": "stub_tool", "arguments": '{"query": "hello"}'}, + ) + role = ( + getattr(ChatMessageEnum, "ASSISTANT", "assistant") + if ChatMessageEnum + else "assistant" + ) + return Message(type=role, content="", tool_calls=[tool_call]) + + +class TestToolPlanningOutputMessages: + def test_a_tool_call_round_reports_its_calls( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + ToolPlanningLLM().call(prompt="hello") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + messages = output_messages(record) + assert messages, "the tool-planning round reported no output messages" + assert messages[0]["finish_reason"] == "tool_calls" + serialised = json.dumps(messages) + assert "stub_tool" in serialised + assert "hello" in serialised + + +# --------------------------------------------------------------------------- +# The agent output boundary +# --------------------------------------------------------------------------- + + +class EnvelopeAgent(StubAgent): + """The envelope ``AgentTemplate.customized_execute`` really returns.""" + + def execute(self, input_object: Any, agent_input: dict) -> dict: + return { + **agent_input, + "output": f"echo:{input_object.get_data('input')}", + } + + +class TestAgentOutputBoundary: + def test_the_span_reports_the_answer_not_the_envelope( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + run_agent(build_agent("envelope_agent", EnvelopeAgent)) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert json.loads(record.au["au.agent.output"]) == "echo:hello" + assert message_text(output_messages(record)) == "echo:hello" + + def test_the_span_never_echoes_the_agent_input( + self, loongsuite_harness: Any + ) -> None: + with capture_on(): + run_agent(build_agent("envelope_agent", EnvelopeAgent)) + record = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert "kwargs" not in record.au["au.agent.output"] + # The input keeps its own carrier. + assert record.au["au.agent.input"] + + +# --------------------------------------------------------------------------- +# The per-call model override +# --------------------------------------------------------------------------- + + +class ConfiguredModelLLM(StubLLM): + """An LLM configured with one model, callable with another.""" + + name = "configured_model_llm" + model_name = "qwen-turbo" + + +class TestPerCallModelOverride: + def test_a_call_that_names_a_model_reports_that_model( + self, loongsuite_harness: Any + ) -> None: + ConfiguredModelLLM().call(prompt="hello", model="qwen-plus") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.gen_ai["gen_ai.request.model"] == "qwen-plus" + # The component's own name stays where it was. + assert record.au["au.llm.name"] == "configured_model_llm" + + def test_a_call_without_an_override_reports_the_configured_model( + self, loongsuite_harness: Any + ) -> None: + ConfiguredModelLLM().call(prompt="hello") + record = one_layer_record(loongsuite_harness, LLM_SPAN) + assert record.gen_ai["gen_ai.request.model"] == "qwen-turbo" + + +# --------------------------------------------------------------------------- +# ReAct STEP spans +# --------------------------------------------------------------------------- + + +class RoundDrivingAgent(StubAgent): + """An agent whose body drives the framework's own round hooks.""" + + rounds: int = 2 + explode_at: Optional[int] = None + + def execute(self, input_object: Any, agent_input: dict) -> dict: + llm = StubLLM() + for index in range(self.rounds): + self.on_before_llm_round(None, round_index=index) + if self.explode_at == index: + # The round's LLM call never returns, so the matching + # ``on_after_llm_round`` never runs and the step stays open. + raise RuntimeError("round exploded") + output = llm.call(prompt="hello") + self.on_after_llm_round(None, output, round_index=index) + return {"output": "done"} + + +class ToolCallingLoopAgent(StubAgent): + """An agent whose body runs the framework's real tool-calling loop.""" + + def execute(self, input_object: Any, agent_input: dict) -> dict: + llm = ToolPlanningLLM() + output = self.run_tool_calling_loop( + llm, LoopContext(), input_object, max_iterations=3 + ) + return {"output": output.text} + + +class LoopContext: + """The slice of ``AgentContext`` the tool-calling loop uses.""" + + def __init__(self) -> None: + self.tools_schema: list = [] + self.messages: list = [{"role": "user", "content": "hello"}] + + def build_messages(self) -> list: + return list(self.messages) + + def append_message(self, message: Any) -> None: + self.messages.append(message) + + +def build_round_agent( + name: str, rounds: int = 2, explode_at: Optional[int] = None +) -> Any: + agent = RoundDrivingAgent() + agent.rounds = rounds + agent.explode_at = explode_at + agent.agent_model = AgentModel(info={"name": name}) + return agent + + +def _ensure_stub_tool_registered() -> None: + """Register the real ``stub_tool`` so the native loop's + ``execute_tool_call`` dispatches to a successful tool, not a + not-found error path.""" + from agentuniverse.agent.action.tool.tool_manager import ToolManager + + from .conftest import StubTool + + manager = ToolManager() + probe = StubTool() + code = probe.get_instance_code() + if code not in manager._instance_obj_map: + manager.register(code, StubTool()) + + +def build_loop_agent(name: str) -> Any: + _ensure_stub_tool_registered() + agent = ToolCallingLoopAgent() + agent.agent_model = AgentModel(info={"name": name}) + return agent + + +@pytest.mark.skipif( + not hasattr(StubAgent, "run_tool_calling_loop"), + reason="Stable ReAct uses LangChain, without native round hook APIs", +) +class TestReactSteps: + def test_each_round_gets_its_own_step_span( + self, loongsuite_harness: Any + ) -> None: + run_agent(build_round_agent("round_agent")) + steps = step_records(loongsuite_harness) + assert len(steps) == 2, span_names(loongsuite_harness) + assert [step.gen_ai["gen_ai.react.round"] for step in steps] == [1, 2] + for step in steps: + assert step.kind == "INTERNAL" + assert step.gen_ai["gen_ai.span.kind"] == "STEP" + assert step.gen_ai["gen_ai.operation.name"] == "react" + assert step.au["au.span.kind"] == "step" + assert step.parent == "au.agent.round_agent" + + def test_the_step_closes_when_the_round_aborts( + self, loongsuite_harness: Any + ) -> None: + try: + run_agent(build_round_agent("aborting_agent", explode_at=0)) + except RuntimeError: + pass + steps = step_records(loongsuite_harness) + assert len(steps) == 1, span_names(loongsuite_harness) + assert steps[0].status == "ERROR" + agent = one_layer_record(loongsuite_harness, AGENT_SPAN) + assert agent.au["au.agent.status"] == "error" + + def test_the_real_loop_records_one_step_per_round( + self, loongsuite_harness: Any + ) -> None: + run_agent(build_loop_agent("loop_agent")) + steps = step_records(loongsuite_harness) + assert len(steps) == 2, span_names(loongsuite_harness) + assert [step.gen_ai["gen_ai.react.round"] for step in steps] == [1, 2] + assert steps[0].gen_ai["gen_ai.react.finish_reason"] == "tool_calls" + assert steps[1].gen_ai["gen_ai.react.finish_reason"] == "stop" + + def test_the_round_usage_still_reaches_the_agent( + self, loongsuite_harness: Any + ) -> None: + run_agent(build_loop_agent("loop_agent")) + agent = one_layer_record(loongsuite_harness, AGENT_SPAN) + # Only the answering round reported usage (2 in, 3 out). + assert agent.au["au.agent.usage.total_tokens"] == 5 diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_runtime_contract.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_runtime_contract.py new file mode 100644 index 000000000..108978866 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_runtime_contract.py @@ -0,0 +1,344 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import importlib.metadata as _metadata +import json + +import httpx +import pytest +from agentuniverse.agent.agent import Agent +from agentuniverse.base.annotation import trace as trace_module +from agentuniverse.llm.llm_channel.llm_channel import LLMChannel +from openai import OpenAI + +from opentelemetry.instrumentation.agentuniverse import ( + AgentUniverseInstrumentor, +) +from opentelemetry.instrumentation.agentuniverse._step import ( + REACT_STEP_HOOKS, +) + +from .baseline import collect_metrics, counter_value +from .conftest import NATIVE_WRAPPER_GLOBALS +from .runtime_app import Workload +from .runtime_provider import MODEL, prompt +from .runtime_provider import Provider as ToolProvider + + +def _is_alpha(): + raw = _metadata.version("agentUniverse") + parts = raw.split(".") + return (int(parts[0]), int(parts[1])) >= (0, 1) + + +_IS_ALPHA = _is_alpha() + + +def _stream_event_usage(chunk): + """Read token usage from a stream chunk across SDK layouts. + + stable 0.0.19.1 yields raw OpenAI chunk dicts on ``chunk.raw``; alpha + 0.1.0a2 yields LLMStreamEvent objects that carry a typed ``usage`` and no + ``raw`` attribute. Both must still prove at least one usage-bearing chunk. + """ + raw = getattr(chunk, "raw", None) + if isinstance(raw, dict): + return raw.get("usage") + return getattr(chunk, "usage", None) + + +def test_stable_without_native_round_hooks_enables_all_core_wrappers(): + # stable 0.0.19.1 ships no native round hooks; alpha ships them natively. + had_native_hooks = any(hasattr(Agent, name) for name in REACT_STEP_HOOKS) + original = { + name: getattr(trace_module, name) for name in NATIVE_WRAPPER_GLOBALS + } + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument() + assert instrumentor.is_instrumented_by_opentelemetry + assert all( + getattr(trace_module, name) is not value + for name, value in original.items() + ) + instrumentor.uninstrument() + assert all( + getattr(trace_module, name) is value + for name, value in original.items() + ) + if had_native_hooks: + # alpha: our wrappers must be removed, the native hooks restored. + assert all(hasattr(Agent, name) for name in REACT_STEP_HOOKS) + else: + # stable: no native round hooks and still none after uninstall. + assert not any(hasattr(Agent, name) for name in REACT_STEP_HOOKS) + + +def test_partial_round_hook_install_is_atomic(monkeypatch): + original = { + name: getattr(trace_module, name) for name in NATIVE_WRAPPER_GLOBALS + } + # stable 0.0.19.1 ships no native round hooks; alpha ships them. Snapshot + # per-name existence/identity; only names that are missing get a synthetic + # seam lambda, so existing hooks keep their identity. + native_hooks = { + name: getattr(Agent, name, None) for name in REACT_STEP_HOOKS + } + for name in REACT_STEP_HOOKS: + if not hasattr(Agent, name): + monkeypatch.setattr( + Agent, name, lambda *a, **k: None, raising=False + ) + # Snapshot hooks as seen by the install attempt (synthetic + native). + after_seam = {name: getattr(Agent, name) for name in REACT_STEP_HOOKS} + # Force the second round-hook assignment to fail mid-install; the whole + # install must roll back, leaving neither instrumentation nor side effects. + # Class attribute assignment goes through the metaclass __setattr__. + metaclass = type(Agent) + real_setattr = metaclass.__setattr__ + events: list[str] = [] + after_failures = {"n": 0} + + def fake_setattr(cls, name, value): + if ( + cls is Agent + and name == "on_after_llm_round" + and after_failures["n"] == 0 + ): + after_failures["n"] += 1 + events.append("after-assign-failed") + raise RuntimeError("boom: second hook assignment failed") + events.append("assign:" + name) + return real_setattr(cls, name, value) + + monkeypatch.setattr(metaclass, "__setattr__", fake_setattr) + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument() + assert not instrumentor.is_instrumented_by_opentelemetry + # Exactly one failure, on the second hook; the first hook installed first. + assert events.count("after-assign-failed") == 1 + assert events[0].startswith("assign:on_before_llm_round") + assert all( + getattr(trace_module, name) is value + for name, value in original.items() + ) + # Mid-install rollback restored hooks to the seam state (not half-installed). + assert all( + getattr(Agent, name) is value for name, value in after_seam.items() + ) + # Explicit undo: names that existed keep their original object; names that + # were absent (stable) are absent again after teardown. + monkeypatch.undo() + for name, value in native_hooks.items(): + if value is None: + assert not hasattr(Agent, name) + else: + assert getattr(Agent, name) is value + assert metaclass.__setattr__ is real_setattr + + +@pytest.mark.parametrize("streaming", [False, True]) +def test_stable_sdk_raw_tool_calls_and_fragments( + streaming, loongsuite_harness, monkeypatch +): + """A real SDK/channel call; this does not claim native ReAct dispatch IDs.""" + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + backend = ToolProvider() + channel = LLMChannel( + channel_name="openai", + channel_model_name=MODEL, + channel_api_base="http://provider.invalid/v1", + ext_params={ + "reasoning_effort": "high", + "stream_options": {"include_usage": True}, + }, + ) + channel.channel_model_config = dict( + streaming=streaming, temperature=0, max_tokens=512 + ) + with OpenAI( + api_key="synthetic", + base_url="http://provider.invalid/v1", + max_retries=0, + http_client=httpx.Client( + transport=httpx.MockTransport(backend.respond) + ), + ) as client: + channel.client = client + result = channel.call( + messages=[{"role": "user", "content": prompt("rawStable", 1)}] + ) + if streaming: + chunks = list(result) + assert any(_stream_event_usage(c) for c in chunks) + else: + assert result.raw["choices"][0]["message"]["tool_calls"] + spans = loongsuite_harness.spans() + assert len(spans) == 1 + attrs = spans[0].attributes + messages = json.loads(attrs["gen_ai.output.messages"]) + calls = [ + p + for message in messages + for p in message["parts"] + if p["type"] == "tool_call" + ] + expected = backend.responses[0]["tool_calls"] + assert {p["id"] for p in calls} == {c["id"] for c in expected} + for part, call in zip(calls, expected): + assert part["name"] == call["function"]["name"] + assert part["arguments"] == json.loads(call["function"]["arguments"]) + assert list(attrs["gen_ai.response.finish_reasons"]) == ["tool_calls"] + assert attrs["gen_ai.usage.input_tokens"] == 20 + assert attrs["gen_ai.usage.output_tokens"] == 10 + + +@pytest.mark.parametrize("mode", ["SPAN_ONLY", "NO_CONTENT"]) +@pytest.mark.parametrize("reasoning", [True, False]) +def test_stable_native_react_conversations( + mode, reasoning, monkeypatch, configured_providers, session_exporter +): + rounds_per_turn = 3 if _IS_ALPHA else 4 + expected_llm_calls = rounds_per_turn * 3 * 2 + expected_tool_spans = 18 + expected_agent_in = rounds_per_turn * 20 + expected_agent_out = rounds_per_turn * 10 + expected_total_tokens = expected_llm_calls * 30 + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", mode + ) + provider, meter_provider, reader = configured_providers + instrumentor = AgentUniverseInstrumentor() + instrumentor.instrument( + tracer_provider=provider, meter_provider=meter_provider + ) + assert instrumentor.is_instrumented_by_opentelemetry + workload = Workload(reasoning=reasoning) + result = asyncio.run(workload.run(provider)) + assert result["request_count"] == expected_llm_calls + assert len(result["scenarios"]) == 6 + spans = session_exporter.get_finished_spans() + llms = [s for s in spans if s.attributes.get("gen_ai.span.kind") == "LLM"] + agents = [ + s for s in spans if s.attributes.get("gen_ai.span.kind") == "AGENT" + ] + tools = [ + s for s in spans if s.attributes.get("gen_ai.span.kind") == "TOOL" + ] + assert (len(llms), len(agents), len(tools)) == ( + expected_llm_calls, + 6, + expected_tool_spans, + ) + if not _IS_ALPHA: + assert all("gen_ai.tool.call.id" not in s.attributes for s in tools) + by_id = {s.context.span_id: s for s in spans} + sessions = {"auSync", "auStreamA", "auStreamB"} + for span in llms + tools + agents: + assert span.attributes["gen_ai.session.id"] in sessions + assert span.parent is not None + parent = by_id[span.parent.span_id] + assert parent.context.trace_id == span.context.trace_id + if span in llms + tools: + assert parent.attributes.get("gen_ai.span.kind") in ( + "AGENT", + "STEP", + ) + assert ( + parent.attributes["gen_ai.session.id"] + == span.attributes["gen_ai.session.id"] + ) + expected = { + r.get("id", str(i)): r + for i, r in enumerate(workload.backend.responses) + } + for span in llms: + attrs = span.attributes + assert attrs["gen_ai.usage.input_tokens"] == 20 + assert attrs["gen_ai.usage.output_tokens"] == 10 + assert attrs["au.llm.usage.total_tokens"] == 30 + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.request.model"] == MODEL + assert attrs["gen_ai.request.temperature"] == 0 + assert attrs["gen_ai.usage.cache_read.input_tokens"] == 5 + resp_id = attrs.get("gen_ai.response.id", "") + response = expected.get(resp_id) + if response: + assert attrs["gen_ai.session.id"] == response["worker"] + if mode == "SPAN_ONLY": + in_raw = attrs.get("gen_ai.input.messages") + out_raw = attrs.get("gen_ai.output.messages") + if in_raw is not None and response: + inputs = json.loads(in_raw) + serialized = json.dumps(inputs) + worker = response["worker"] + assert worker in serialized + if out_raw is not None and response: + outputs = json.loads(out_raw) + parts = [pt for m in outputs for pt in m["parts"]] + texts = [pt["content"] for pt in parts if pt["type"] == "text"] + reasons = [ + pt["content"] for pt in parts if pt["type"] == "reasoning" + ] + if response.get("content") is not None: + assert "".join(texts) == response["content"] + assert reasons == ( + [response["reasoning"]] if reasoning else [] + ) + for span in tools: + attrs = span.attributes + if mode == "SPAN_ONLY": + args = json.loads(attrs["gen_ai.tool.call.arguments"]) + assert args["worker"] == attrs["gen_ai.session.id"] + actual = [ + x + for x in workload.tool_calls + if x["args"] == args and x["name"] == attrs["gen_ai.tool.name"] + ] + assert len(actual) == 1 + assert str(actual[0]["result"]) in attrs["gen_ai.tool.call.result"] + for span in agents: + assert ( + span.attributes["gen_ai.usage.input_tokens"] == expected_agent_in + ) + assert ( + span.attributes["gen_ai.usage.output_tokens"] == expected_agent_out + ) + if mode == "SPAN_ONLY": + outputs = json.loads(span.attributes["gen_ai.output.messages"]) + assert len(outputs) == 1 and len(outputs[0]["parts"]) == 1 + assert outputs[0]["parts"][0]["content"].startswith("[") + metrics = collect_metrics(reader) + assert counter_value(metrics, "llm_total_tokens") == expected_total_tokens + assert ( + counter_value(metrics, "agent_total_tokens") == expected_total_tokens + ) + if mode == "NO_CONTENT": + for span in spans: + assert not any( + k in span.attributes + for k in ( + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", + ) + ) + assert not any( + k.endswith((".input", ".output")) + for k in span.attributes + if k.startswith("au.") + ) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_runtime_lifecycle.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_runtime_lifecycle.py new file mode 100644 index 000000000..e0a1da507 --- /dev/null +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/test_runtime_lifecycle.py @@ -0,0 +1,217 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import importlib.metadata as _metadata +import inspect + +import pytest +from agentuniverse.agent.agent import Agent +from agentuniverse.base.annotation import trace as trace_module +from agentuniverse.base.annotation.trace import trace_llm +from agentuniverse.llm.llm_output import LLMOutput, TokenUsage + +from opentelemetry import context +from opentelemetry.instrumentation import agentuniverse as instrumentation +from opentelemetry.instrumentation.agentuniverse._hooks import ( + current_tool_call_id, +) +from opentelemetry.instrumentation.agentuniverse._usage import _current_call +from opentelemetry.trace import get_current_span, set_span_in_context + +from .conftest import NATIVE_WRAPPER_GLOBALS, StreamingLLM, StubLLM +from .runtime_app import Workload +from .runtime_provider import USAGE + + +def _is_alpha(): + raw = _metadata.version("agentUniverse") + parts = raw.split(".") + return (int(parts[0]), int(parts[1])) >= (0, 1) + + +def test_name_proxy_preserves_otel_context_for_genai_events( + configured_providers, +): + from opentelemetry.instrumentation.agentuniverse._common import ( + NamePreservingSpan, + ) + + provider, _, _ = configured_providers + span = provider.get_tracer("test").start_span("native-name") + try: + proxy = NamePreservingSpan(span) + current = get_current_span(set_span_in_context(proxy)) + assert current.get_span_context() == span.get_span_context() + assert current.get_span_context().is_valid + proxy.update_name("canonical-name") + assert span.name == "native-name" + finally: + span.end() + + +def test_auxiliary_install_failure_restores_every_hook_and_allows_retry( + monkeypatch, +): + original = { + name: getattr(trace_module, name) for name in NATIVE_WRAPPER_GLOBALS + } + before = getattr(Agent, "on_before_llm_round", None) + after = getattr(Agent, "on_after_llm_round", None) + install = instrumentation.install_auxiliary_hooks + + def fail(): + raise RuntimeError("synthetic auxiliary hook install failure") + + monkeypatch.setattr(instrumentation, "install_auxiliary_hooks", fail) + instrumentor = instrumentation.AgentUniverseInstrumentor() + instrumentor.instrument() + assert not instrumentor.is_instrumented_by_opentelemetry + assert getattr(Agent, "on_before_llm_round", None) is before + assert getattr(Agent, "on_after_llm_round", None) is after + assert all( + getattr(trace_module, name) is value + for name, value in original.items() + ) + monkeypatch.setattr(instrumentation, "install_auxiliary_hooks", install) + instrumentor.instrument() + assert instrumentor.is_instrumented_by_opentelemetry + + +def test_uninstrument_preserves_later_hook_owners(): + converter = inspect.getattr_static(TokenUsage, "from_openai") + instrumentor = instrumentation.AgentUniverseInstrumentor() + instrumentor.instrument() + + def third_party(*args, **kwargs): + return None + + third_converter = classmethod(third_party) + try: + TokenUsage.from_openai = third_converter + instrumentor.uninstrument() + assert ( + inspect.getattr_static(TokenUsage, "from_openai") + is third_converter + ) + finally: + TokenUsage.from_openai = converter + + +def test_provider_totals_do_not_mutate_business_usage(loongsuite_harness): + original = TokenUsage.from_openai(USAGE) + response = LLMOutput(text="answer", usage=original, raw={"usage": USAGE}) + + class NativeLLM(StubLLM): + @trace_llm + def call(self, **kwargs): + return response + + assert NativeLLM().call() is response + assert response.usage is original + assert original.total_tokens == 38 + assert ( + loongsuite_harness.spans()[0].attributes["au.llm.usage.total_tokens"] + == 30 + ) + + +def test_unread_sync_stream_close_finalizes_once(loongsuite_harness): + before = context.get_current() + stream = StreamingLLM().call(prompt="hello") + assert context.get_current() is before + stream.close() + stream.close() + assert len(loongsuite_harness.spans()) == 1 + assert context.get_current() is before + assert _current_call.get() is None + + +def test_cross_task_cancel_preserves_business_exception_and_origin( + loongsuite_harness, +): + error = asyncio.CancelledError("synthetic cancellation") + + class CancelLLM(StubLLM): + @trace_llm + async def call(self, **kwargs): + async def stream(): + yield LLMOutput(text="first") + raise error + + return stream() + + async def run(): + before = context.get_current() + stream = await CancelLLM().call() + assert context.get_current() is before + + async def consume(): + assert (await stream.__anext__()).text == "first" + with pytest.raises(asyncio.CancelledError) as raised: + await stream.__anext__() + assert raised.value is error + assert _current_call.get() is None + assert current_tool_call_id() is None + + await asyncio.create_task(consume()) + await stream.aclose() + assert context.get_current() is before + StubLLM().call(prompt="clean sibling") + assert context.get_current() is before + + asyncio.run(run()) + spans = loongsuite_harness.spans() + assert len(spans) == 2 + assert all(span.parent is None for span in spans) + + +def test_concurrent_collector_fault_keeps_business_and_clean_sibling( + monkeypatch, configured_providers, session_exporter +): + from opentelemetry.instrumentation.agentuniverse import _llm + + provider, meter_provider, _ = configured_providers + instrumentor = instrumentation.AgentUniverseInstrumentor() + instrumentor.instrument( + tracer_provider=provider, meter_provider=meter_provider + ) + original = _llm.chunk_record + failures = [] + + def fail_once(chunk): + if not failures: + failures.append("synthetic collector fault") + raise RuntimeError(failures[0]) + return original(chunk) + + monkeypatch.setattr(_llm, "chunk_record", fail_once) + before = context.get_current() + result = asyncio.run(Workload().run(provider)) + import importlib.metadata as _im + + _raw = _im.version("agentUniverse") + _parts = _raw.split(".") + _is_a = (int(_parts[0]), int(_parts[1])) >= (0, 1) + expected_calls = 18 if _is_a else 24 + assert result["request_count"] == expected_calls + assert len(failures) <= 1 + assert context.get_current() is before + assert _current_call.get() is None + assert current_tool_call_id() is None + StubLLM().call(prompt="clean sibling") + assert context.get_current() is before + spans = session_exporter.get_finished_spans() + assert len(spans) >= expected_calls + 6 + 18 + assert spans[-1].parent is None diff --git a/loongsuite-distro/src/loongsuite/distro/bootstrap_registry/loongsuite_instrumentation_agentuniverse.py b/loongsuite-distro/src/loongsuite/distro/bootstrap_registry/loongsuite_instrumentation_agentuniverse.py new file mode 100644 index 000000000..346cd01a6 --- /dev/null +++ b/loongsuite-distro/src/loongsuite/distro/bootstrap_registry/loongsuite_instrumentation_agentuniverse.py @@ -0,0 +1,30 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT. THIS FILE WAS AUTOGENERATED FROM INSTRUMENTATION PACKAGES. +# RUN `python scripts/loongsuite/generate_loongsuite_bootstrap.py` TO REGENERATE. +# +# Generated with options: +# --upstream-version: (from source) +# --loongsuite-version: (from source) + +REGISTRY = { + "source": "loongsuite", + "package": "loongsuite-instrumentation-agentuniverse", + "instrumentation": "loongsuite-instrumentation-agentuniverse==0.10.0.dev", + "libraries": [ + "agentUniverse >= 0.0.19", + ], + "default": False, +} diff --git a/pyproject.toml b/pyproject.toml index 0425351e2..bc9e87ab2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,6 +205,8 @@ ignore = [ "instrumentation-loongsuite/loongsuite-instrumentation-widesearch/tests/**/*.py" = ["E402", "F811"] "instrumentation-loongsuite/loongsuite-instrumentation-wildtool/**/*.py" = ["PLC0415"] "instrumentation-loongsuite/loongsuite-instrumentation-wildtool/tests/**/*.py" = ["E402", "F811"] +"instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/**/*.py" = ["PLC0415"] +"instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests/**/*.py" = ["E402", "F811"] # Microsoft Agent Framework is an optional dependency and its internal modules # are imported lazily to keep auto-instrumentation usable when MAF is absent. "instrumentation-loongsuite/loongsuite-instrumentation-microsoft-agent-framework/**/*.py" = ["PLC0415"] diff --git a/tox-loongsuite.ini b/tox-loongsuite.ini index b679d3c80..667c28700 100644 --- a/tox-loongsuite.ini +++ b/tox-loongsuite.ini @@ -107,6 +107,13 @@ envlist = py3{10,11,12,13}-test-loongsuite-instrumentation-a2a lint-loongsuite-instrumentation-a2a + + ; loongsuite-instrumentation-agentuniverse + ; agentUniverse 0.0.19.1 pins numpy<2 and pyarrow<17, neither of + ; which publish cp313 wheels (numpy<2 cannot even build on 3.13), so the + ; matrix stops at the highest Python agentUniverse itself supports. + py3{10,11,12}-test-loongsuite-instrumentation-agentuniverse + lint-loongsuite-instrumentation-agentuniverse ; loongsuite-instrumentation-llama-index py3{10,11,12,13}-test-loongsuite-instrumentation-llama-index lint-loongsuite-instrumentation-llama-index @@ -169,6 +176,11 @@ test_deps = opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils +agentuniverse_exporter_deps = + opentelemetry-proto@{env:CORE_REPO}\#egg=opentelemetry-proto&subdirectory=opentelemetry-proto + opentelemetry-exporter-otlp-proto-common@{env:CORE_REPO}\#egg=opentelemetry-exporter-otlp-proto-common&subdirectory=exporter/opentelemetry-exporter-otlp-proto-common + opentelemetry-exporter-otlp-proto-grpc@{env:CORE_REPO}\#egg=opentelemetry-exporter-otlp-proto-grpc&subdirectory=exporter/opentelemetry-exporter-otlp-proto-grpc + opentelemetry-exporter-otlp-proto-http@{env:CORE_REPO}\#egg=opentelemetry-exporter-otlp-proto-http&subdirectory=exporter/opentelemetry-exporter-otlp-proto-http deps = lint: -r dev-requirements.txt coverage: pytest @@ -261,6 +273,11 @@ deps = a2a: {[testenv]test_deps} a2a: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-a2a/test-requirements.txt + agentuniverse: {[testenv]test_deps} + ; Framework exporters must match the core development SDK used by this lane. + agentuniverse: {[testenv]agentuniverse_exporter_deps} + agentuniverse: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/test-requirements.txt + llama-index: {[testenv]test_deps} llama-index: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-llama-index/test-requirements.txt @@ -382,6 +399,9 @@ commands = test-loongsuite-instrumentation-a2a: pytest {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-a2a/tests {posargs} lint-loongsuite-instrumentation-a2a: python -m ruff check {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-a2a + test-loongsuite-instrumentation-agentuniverse: pytest {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse/tests {posargs} + lint-loongsuite-instrumentation-agentuniverse: python -m ruff check {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-agentuniverse + test-loongsuite-instrumentation-llama-index: pytest {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-llama-index/tests {posargs} lint-loongsuite-instrumentation-llama-index: python -m ruff check {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-llama-index @@ -486,3 +506,15 @@ commands = pyright ; FIXME(cirilla-zmh): support package as loongsuite instrumentation module + +[testenv:lint-loongsuite-instrumentation-agentuniverse] +; The env name contains the factor 'agentuniverse', so the factor-conditional +; deps in [testenv] would also pull the package's test-requirements.txt into +; this lint env. That file requires agentUniverse>=0.0.19 (which needs +; sphinx-rtd-theme>=2.0.0,<3.0.0) while the lint deps pin +; sphinx-rtd-theme==2.0.0rc4, and the package's requires-python is <3.13 while +; this env runs on 3.13, so uv cannot resolve the environment at all. Linting +; only needs ruff, so keep this env on the plain lint deps. +deps = + {[testenv]test_deps} + -r {toxinidir}/dev-requirements.txt