From a764a85b8a3bc0882a1527695b3209a39a5a620a Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 11:10:10 +0200 Subject: [PATCH 01/25] Split observability and Phoenix extras Co-Authored-By: Claude Code --- pyproject.toml | 5 +++-- uv.lock | 10 +++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 87e34649..e720bc5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,10 +37,11 @@ dependencies = [ ] [project.optional-dependencies] -observability = [ +observability = ["opentelemetry-api>=1.30.0,<2.0.0"] +phoenix = [ + "opentelemetry-api>=1.30.0,<2.0.0", "openinference-instrumentation>=0.1.38,<0.2.0", "openinference-semantic-conventions>=0.1.20,<0.2.0", - "opentelemetry-api>=1.30.0,<2.0.0", ] [project.urls] diff --git a/uv.lock b/uv.lock index aa3aaa5a..40e2514b 100644 --- a/uv.lock +++ b/uv.lock @@ -284,6 +284,9 @@ dependencies = [ [package.optional-dependencies] observability = [ + { name = "opentelemetry-api" }, +] +phoenix = [ { name = "openinference-instrumentation" }, { name = "openinference-semantic-conventions" }, { name = "opentelemetry-api" }, @@ -320,9 +323,10 @@ requires-dist = [ { name = "instructor", specifier = ">=1.17.0,<2" }, { name = "litellm", specifier = ">=1.50.0,<2.0.0" }, { name = "mcp", extras = ["cli"], specifier = ">=1.8.0,<2" }, - { name = "openinference-instrumentation", marker = "extra == 'observability'", specifier = ">=0.1.38,<0.2.0" }, - { name = "openinference-semantic-conventions", marker = "extra == 'observability'", specifier = ">=0.1.20,<0.2.0" }, + { name = "openinference-instrumentation", marker = "extra == 'phoenix'", specifier = ">=0.1.38,<0.2.0" }, + { name = "openinference-semantic-conventions", marker = "extra == 'phoenix'", specifier = ">=0.1.20,<0.2.0" }, { name = "opentelemetry-api", marker = "extra == 'observability'", specifier = ">=1.30.0,<2.0.0" }, + { name = "opentelemetry-api", marker = "extra == 'phoenix'", specifier = ">=1.30.0,<2.0.0" }, { name = "pydantic", specifier = ">=2.11.4,<3.0.0" }, { name = "pyfiglet", specifier = ">=1.0.2,<2.0.0" }, { name = "pyyaml", specifier = ">=6.0.2,<7.0.0" }, @@ -330,7 +334,7 @@ requires-dist = [ { name = "rich", specifier = ">=13.7.1,<14.0.0" }, { name = "textual", specifier = ">=5.3.0,<6.0.0" }, ] -provides-extras = ["observability"] +provides-extras = ["observability", "phoenix"] [package.metadata.requires-dev] dev = [ From 878e072d417abbe88c9f45fac8d465e80166dc62 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 11:10:44 +0200 Subject: [PATCH 02/25] Rename observability tracing modules Observability suite: 57 passed after updating imports. Co-Authored-By: Claude Code --- atomic-agents/atomic_agents/observability/__init__.py | 4 ++-- .../observability/{openinference_tracing.py => tracing.py} | 0 .../tests/observability/test_opentelemetry_discovery.py | 4 ++-- .../{test_openinference_tracing.py => test_tracing.py} | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename atomic-agents/atomic_agents/observability/{openinference_tracing.py => tracing.py} (100%) rename atomic-agents/tests/observability/{test_openinference_tracing.py => test_tracing.py} (99%) diff --git a/atomic-agents/atomic_agents/observability/__init__.py b/atomic-agents/atomic_agents/observability/__init__.py index 19bb0e22..94f59452 100644 --- a/atomic-agents/atomic_agents/observability/__init__.py +++ b/atomic-agents/atomic_agents/observability/__init__.py @@ -48,9 +48,9 @@ def _load_tracing_module(): ".".join(str(part) for part in _MINIMUM_OPENINFERENCE_VERSION), ) return None - from atomic_agents.observability import openinference_tracing + from atomic_agents.observability import tracing - return openinference_tracing + return tracing class AtomicAgentsInstrumentor: diff --git a/atomic-agents/atomic_agents/observability/openinference_tracing.py b/atomic-agents/atomic_agents/observability/tracing.py similarity index 100% rename from atomic-agents/atomic_agents/observability/openinference_tracing.py rename to atomic-agents/atomic_agents/observability/tracing.py diff --git a/atomic-agents/tests/observability/test_opentelemetry_discovery.py b/atomic-agents/tests/observability/test_opentelemetry_discovery.py index 6c40f555..5be31be6 100644 --- a/atomic-agents/tests/observability/test_opentelemetry_discovery.py +++ b/atomic-agents/tests/observability/test_opentelemetry_discovery.py @@ -20,7 +20,7 @@ import atomic_agents.observability as observability from atomic_agents.observability import AtomicAgentsInstrumentor -from test_openinference_tracing import CalculationInputSchema, CalculatorTool, QuestionSchema, make_agent, sync_client +from test_tracing import CalculationInputSchema, CalculatorTool, QuestionSchema, make_agent, sync_client @pytest.fixture @@ -223,7 +223,7 @@ def test_normal_import_does_not_require_optional_observability_packages() -> Non sys.modules['opentelemetry'] = None import atomic_agents import atomic_agents.observability -assert 'atomic_agents.observability.openinference_tracing' not in sys.modules +assert 'atomic_agents.observability.tracing' not in sys.modules print('optional-free import passed') """ result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=45) diff --git a/atomic-agents/tests/observability/test_openinference_tracing.py b/atomic-agents/tests/observability/test_tracing.py similarity index 99% rename from atomic-agents/tests/observability/test_openinference_tracing.py rename to atomic-agents/tests/observability/test_tracing.py index 6fb3bc88..936492da 100644 --- a/atomic-agents/tests/observability/test_openinference_tracing.py +++ b/atomic-agents/tests/observability/test_tracing.py @@ -25,7 +25,7 @@ from atomic_agents.context import BaseDynamicContextProvider, SystemPromptGenerator from atomic_agents.observability import AtomicAgentsInstrumentor import atomic_agents.observability as observability -from atomic_agents.observability import openinference_tracing as tracing +from atomic_agents.observability import tracing as tracing class QuestionSchema(BaseIOSchema): From adefd60034f42e4f82c2aaae8701a209a6b8f4e0 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 11:19:34 +0200 Subject: [PATCH 03/25] Emit GenAI tracing attributes with plain OpenTelemetry Add typed tracing configuration, optional OpenInference attributes, privacy controls, payload truncation, suppression and error types. Focused observability regressions pass, including 12 assertion-failing negative controls. Changed-body complexity stays below 6; incomplete LCOV inventory measurements are tracked in SQ-145. Co-Authored-By: Claude Code --- .../atomic_agents/observability/__init__.py | 44 +-- .../atomic_agents/observability/tracing.py | 324 ++++++++++++++---- .../test_opentelemetry_discovery.py | 4 +- .../tests/observability/test_tracing.py | 193 +++++++++-- 4 files changed, 446 insertions(+), 119 deletions(-) diff --git a/atomic-agents/atomic_agents/observability/__init__.py b/atomic-agents/atomic_agents/observability/__init__.py index 94f59452..41d91622 100644 --- a/atomic-agents/atomic_agents/observability/__init__.py +++ b/atomic-agents/atomic_agents/observability/__init__.py @@ -1,5 +1,5 @@ """ -Opt-in OpenInference tracing for Atomic Agents. +Opt-in OpenTelemetry tracing for Atomic Agents. Emits an AGENT span per ``AtomicAgent`` run and a TOOL span per ``BaseTool`` run, rendered by Arize Phoenix and any other OpenTelemetry backend. Needs the ``observability`` extra: @@ -11,10 +11,10 @@ import importlib.util import logging from importlib.metadata import PackageNotFoundError, version -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING if TYPE_CHECKING: - from openinference.instrumentation import TraceConfig + from atomic_agents.observability.tracing import PayloadVisibility, TracingConfig from opentelemetry.trace import TracerProvider logger = logging.getLogger(__name__) @@ -27,8 +27,11 @@ _MINIMUM_OPENINFERENCE_VERSION = (0, 1, 38) -def _openinference_is_installed() -> bool: - return importlib.util.find_spec("openinference") is not None +def _opentelemetry_is_installed() -> bool: + try: + return importlib.util.find_spec("opentelemetry.trace") is not None + except ModuleNotFoundError: + return False def _openinference_is_too_old() -> bool: @@ -41,21 +44,20 @@ def _openinference_is_too_old() -> bool: def _load_tracing_module(): - if _openinference_is_too_old(): - logger.warning( - "Atomic Agents tracing is disabled: %s %s or newer is required.", - _OPENINFERENCE_DISTRIBUTION, - ".".join(str(part) for part in _MINIMUM_OPENINFERENCE_VERSION), - ) - return None from atomic_agents.observability import tracing return tracing +def __getattr__(name: str) -> object: + if name == "TracingConfig": + return _load_tracing_module().TracingConfig + raise AttributeError(name) + + class AtomicAgentsInstrumentor: """ - Installs OpenInference AGENT and TOOL spans around ``AtomicAgent`` and ``BaseTool`` calls. + Installs AGENT and TOOL spans around ``AtomicAgent`` and ``BaseTool`` calls. The public surface mirrors OpenTelemetry's ``BaseInstrumentor`` (``instrument`` / ``uninstrument``) without depending on it, because Arize Phoenix only guarantees ``opentelemetry-api`` and @@ -65,8 +67,8 @@ class AtomicAgentsInstrumentor: def instrument( self, *, - tracer_provider: Optional["TracerProvider"] = None, - config: Optional["TraceConfig"] = None, + tracer_provider: "TracerProvider | None" = None, + config: "TracingConfig | PayloadVisibility | None" = None, **options: object, ) -> None: """ @@ -75,21 +77,19 @@ def instrument( Args: tracer_provider: An OpenTelemetry ``TracerProvider``. Defaults to the global provider. - config: An ``openinference.instrumentation.TraceConfig`` (for example ``hide_inputs=True``). + config: Payload visibility and truncation settings. OpenInference TraceConfig is also accepted. **options: Launcher options accepted for OpenTelemetry auto-instrumentation compatibility. Raises: ModuleNotFoundError: When the ``observability`` extra is not installed, except during OpenTelemetry auto-discovery, which skips unavailable optional instrumentation. """ - if not _openinference_is_installed(): + if not _opentelemetry_is_installed(): if options.get("skip_dep_check") is True: logger.debug(_INSTALL_HINT) return raise ModuleNotFoundError(_INSTALL_HINT) - tracing = _load_tracing_module() - if tracing is not None: - tracing.install(tracer_provider=tracer_provider, config=config) + _load_tracing_module().install(tracer_provider=tracer_provider, config=config) def uninstrument(self) -> None: """Restore the original framework methods. No-op when nothing is installed.""" @@ -99,9 +99,9 @@ def uninstrument(self) -> None: @property def is_instrumented_by_opentelemetry(self) -> bool: - if not _openinference_is_installed() or _openinference_is_too_old(): + if not _opentelemetry_is_installed(): return False return _load_tracing_module().is_installed() -__all__ = ["AtomicAgentsInstrumentor"] +__all__ = ["AtomicAgentsInstrumentor", "TracingConfig"] diff --git a/atomic-agents/atomic_agents/observability/tracing.py b/atomic-agents/atomic_agents/observability/tracing.py index 7959be40..85be9dea 100644 --- a/atomic-agents/atomic_agents/observability/tracing.py +++ b/atomic-agents/atomic_agents/observability/tracing.py @@ -1,5 +1,5 @@ """ -OpenInference span emission for ``AtomicAgent`` runs and ``BaseTool`` runs. +OpenTelemetry span emission for ``AtomicAgent`` runs and ``BaseTool`` runs. Loaded lazily by :class:`atomic_agents.observability.AtomicAgentsInstrumentor`; importing this module requires the ``observability`` extra. @@ -12,14 +12,17 @@ import inspect import json import logging +from contextlib import contextmanager +from dataclasses import dataclass, replace +import os from types import FunctionType -from typing import TYPE_CHECKING, Any, AsyncGenerator, Callable, Dict, Generator, List, Optional, Type +from typing import TYPE_CHECKING, Any, AsyncGenerator, Callable, Dict, Generator, List, Optional, Type, Protocol, Iterator -from openinference.instrumentation import OITracer, TraceConfig -from openinference.semconv.trace import SpanAttributes from opentelemetry import context as context_api from opentelemetry import trace as trace_api -from opentelemetry.trace import Status, StatusCode +from opentelemetry.trace import Span, Status, StatusCode, Tracer +from opentelemetry.util.types import AttributeValue +from pydantic import BaseModel from atomic_agents import __version__ from atomic_agents.agents.atomic_agent import AtomicAgent @@ -49,17 +52,102 @@ ATTR_SYSTEM_PROMPT_SHA256 = "atomic_agents.system_prompt.sha256" ATTR_STREAM_CLOSED_EARLY = "atomic_agents.stream.closed_early" +ATTR_OPERATION = "gen_ai.operation.name" +ATTR_AGENT_NAME = "gen_ai.agent.name" +ATTR_REQUEST_MODEL = "gen_ai.request.model" +ATTR_PROVIDER_NAME = "gen_ai.provider.name" +ATTR_CONVERSATION_ID = "gen_ai.conversation.id" +ATTR_TOOL_NAME = "gen_ai.tool.name" +ATTR_TOOL_DESCRIPTION = "gen_ai.tool.description" +ATTR_TOOL_TYPE = "gen_ai.tool.type" +ATTR_TOOL_ARGUMENTS = "gen_ai.tool.call.arguments" +ATTR_TOOL_RESULT = "gen_ai.tool.call.result" +ATTR_SPAN_KIND = "openinference.span.kind" +ATTR_INPUT_VALUE = "input.value" +ATTR_INPUT_MIME_TYPE = "input.mime_type" +ATTR_OUTPUT_VALUE = "output.value" +ATTR_OUTPUT_MIME_TYPE = "output.mime_type" +ATTR_OI_AGENT_NAME = "agent.name" +ATTR_OI_TOOL_NAME = "tool.name" +ATTR_OI_TOOL_DESCRIPTION = "tool.description" +ATTR_TOOL_PARAMETERS = "tool.parameters" +ATTR_SESSION_ID = "session.id" +ATTR_INPUT = "atomic_agents.input" +ATTR_OUTPUT = "atomic_agents.output" +ATTR_AGENT_CLASS = "atomic_agents.agent.class" +ATTR_ERROR_TYPE = "error.type" + + +class PayloadVisibility(Protocol): + hide_inputs: bool + hide_outputs: bool + + +@dataclass(frozen=True) +class TracingConfig: + hide_inputs: bool = False + hide_outputs: bool = False + max_attribute_length: int = 4096 + openinference_attributes: bool | None = None + llm_spans: bool = True + + @classmethod + def from_environment(cls) -> "TracingConfig": + return cls( + hide_inputs=os.environ.get("OPENINFERENCE_HIDE_INPUTS", "").lower() in ("true", "1"), + hide_outputs=os.environ.get("OPENINFERENCE_HIDE_OUTPUTS", "").lower() in ("true", "1"), + max_attribute_length=int(os.environ.get("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", "4096")), + ) + + @classmethod + def coerce(cls, config: "TracingConfig | PayloadVisibility | None") -> "TracingConfig": + if config is None: + return cls.from_environment() + if isinstance(config, cls): + return config + return cls(hide_inputs=config.hide_inputs, hide_outputs=config.hide_outputs) + + +def _import_context_attributes() -> Callable[[], dict[str, AttributeValue]] | None: + try: + from openinference.instrumentation import get_attributes_from_context + except ImportError: + return None + from atomic_agents.observability import _openinference_is_too_old + + if _openinference_is_too_old(): + logger.warning("OpenInference context attributes disabled: 0.1.38 or newer is required.") + return None + return lambda: dict(get_attributes_from_context()) + + +def _context_attributes() -> dict[str, AttributeValue]: + reader = _current_installation().context_attributes + return reader() if reader is not None else {} + + +def _current_installation() -> "_Installation": + assert _installation is not None + return _installation + + +def _suppressed() -> bool: + return bool(context_api.get_value("suppress_instrumentation")) + + _active_tool: contextvars.ContextVar[Optional[BaseTool]] = contextvars.ContextVar("atomic_agents_active_tool", default=None) class _Installation: def __init__( self, - tracer: OITracer, + tracer: Tracer, tracer_provider: Optional["TracerProvider"], - config: TraceConfig, + config: TracingConfig, original_init_subclass: classmethod, + context_attributes: Callable[[], dict[str, AttributeValue]] | None, ): + self.context_attributes = context_attributes self.tracer = tracer self.tracer_provider = tracer_provider self.config = config @@ -73,18 +161,25 @@ def is_installed() -> bool: return _installation is not None -def install(*, tracer_provider: Optional["TracerProvider"] = None, config: Optional[TraceConfig] = None) -> None: +def install( + *, tracer_provider: Optional["TracerProvider"] = None, config: TracingConfig | PayloadVisibility | None = None +) -> None: global _installation - config = config or TraceConfig() + config = TracingConfig.coerce(config) if _installation is not None: + if config.openinference_attributes is None: + config = replace(config, openinference_attributes=_installation.config.openinference_attributes) _warn_if_reinstalled_differently(_installation, tracer_provider, config) return - tracer = OITracer(trace_api.get_tracer(TRACER_NAME, __version__, tracer_provider), config=config) + context_attributes = _import_context_attributes() + if config.openinference_attributes is None: + config = replace(config, openinference_attributes=context_attributes is not None) + tracer = trace_api.get_tracer(TRACER_NAME, __version__, tracer_provider) original_init_subclass = BaseTool.__dict__["__init_subclass__"] - _installation = _Installation(tracer, tracer_provider, config, original_init_subclass) - _instrument_agent_class(tracer) - _instrument_existing_tool_classes(tracer) - _hook_future_tool_classes(tracer, original_init_subclass) + _installation = _Installation(tracer, tracer_provider, config, original_init_subclass, context_attributes) + _instrument_agent_class() + _instrument_existing_tool_classes() + _hook_future_tool_classes(original_init_subclass) def uninstall() -> None: @@ -100,7 +195,7 @@ def uninstall() -> None: def _warn_if_reinstalled_differently( - installed: _Installation, tracer_provider: Optional["TracerProvider"], config: TraceConfig + installed: _Installation, tracer_provider: Optional["TracerProvider"], config: TracingConfig ) -> None: same_provider = tracer_provider is None or tracer_provider is installed.tracer_provider same_config = config == installed.config @@ -111,11 +206,11 @@ def _warn_if_reinstalled_differently( ) -def _instrument_agent_class(tracer: OITracer) -> None: +def _instrument_agent_class() -> None: for method_name, wrap in _AGENT_WRAPPERS.items(): original = AtomicAgent.__dict__[method_name] if not getattr(original, TRACED_MARKER, False): - setattr(AtomicAgent, method_name, _mark_traced(wrap(tracer, method_name, original), original, False)) + setattr(AtomicAgent, method_name, _mark_traced(wrap(method_name, original), original, False)) def _mark_traced(wrapper: Callable, original: Callable, inherited: bool) -> Callable: @@ -146,25 +241,25 @@ def _all_subclasses(base: Type) -> List[Type]: return found -def _instrument_existing_tool_classes(tracer: OITracer) -> None: +def _instrument_existing_tool_classes() -> None: for tool_class in _all_subclasses(BaseTool): - _instrument_tool_class(tool_class, tracer) + _instrument_tool_class(tool_class) -def _hook_future_tool_classes(tracer: OITracer, original_init_subclass: classmethod) -> None: +def _hook_future_tool_classes(original_init_subclass: classmethod) -> None: def traced_init_subclass(cls, **kwargs): original_init_subclass.__func__(cls, **kwargs) - _instrument_tool_class(cls, tracer) + _instrument_tool_class(cls) setattr(BaseTool, "__init_subclass__", classmethod(traced_init_subclass)) -def _instrument_tool_class(tool_class: Type, tracer: OITracer) -> None: +def _instrument_tool_class(tool_class: Type) -> None: for method_name in TOOL_METHOD_NAMES: method = inspect.getattr_static(tool_class, method_name, None) if _is_traceable_function(method): inherited = method_name not in tool_class.__dict__ - setattr(tool_class, method_name, _mark_traced(_trace_tool_method(tracer, method), method, inherited)) + setattr(tool_class, method_name, _mark_traced(_trace_tool_method(method), method, inherited)) def _is_traceable_function(method: Any) -> bool: @@ -187,7 +282,12 @@ def _agent_display_name(agent: AtomicAgent) -> str: def _agent_start_attributes(agent: AtomicAgent) -> Dict[str, Any]: try: return { - SpanAttributes.AGENT_NAME: _agent_display_name(agent), + ATTR_OPERATION: "invoke_agent", + ATTR_AGENT_NAME: type(agent).__name__, + ATTR_AGENT_CLASS: _agent_display_name(agent), + ATTR_REQUEST_MODEL: agent.model, + **_provider_attributes(agent), + **_optional_attributes("AGENT", {ATTR_OI_AGENT_NAME: _agent_display_name(agent)}), ATTR_MODEL: agent.model, ATTR_MODE: getattr(agent.mode, "value", str(agent.mode)), ATTR_INPUT_SCHEMA: agent.input_schema.__name__, @@ -221,22 +321,62 @@ def _user_input_from_call(args: tuple, kwargs: dict) -> Any: return args[0] if args else kwargs.get("user_input") -def _set_input_safely(span: Any, value: Any) -> None: +def _provider_attributes(agent: AtomicAgent) -> dict[str, str]: + provider = getattr(getattr(agent.client, "provider", None), "value", None) + return {ATTR_PROVIDER_NAME: provider} if isinstance(provider, str) else {} + + +def _optional_attributes(kind: str, attributes: dict[str, AttributeValue]) -> dict[str, AttributeValue]: + if not _current_installation().config.openinference_attributes: + return {} + context = _context_attributes() + session = context.get(ATTR_SESSION_ID) + if session is not None: + context[ATTR_CONVERSATION_ID] = session + return {ATTR_SPAN_KIND: kind, **attributes, **context} + + +def _record_payload( + span: Span, key_pairs: tuple[tuple[str, str | None], ...], value: object, hide: bool, config: TracingConfig +) -> None: if value is None: return try: - span.set_input(value) + serialized = value.model_dump_json() if isinstance(value, BaseModel) else json.dumps(value, default=str) except Exception as error: - logger.warning("Could not record span input: %s", error) + logger.warning("Could not serialize tracing payload: %s", error) + return + for key, mime_key in key_pairs: + _write_payload(span, key, mime_key, serialized, hide, config) -def _set_output_safely(span: Any, value: Any) -> None: - if value is None: +def _write_payload(span: Span, key: str, mime_key: str | None, serialized: str, hide: bool, config: TracingConfig) -> None: + if hide: + span.set_attribute(key, "__REDACTED__") return - try: - span.set_output(value) - except Exception as error: - logger.warning("Could not record span output: %s", error) + if len(serialized) > config.max_attribute_length: + span.set_attribute(key + ".original_length", len(serialized)) + serialized = serialized[: config.max_attribute_length] + "...[truncated]" + span.set_attribute(key, serialized) + if mime_key is not None: + span.set_attribute(mime_key, "application/json") + + +def _payload_keys(primary: str, output: bool) -> tuple[tuple[str, str | None], ...]: + keys: tuple[tuple[str, str | None], ...] = ((primary, None),) + if _current_installation().config.openinference_attributes: + keys += ((ATTR_OUTPUT_VALUE, ATTR_OUTPUT_MIME_TYPE),) if output else ((ATTR_INPUT_VALUE, ATTR_INPUT_MIME_TYPE),) + return keys + + +def _set_input_safely(span: Span, value: object, primary: str = ATTR_INPUT) -> None: + config = _current_installation().config + _record_payload(span, _payload_keys(primary, False), value, config.hide_inputs, config) + + +def _set_output_safely(span: Span, value: object, primary: str = ATTR_OUTPUT) -> None: + config = _current_installation().config + _record_payload(span, _payload_keys(primary, True), value, config.hide_outputs, config) def _finish_agent_span(span: Any, output: Any) -> None: @@ -250,18 +390,34 @@ def _mark_closed_early(span: Any, output: Any) -> None: def _record_failure(span: Any, error: BaseException) -> None: + span.set_attribute(ATTR_ERROR_TYPE, type(error).__qualname__) span.record_exception(error) span.set_status(Status(StatusCode.ERROR, str(error))) -def _trace_agent_call(tracer: OITracer, method_name: str, wrapped: Callable) -> Callable: +@contextmanager +def _agent_span(agent: AtomicAgent, args: tuple, kwargs: dict) -> Iterator[Span]: + installed = _current_installation() + with installed.tracer.start_as_current_span( + f"invoke_agent {type(agent).__name__}", + attributes=_agent_start_attributes(agent), + record_exception=False, + set_status_on_exception=False, + ) as span: + _set_input_safely(span, _user_input_from_call(args, kwargs)) + try: + yield span + except BaseException as error: + _record_failure(span, error) + raise + + +def _trace_agent_call(method_name: str, wrapped: Callable) -> Callable: @functools.wraps(wrapped) def traced(agent, *args, **kwargs): - span_name = f"{_agent_display_name(agent)}.{method_name}" - with tracer.start_as_current_span( - span_name, openinference_span_kind="agent", attributes=_agent_start_attributes(agent) - ) as span: - _set_input_safely(span, _user_input_from_call(args, kwargs)) + if _suppressed(): + return wrapped(agent, *args, **kwargs) + with _agent_span(agent, args, kwargs) as span: response = wrapped(agent, *args, **kwargs) _finish_agent_span(span, response) return response @@ -269,14 +425,12 @@ def traced(agent, *args, **kwargs): return traced -def _trace_agent_async_call(tracer: OITracer, method_name: str, wrapped: Callable) -> Callable: +def _trace_agent_async_call(method_name: str, wrapped: Callable) -> Callable: @functools.wraps(wrapped) async def traced(agent, *args, **kwargs): - span_name = f"{_agent_display_name(agent)}.{method_name}" - with tracer.start_as_current_span( - span_name, openinference_span_kind="agent", attributes=_agent_start_attributes(agent) - ) as span: - _set_input_safely(span, _user_input_from_call(args, kwargs)) + if _suppressed(): + return await wrapped(agent, *args, **kwargs) + with _agent_span(agent, args, kwargs) as span: response = await wrapped(agent, *args, **kwargs) _finish_agent_span(span, response) return response @@ -284,10 +438,11 @@ async def traced(agent, *args, **kwargs): return traced -def _start_agent_span(tracer: OITracer, agent: AtomicAgent, method_name: str, args: tuple, kwargs: dict) -> Any: - span = tracer.start_span( - f"{_agent_display_name(agent)}.{method_name}", - openinference_span_kind="agent", +def _start_agent_span(agent: AtomicAgent, method_name: str, args: tuple, kwargs: dict) -> Span: + if _suppressed(): + return trace_api.INVALID_SPAN + span = _current_installation().tracer.start_span( + f"invoke_agent {type(agent).__name__}", attributes=_agent_start_attributes(agent), ) _set_input_safely(span, _user_input_from_call(args, kwargs)) @@ -355,10 +510,10 @@ def mark_cancelled(self) -> None: _mark_closed_early(self.span, self.last) -def _trace_agent_stream(tracer: OITracer, method_name: str, wrapped: Callable) -> Callable: +def _trace_agent_stream(method_name: str, wrapped: Callable) -> Callable: @functools.wraps(wrapped) def traced(agent, *args, **kwargs): - span = _start_agent_span(tracer, agent, method_name, args, kwargs) + span = _start_agent_span(agent, method_name, args, kwargs) cursor = _SyncStreamCursor(span, wrapped(agent, *args, **kwargs)) try: while cursor.advance(): @@ -377,10 +532,10 @@ def traced(agent, *args, **kwargs): return traced -def _trace_agent_async_stream(tracer: OITracer, method_name: str, wrapped: Callable) -> Callable: +def _trace_agent_async_stream(method_name: str, wrapped: Callable) -> Callable: @functools.wraps(wrapped) async def traced(agent, *args, **kwargs): - span = _start_agent_span(tracer, agent, method_name, args, kwargs) + span = _start_agent_span(agent, method_name, args, kwargs) cursor = _AsyncStreamCursor(span, wrapped(agent, *args, **kwargs)) try: while await cursor.advance(): @@ -401,50 +556,73 @@ async def traced(agent, *args, **kwargs): return traced -def _describe_tool(span: Any, tool: BaseTool) -> None: +def _tool_start_attributes(tool: BaseTool) -> dict[str, AttributeValue]: + attributes: dict[str, AttributeValue] = { + ATTR_OPERATION: "execute_tool", + ATTR_TOOL_NAME: type(tool).__name__, + ATTR_TOOL_TYPE: "function", + **_optional_attributes("TOOL", {}), + } try: - span.set_tool( - name=tool.tool_name, - description=tool.tool_description, - parameters=tool.input_schema.model_json_schema(), - ) + attributes[ATTR_TOOL_DESCRIPTION] = tool.tool_description + if _current_installation().config.openinference_attributes: + attributes.update( + { + ATTR_OI_TOOL_NAME: tool.tool_name, + ATTR_OI_TOOL_DESCRIPTION: tool.tool_description, + ATTR_TOOL_PARAMETERS: json.dumps(tool.input_schema.model_json_schema()), + } + ) except Exception as error: logger.warning("Could not describe tool %s for tracing: %s", type(tool).__name__, error) + return attributes + + +@contextmanager +def _tool_span(tool: BaseTool, args: tuple, kwargs: dict) -> Iterator[Span]: + with _current_installation().tracer.start_as_current_span( + f"execute_tool {type(tool).__name__}", + attributes=_tool_start_attributes(tool), + record_exception=False, + set_status_on_exception=False, + ) as span: + _set_input_safely(span, _tool_input_from_call(args, kwargs), ATTR_TOOL_ARGUMENTS) + try: + yield span + except BaseException as error: + _record_failure(span, error) + raise def _tool_input_from_call(args: tuple, kwargs: dict) -> Any: return args[0] if args else kwargs.get("params") -def _trace_tool_method(tracer: OITracer, wrapped: Callable) -> Callable: +def _trace_tool_method(wrapped: Callable) -> Callable: if inspect.iscoroutinefunction(wrapped): - return _trace_tool_async_method(tracer, wrapped) + return _trace_tool_async_method(wrapped) @functools.wraps(wrapped) def traced(tool, *args, **kwargs): - if _active_tool.get() is tool: + if _active_tool.get() is tool or _suppressed(): return wrapped(tool, *args, **kwargs) - with tracer.start_as_current_span(type(tool).__name__, openinference_span_kind="tool") as span: - _describe_tool(span, tool) - _set_input_safely(span, _tool_input_from_call(args, kwargs)) + with _tool_span(tool, args, kwargs) as span: result = _run_as_active_tool(tool, wrapped, args, kwargs) - _set_output_safely(span, result) + _set_output_safely(span, result, ATTR_TOOL_RESULT) span.set_status(Status(StatusCode.OK)) return result return traced -def _trace_tool_async_method(tracer: OITracer, wrapped: Callable) -> Callable: +def _trace_tool_async_method(wrapped: Callable) -> Callable: @functools.wraps(wrapped) async def traced(tool, *args, **kwargs): - if _active_tool.get() is tool: + if _active_tool.get() is tool or _suppressed(): return await wrapped(tool, *args, **kwargs) - with tracer.start_as_current_span(type(tool).__name__, openinference_span_kind="tool") as span: - _describe_tool(span, tool) - _set_input_safely(span, _tool_input_from_call(args, kwargs)) + with _tool_span(tool, args, kwargs) as span: result = await _await_as_active_tool(tool, wrapped, args, kwargs) - _set_output_safely(span, result) + _set_output_safely(span, result, ATTR_TOOL_RESULT) span.set_status(Status(StatusCode.OK)) return result @@ -467,7 +645,7 @@ async def _await_as_active_tool(tool: BaseTool, wrapped: Callable, args: tuple, _active_tool.reset(token) -_AGENT_WRAPPERS: Dict[str, Callable[[OITracer, str, Callable], Callable]] = { +_AGENT_WRAPPERS: Dict[str, Callable[[str, Callable], Callable]] = { AGENT_SYNC_METHOD: _trace_agent_call, AGENT_ASYNC_METHOD: _trace_agent_async_call, AGENT_SYNC_STREAM_METHOD: _trace_agent_stream, diff --git a/atomic-agents/tests/observability/test_opentelemetry_discovery.py b/atomic-agents/tests/observability/test_opentelemetry_discovery.py index 5be31be6..0828d70d 100644 --- a/atomic-agents/tests/observability/test_opentelemetry_discovery.py +++ b/atomic-agents/tests/observability/test_opentelemetry_discovery.py @@ -88,7 +88,7 @@ def test_unknown_launcher_options_preserve_provider_and_privacy( assert [exported_attributes(span)[SpanAttributes.OUTPUT_VALUE] for span in spans] == ["__REDACTED__", "__REDACTED__"] -def test_loader_skips_missing_openinference_and_continues_without_error( +def test_loader_skips_missing_opentelemetry_and_continues_without_error( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: instrumentor = installed_entry_point("opentelemetry_instrumentor") @@ -98,7 +98,7 @@ def test_loader_skips_missing_openinference_and_continues_without_error( sentinel.name = "sentinel" points = (instrumentor, sentinel) monkeypatch.setattr(_load, "entry_points", lambda group: points if group == "opentelemetry_instrumentor" else ()) - monkeypatch.setattr(observability, "_openinference_is_installed", lambda: False) + monkeypatch.setattr(observability, "_opentelemetry_is_installed", lambda: False) monkeypatch.delenv("OTEL_PYTHON_DISABLED_INSTRUMENTATIONS", raising=False) with caplog.at_level(logging.DEBUG): diff --git a/atomic-agents/tests/observability/test_tracing.py b/atomic-agents/tests/observability/test_tracing.py index 936492da..6e6ef8ac 100644 --- a/atomic-agents/tests/observability/test_tracing.py +++ b/atomic-agents/tests/observability/test_tracing.py @@ -187,7 +187,7 @@ def test_run_emits_agent_span_with_io_and_framework_attributes(exporter): assert response.answer == "42" (span,) = spans_of(exporter) - assert span.name == "AtomicAgent[QuestionSchema, AnswerSchema].run" + assert span.name == "invoke_agent AtomicAgent" assert kind_of(span) == OpenInferenceSpanKindValues.AGENT.value assert span.attributes[SpanAttributes.AGENT_NAME] == "AtomicAgent[QuestionSchema, AnswerSchema]" assert input_json(span) == {"question": "6*7?"} @@ -266,7 +266,7 @@ def run(self, user_input=None): agent.run(QuestionSchema(question="?")) (span,) = spans_of(exporter) - assert span.name == "ResearchAgent[QuestionSchema, AnswerSchema].run" + assert span.name == "invoke_agent ResearchAgent" def test_agent_span_nests_under_a_user_chain_span(exporter, provider_tracer): @@ -345,7 +345,7 @@ def create_partial(**kwargs): assert partials == ["4", "42"] assert all(span is trace_api.INVALID_SPAN for span in current_between_yields) chunk_span, agent_span = spans_of(exporter) - assert agent_span.name == "AtomicAgent[QuestionSchema, AnswerSchema].run_stream" + assert agent_span.name == "invoke_agent AtomicAgent" assert chunk_span.parent.span_id == agent_span.context.span_id assert output_json(agent_span) == {"answer": "42"} assert agent_span.status.status_code == StatusCode.OK @@ -426,7 +426,7 @@ async def test_run_async_emits_agent_span(exporter): assert response.answer == "42" (span,) = spans_of(exporter) - assert span.name == "AtomicAgent[QuestionSchema, AnswerSchema].run_async" + assert span.name == "invoke_agent AtomicAgent" assert output_json(span) == {"answer": "42"} assert span.status.status_code == StatusCode.OK @@ -446,7 +446,7 @@ async def create_partial(**kwargs): assert partials == ["4", "42"] chunk_span, agent_span = spans_of(exporter) - assert agent_span.name == "AtomicAgent[QuestionSchema, AnswerSchema].run_async_stream" + assert agent_span.name == "invoke_agent AtomicAgent" assert chunk_span.parent.span_id == agent_span.context.span_id assert output_json(agent_span) == {"answer": "42"} @@ -484,7 +484,7 @@ def test_tool_defined_before_instrumentation_emits_tool_span(exporter): assert result.result == "3.0" (span,) = spans_of(exporter) - assert span.name == "CalculatorTool" + assert span.name == "execute_tool CalculatorTool" assert kind_of(span) == OpenInferenceSpanKindValues.TOOL.value assert span.attributes[SpanAttributes.TOOL_NAME] == "CalculationInputSchema" assert span.attributes[SpanAttributes.TOOL_DESCRIPTION] == "Expression to evaluate." @@ -510,7 +510,7 @@ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema: LateTool().run(CalculationInputSchema(expression="x")) (span,) = spans_of(exporter) - assert span.name == "LateTool" + assert span.name == "execute_tool LateTool" assert LateTool().input_schema is CalculationInputSchema @@ -522,14 +522,14 @@ def run(self, params: CalculationInputSchema, precision: int = 2) -> Calculation LoudCalculatorTool().run(CalculationInputSchema(expression="2*2")) (span,) = spans_of(exporter) - assert span.name == "LoudCalculatorTool" + assert span.name == "execute_tool LoudCalculatorTool" def test_sync_run_delegating_to_arun_emits_one_span(exporter): AsyncCalculatorTool().run(CalculationInputSchema(expression="2*2")) (span,) = spans_of(exporter) - assert span.name == "AsyncCalculatorTool" + assert span.name == "execute_tool AsyncCalculatorTool" assert output_json(span) == {"result": "3"} @@ -538,7 +538,7 @@ async def test_run_async_delegating_to_thread_emits_one_span(exporter): await ThreadedCalculatorTool().run_async(CalculationInputSchema(expression="3*3")) (span,) = spans_of(exporter) - assert span.name == "ThreadedCalculatorTool" + assert span.name == "execute_tool ThreadedCalculatorTool" assert output_json(span) == {"result": "3"} @@ -546,7 +546,7 @@ def test_mixin_defined_run_is_traced(exporter): MixinCalculatorTool().run(CalculationInputSchema(expression="x")) (span,) = spans_of(exporter) - assert span.name == "MixinCalculatorTool" + assert span.name == "execute_tool MixinCalculatorTool" def test_static_run_is_left_alone(exporter): @@ -674,31 +674,31 @@ def test_phoenix_entry_point_loads_and_instruments_with_only_a_tracer_provider() @pytest.mark.parametrize("options", [{}, {"skip_dep_check": False}]) -def test_instrument_without_openinference_raises_install_hint(monkeypatch: pytest.MonkeyPatch, options: dict[str, object]): - monkeypatch.setattr(observability, "_openinference_is_installed", lambda: False) +def test_instrument_without_opentelemetry_raises_install_hint(monkeypatch: pytest.MonkeyPatch, options: dict[str, object]): + monkeypatch.setattr(observability, "_opentelemetry_is_installed", lambda: False) with pytest.raises(ImportError, match="atomic-agents\\[observability\\]"): AtomicAgentsInstrumentor().instrument(**options) -def test_uninstrument_without_openinference_is_a_no_op(monkeypatch): - monkeypatch.setattr(observability, "_openinference_is_installed", lambda: False) +def test_uninstrument_without_opentelemetry_is_a_no_op(monkeypatch): + monkeypatch.setattr(observability, "_opentelemetry_is_installed", lambda: False) AtomicAgentsInstrumentor().uninstrument() assert not AtomicAgentsInstrumentor().is_instrumented_by_opentelemetry -def test_openinference_below_the_floor_disables_tracing_with_a_warning(monkeypatch, caplog): +def test_openinference_below_the_floor_only_disables_context_with_a_warning(monkeypatch, caplog): monkeypatch.setattr(observability, "version", lambda distribution: "0.1.20") with caplog.at_level(logging.WARNING): AtomicAgentsInstrumentor().instrument() assert "0.1.38 or newer is required" in caplog.text - assert not AtomicAgentsInstrumentor().is_instrumented_by_opentelemetry - assert AtomicAgent.__dict__["run"].__name__ == "run" - assert not hasattr(AtomicAgent.__dict__["run"], tracing.TRACED_MARKER) + assert AtomicAgentsInstrumentor().is_instrumented_by_opentelemetry + assert tracing._context_attributes() == {} + AtomicAgentsInstrumentor().uninstrument() class CountingProvider(BaseDynamicContextProvider): @@ -790,7 +790,7 @@ async def run_async(self, params: CalculationInputSchema) -> CalculationOutputSc ForgeStyleTool().run(CalculationInputSchema(expression="x")) tool_span, root_span = spans_of(exporter) - assert tool_span.name == "ForgeStyleTool" + assert tool_span.name == "execute_tool ForgeStyleTool" assert tool_span.parent.span_id == root.get_span_context().span_id assert tool_span.context.trace_id == root_span.context.trace_id @@ -812,7 +812,7 @@ def run(self, params): GeneratedTool().run(CalculationInputSchema(expression="x")) (span,) = spans_of(exporter) - assert span.name == "GeneratedTool" + assert span.name == "execute_tool GeneratedTool" assert output_json(span) == {"result": "mcp"} @@ -847,5 +847,154 @@ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema: finally: instrumentor.uninstrument() - assert [span.name for span in spans_of(exporter)] == ["BetweenCyclesTool"] + assert [span.name for span in spans_of(exporter)] == ["execute_tool BetweenCyclesTool"] assert BetweenCyclesTool.__dict__["run"] is original_run + + +@pytest.fixture +def configured_tracing(): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + yield provider, exporter + AtomicAgentsInstrumentor().uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("openinference_enabled", [False, True]) +def test_genai_attributes_survive_without_openinference(monkeypatch, configured_tracing, openinference_enabled): + provider, exporter = configured_tracing + monkeypatch.setattr(tracing, "_import_context_attributes", lambda: None) + config = tracing.TracingConfig(openinference_attributes=True if openinference_enabled else None) + AtomicAgentsInstrumentor().instrument(tracer_provider=provider, config=config) + client = sync_client() + client.provider = types.SimpleNamespace(value="openai") + make_agent(client).run(QuestionSchema(question="question")) + CalculatorTool().run(CalculationInputSchema(expression="1+1")) + + agent_span, tool_span = spans_of(exporter) + assert [agent_span.name, tool_span.name] == ["invoke_agent AtomicAgent", "execute_tool CalculatorTool"] + assert agent_span.attributes["gen_ai.operation.name"] == "invoke_agent" + assert agent_span.attributes["gen_ai.agent.name"] == "AtomicAgent" + assert agent_span.attributes["gen_ai.request.model"] == "test-model" + assert agent_span.attributes["gen_ai.provider.name"] == "openai" + assert agent_span.attributes["atomic_agents.agent.class"] == "AtomicAgent[QuestionSchema, AnswerSchema]" + assert json.loads(agent_span.attributes["atomic_agents.input"]) == {"question": "question"} + assert json.loads(agent_span.attributes["atomic_agents.output"]) == {"answer": "42"} + assert tool_span.attributes["gen_ai.operation.name"] == "execute_tool" + assert tool_span.attributes["gen_ai.tool.name"] == "CalculatorTool" + assert tool_span.attributes["gen_ai.tool.description"] == "Expression to evaluate." + assert tool_span.attributes["gen_ai.tool.type"] == "function" + assert json.loads(tool_span.attributes["gen_ai.tool.call.arguments"]) == {"expression": "1+1"} + assert json.loads(tool_span.attributes["gen_ai.tool.call.result"]) == {"result": "3.00"} + assert ("openinference.span.kind" in agent_span.attributes) is openinference_enabled + assert ("openinference.span.kind" in tool_span.attributes) is openinference_enabled + if openinference_enabled: + assert agent_span.attributes["input.value"] == agent_span.attributes["atomic_agents.input"] + assert tool_span.attributes["input.value"] == tool_span.attributes["gen_ai.tool.call.arguments"] + assert tool_span.attributes["output.value"] == tool_span.attributes["gen_ai.tool.call.result"] + else: + assert not ({"input.value", "output.value", "agent.name"} & agent_span.attributes.keys()) + assert not ({"input.value", "output.value", "tool.name", "tool.parameters"} & tool_span.attributes.keys()) + + +@pytest.mark.parametrize("config_class", [tracing.TracingConfig, TraceConfig]) +def test_both_visibility_configs_redact_primary_payloads(configured_tracing, config_class): + provider, exporter = configured_tracing + AtomicAgentsInstrumentor().instrument(tracer_provider=provider, config=config_class(hide_inputs=True, hide_outputs=True)) + make_agent(sync_client()).run(QuestionSchema(question="secret")) + CalculatorTool().run(CalculationInputSchema(expression="secret")) + + agent_span, tool_span = spans_of(exporter) + assert agent_span.attributes["atomic_agents.input"] == "__REDACTED__" + assert agent_span.attributes["atomic_agents.output"] == "__REDACTED__" + assert tool_span.attributes["gen_ai.tool.call.arguments"] == "__REDACTED__" + assert tool_span.attributes["gen_ai.tool.call.result"] == "__REDACTED__" + + +@pytest.mark.parametrize("hide_outputs", [False, True]) +def test_large_tool_result_is_truncated_unless_hidden(configured_tracing, hide_outputs): + provider, exporter = configured_tracing + AtomicAgentsInstrumentor().instrument( + tracer_provider=provider, config=tracing.TracingConfig(hide_outputs=hide_outputs, openinference_attributes=True) + ) + + class LargeResultTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]): + def run(self, params: CalculationInputSchema) -> CalculationOutputSchema: + return CalculationOutputSchema(result="x" * 10240) + + result = LargeResultTool().run(CalculationInputSchema(expression="large")) + (span,) = spans_of(exporter) + serialized = result.model_dump_json() + expected = "__REDACTED__" if hide_outputs else serialized[:4096] + "...[truncated]" + assert span.attributes["gen_ai.tool.call.result"] == expected + assert span.attributes["output.value"] == expected + if hide_outputs: + assert "gen_ai.tool.call.result.original_length" not in span.attributes + assert "output.value.original_length" not in span.attributes + else: + assert span.attributes["gen_ai.tool.call.result.original_length"] == len(serialized) + assert span.attributes["output.value.original_length"] == len(serialized) + + +@pytest.mark.parametrize("agent_method", ["run", "run_async", "run_stream", "run_async_stream"]) +@pytest.mark.asyncio +async def test_suppression_skips_agent_and_tool_spans(exporter, agent_method): + from opentelemetry import context + + client = async_client() if "async" in agent_method else sync_client() + agent = make_agent(client) + token = context.attach(context.set_value("suppress_instrumentation", True)) + try: + result = getattr(agent, agent_method)(QuestionSchema(question="suppressed")) + assert await final_agent_answer(agent_method, result) == "42" + assert CalculatorTool().run(CalculationInputSchema(expression="1+1")).result == "3.00" + assert (await ThreadedCalculatorTool().run_async(CalculationInputSchema(expression="1+1"))).result == "3" + finally: + context.detach(token) + assert spans_of(exporter) == [] + + +def test_failures_record_qualified_error_type(exporter): + client = sync_client() + client.chat.completions.create.side_effect = ValueError("agent failure") + with pytest.raises(ValueError, match="agent failure"): + make_agent(client).run(QuestionSchema(question="fail")) + with pytest.raises(ValueError, match="boom"): + FailingTool().run(CalculationInputSchema(expression="fail")) + for span in spans_of(exporter): + assert span.attributes["error.type"] == "ValueError" + assert span.status.status_code == StatusCode.ERROR + assert len(span.events) == 1 + + +def test_tracing_config_environment_and_coercion(monkeypatch): + monkeypatch.setenv("OPENINFERENCE_HIDE_INPUTS", "TrUe") + monkeypatch.setenv("OPENINFERENCE_HIDE_OUTPUTS", "1") + monkeypatch.setenv("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", "128") + from atomic_agents.observability import TracingConfig + + config = TracingConfig.from_environment() + assert config == TracingConfig(hide_inputs=True, hide_outputs=True, max_attribute_length=128) + assert TracingConfig.coerce(None) == config + assert TracingConfig.coerce(config) is config + assert TracingConfig.coerce(TraceConfig(hide_inputs=True)) == TracingConfig(hide_inputs=True) + + +def test_session_is_mirrored_to_genai_conversation(exporter): + with using_session("session-1"): + make_agent(sync_client()).run(QuestionSchema(question="?")) + CalculatorTool().run(CalculationInputSchema(expression="?")) + for span in spans_of(exporter): + assert span.attributes["gen_ai.conversation.id"] == "session-1" + + +async def final_agent_answer(agent_method, result): + if agent_method == "run_async": + return (await result).answer + elif agent_method == "run_stream": + return list(result)[-1].answer + elif agent_method == "run_async_stream": + return [partial async for partial in result][-1].answer + else: + return result.answer From 9a482e5b842b11f5043a12c54c4c0adf6eea8587 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 11:29:20 +0200 Subject: [PATCH 04/25] Test api-only tracing in the fresh-install CI job Co-Authored-By: Claude Code --- .github/workflows/code-quality.yml | 79 ++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 9730e160..7fcb029f 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -84,6 +84,85 @@ jobs: run: | uv venv fresh-install-venv uv pip install --python fresh-install-venv "atomic-agents[observability] @ $(ls dist/*.whl)" + uv pip install --python fresh-install-venv opentelemetry-sdk + + - name: Verify OpenInference is absent + run: | + if fresh-install-venv/bin/python -c "import openinference"; then + echo "OpenInference must not be installed in the api-only environment" + exit 1 + fi + + - name: Trace a tool and agent with only OpenTelemetry + run: | + fresh-install-venv/bin/python - <<'PY' + from types import SimpleNamespace + + from instructor import Instructor + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema, BaseTool + from atomic_agents.observability import AtomicAgentsInstrumentor, TracingConfig + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + AtomicAgentsInstrumentor().instrument(tracer_provider=provider, config=TracingConfig(hide_inputs=True)) + + class SmokeInput(BaseIOSchema): + """Text for the fresh-install tracing check.""" + + text: str + + class SmokeOutput(BaseIOSchema): + """Result of the fresh-install tracing check.""" + + text: str + + class SmokeTool(BaseTool[SmokeInput, SmokeOutput]): + def run(self, params: SmokeInput) -> SmokeOutput: + return SmokeOutput(text=params.text) + + class StubCompletions: + def create(self, **options: object) -> SmokeOutput: + return SmokeOutput(text="offline response") + + class StubClient(Instructor): + chat = SimpleNamespace(completions=StubCompletions()) + provider = None + + def __init__(self) -> None: + pass + + def on(self, event: str, handler: object) -> None: + pass + + def off(self, event: str, handler: object) -> None: + pass + + class SmokeAgent(AtomicAgent[SmokeInput, SmokeOutput]): + pass + + assert SmokeTool().run(SmokeInput(text="tool input")).text == "tool input" + agent = SmokeAgent(config=AgentConfig(client=StubClient(), model="offline-model")) + assert agent.run(SmokeInput(text="private agent input")).text == "offline response" + + spans = exporter.get_finished_spans() + for span in spans: + print(f"{span.name}: {sorted(span.attributes)}") + spans_by_name = {span.name: span for span in spans} + tool_span = spans_by_name["execute_tool SmokeTool"] + agent_span = spans_by_name["invoke_agent SmokeAgent"] + assert tool_span.attributes["gen_ai.operation.name"] == "execute_tool" + assert tool_span.attributes["gen_ai.tool.name"] == "SmokeTool" + assert agent_span.attributes["gen_ai.operation.name"] == "invoke_agent" + assert agent_span.attributes["atomic_agents.input"] == "__REDACTED__" + for span in (tool_span, agent_span): + assert not any(key.startswith("openinference.") for key in span.attributes) + print("api-only tracing ok: tool and agent spans, GenAI attributes, redacted input, no OpenInference keys") + PY - name: Import every subpackage the way a new user would run: | From 975d1daebf2ec72496aaad36638efd569191bb21 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 11:33:41 +0200 Subject: [PATCH 05/25] Add stable agent names and scoped conversation tracing Replace tracer provider and config on re-instrumentation with a warning. Cover context isolation, plain-tracer parenting and provider replacement. Co-Authored-By: Claude Code --- .../atomic_agents/agents/atomic_agent.py | 5 + .../atomic_agents/observability/__init__.py | 10 +- .../atomic_agents/observability/tracing.py | 75 ++++++--- .../tests/agents/test_agent_config_name.py | 22 +++ .../test_opentelemetry_discovery.py | 13 +- .../tests/observability/test_tracing.py | 144 +++++++++++++++++- 6 files changed, 235 insertions(+), 34 deletions(-) create mode 100644 atomic-agents/tests/agents/test_agent_config_name.py diff --git a/atomic-agents/atomic_agents/agents/atomic_agent.py b/atomic-agents/atomic_agents/agents/atomic_agent.py index bf314055..5189986a 100644 --- a/atomic-agents/atomic_agents/agents/atomic_agent.py +++ b/atomic-agents/atomic_agents/agents/atomic_agent.py @@ -68,6 +68,10 @@ class BasicChatOutputSchema(BaseIOSchema): class AgentConfig(BaseModel): client: instructor.Instructor = Field(..., description="Client for interacting with the language model.") + name: Optional[str] = Field( + default=None, + description="Stable name for this agent in logs and traces, for example the role it plays. Defaults to the class name.", + ) model: str = Field(default="gpt-5-mini", description="The model to use for generating responses.") history: Optional[BaseChatHistory] = Field(default=None, description="History component for storing chat history.") system_prompt_generator: Optional[BaseSystemPromptGenerator] = Field( @@ -190,6 +194,7 @@ def __init__(self, config: AgentConfig): Args: config (AgentConfig): Configuration for the chat agent. """ + self.name = config.name self.client = config.client self.model = config.model self.history = config.history if config.history is not None else ChatHistory() diff --git a/atomic-agents/atomic_agents/observability/__init__.py b/atomic-agents/atomic_agents/observability/__init__.py index 41d91622..7cf4fe84 100644 --- a/atomic-agents/atomic_agents/observability/__init__.py +++ b/atomic-agents/atomic_agents/observability/__init__.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from atomic_agents.observability.tracing import PayloadVisibility, TracingConfig + from atomic_agents.observability.tracing import PayloadVisibility, TracingConfig, using_conversation from opentelemetry.trace import TracerProvider logger = logging.getLogger(__name__) @@ -50,8 +50,8 @@ def _load_tracing_module(): def __getattr__(name: str) -> object: - if name == "TracingConfig": - return _load_tracing_module().TracingConfig + if name in ("TracingConfig", "using_conversation"): + return getattr(_load_tracing_module(), name) raise AttributeError(name) @@ -73,7 +73,7 @@ def instrument( ) -> None: """ Patch the framework classes. Safe to call more than once; a second call with different - arguments logs a warning and keeps the first installation. + arguments replaces the tracer provider/config and logs a warning. Args: tracer_provider: An OpenTelemetry ``TracerProvider``. Defaults to the global provider. @@ -104,4 +104,4 @@ def is_instrumented_by_opentelemetry(self) -> bool: return _load_tracing_module().is_installed() -__all__ = ["AtomicAgentsInstrumentor", "TracingConfig"] +__all__ = ["AtomicAgentsInstrumentor", "TracingConfig", "using_conversation"] diff --git a/atomic-agents/atomic_agents/observability/tracing.py b/atomic-agents/atomic_agents/observability/tracing.py index 85be9dea..f275c4fc 100644 --- a/atomic-agents/atomic_agents/observability/tracing.py +++ b/atomic-agents/atomic_agents/observability/tracing.py @@ -135,6 +135,21 @@ def _suppressed() -> bool: return bool(context_api.get_value("suppress_instrumentation")) +_conversation_id: contextvars.ContextVar[str | None] = contextvars.ContextVar("atomic_agents_conversation_id", default=None) +_active_agent_span: contextvars.ContextVar[Span | None] = contextvars.ContextVar( + "atomic_agents_active_agent_span", default=None +) + + +@contextmanager +def using_conversation(conversation_id: str) -> Iterator[None]: + token = _conversation_id.set(conversation_id) + try: + yield + finally: + _conversation_id.reset(token) + + _active_tool: contextvars.ContextVar[Optional[BaseTool]] = contextvars.ContextVar("atomic_agents_active_tool", default=None) @@ -169,7 +184,7 @@ def install( if _installation is not None: if config.openinference_attributes is None: config = replace(config, openinference_attributes=_installation.config.openinference_attributes) - _warn_if_reinstalled_differently(_installation, tracer_provider, config) + _replace_installation(_installation, tracer_provider, config) return context_attributes = _import_context_attributes() if config.openinference_attributes is None: @@ -194,16 +209,19 @@ def uninstall() -> None: _installation = None -def _warn_if_reinstalled_differently( +def _replace_installation( installed: _Installation, tracer_provider: Optional["TracerProvider"], config: TracingConfig ) -> None: - same_provider = tracer_provider is None or tracer_provider is installed.tracer_provider - same_config = config == installed.config - if not (same_provider and same_config): - logger.warning( - "AtomicAgentsInstrumentor is already installed; the new tracer_provider/config are ignored. " - "Call uninstrument() first to change them." - ) + if tracer_provider is installed.tracer_provider and config == installed.config: + return + installed.tracer = trace_api.get_tracer(TRACER_NAME, __version__, tracer_provider) + installed.tracer_provider = tracer_provider + installed.config = config + installed.context_attributes = _import_context_attributes() + logger.warning( + "AtomicAgentsInstrumentor tracer provider/config replaced; " + "in-flight spans finish on the provider they started with." + ) def _instrument_agent_class() -> None: @@ -276,6 +294,10 @@ def _restore_tool_class(tool_class: Type) -> None: def _agent_display_name(agent: AtomicAgent) -> str: + return agent.name or type(agent).__name__ + + +def _agent_class_name(agent: AtomicAgent) -> str: return f"{type(agent).__name__}[{agent.input_schema.__name__}, {agent.output_schema.__name__}]" @@ -283,8 +305,8 @@ def _agent_start_attributes(agent: AtomicAgent) -> Dict[str, Any]: try: return { ATTR_OPERATION: "invoke_agent", - ATTR_AGENT_NAME: type(agent).__name__, - ATTR_AGENT_CLASS: _agent_display_name(agent), + ATTR_AGENT_NAME: _agent_display_name(agent), + ATTR_AGENT_CLASS: _agent_class_name(agent), ATTR_REQUEST_MODEL: agent.model, **_provider_attributes(agent), **_optional_attributes("AGENT", {ATTR_OI_AGENT_NAME: _agent_display_name(agent)}), @@ -327,13 +349,23 @@ def _provider_attributes(agent: AtomicAgent) -> dict[str, str]: def _optional_attributes(kind: str, attributes: dict[str, AttributeValue]) -> dict[str, AttributeValue]: + context = _context_attributes() + conversation = _conversation_attributes(context) if not _current_installation().config.openinference_attributes: + return conversation + return {ATTR_SPAN_KIND: kind, **attributes, **context, **conversation} + + +def _conversation_attributes(context: dict[str, AttributeValue]) -> dict[str, AttributeValue]: + conversation_id: AttributeValue | None = _conversation_id.get() + if conversation_id is None: + conversation_id = context.get(ATTR_SESSION_ID) + if conversation_id is None: return {} - context = _context_attributes() - session = context.get(ATTR_SESSION_ID) - if session is not None: - context[ATTR_CONVERSATION_ID] = session - return {ATTR_SPAN_KIND: kind, **attributes, **context} + attributes = {ATTR_CONVERSATION_ID: conversation_id} + if _current_installation().config.openinference_attributes: + attributes[ATTR_SESSION_ID] = conversation_id + return attributes def _record_payload( @@ -399,17 +431,20 @@ def _record_failure(span: Any, error: BaseException) -> None: def _agent_span(agent: AtomicAgent, args: tuple, kwargs: dict) -> Iterator[Span]: installed = _current_installation() with installed.tracer.start_as_current_span( - f"invoke_agent {type(agent).__name__}", + f"invoke_agent {_agent_display_name(agent)}", attributes=_agent_start_attributes(agent), record_exception=False, set_status_on_exception=False, ) as span: _set_input_safely(span, _user_input_from_call(args, kwargs)) + token = _active_agent_span.set(span) try: yield span except BaseException as error: _record_failure(span, error) raise + finally: + _active_agent_span.reset(token) def _trace_agent_call(method_name: str, wrapped: Callable) -> Callable: @@ -442,7 +477,7 @@ def _start_agent_span(agent: AtomicAgent, method_name: str, args: tuple, kwargs: if _suppressed(): return trace_api.INVALID_SPAN span = _current_installation().tracer.start_span( - f"invoke_agent {type(agent).__name__}", + f"invoke_agent {_agent_display_name(agent)}", attributes=_agent_start_attributes(agent), ) _set_input_safely(span, _user_input_from_call(args, kwargs)) @@ -451,17 +486,21 @@ def _start_agent_span(agent: AtomicAgent, method_name: str, args: tuple, kwargs: def _call_within_span(span: Any, function: Callable, *args: Any) -> Any: token = context_api.attach(trace_api.set_span_in_context(span)) + agent_token = _active_agent_span.set(span) try: return function(*args) finally: + _active_agent_span.reset(agent_token) context_api.detach(token) async def _await_within_span(span: Any, awaitable: Any) -> Any: token = context_api.attach(trace_api.set_span_in_context(span)) + agent_token = _active_agent_span.set(span) try: return await awaitable finally: + _active_agent_span.reset(agent_token) context_api.detach(token) diff --git a/atomic-agents/tests/agents/test_agent_config_name.py b/atomic-agents/tests/agents/test_agent_config_name.py new file mode 100644 index 00000000..707e339d --- /dev/null +++ b/atomic-agents/tests/agents/test_agent_config_name.py @@ -0,0 +1,22 @@ +import instructor +from openai import OpenAI + +from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema + + +def test_agent_name_defaults_to_none() -> None: + with OpenAI(api_key="synthetic-test-key") as provider_client: + client = instructor.from_openai(provider_client) + configuration = AgentConfig(client=client) + agent = AtomicAgent[BaseIOSchema, BaseIOSchema](configuration) + assert configuration.name is None + assert agent.name is None + + +def test_explicit_agent_name_is_stored() -> None: + with OpenAI(api_key="synthetic-test-key") as provider_client: + client = instructor.from_openai(provider_client) + configuration = AgentConfig(client=client, name="orchestrator_planning") + agent = AtomicAgent[BaseIOSchema, BaseIOSchema](configuration) + assert configuration.name == "orchestrator_planning" + assert agent.name == "orchestrator_planning" diff --git a/atomic-agents/tests/observability/test_opentelemetry_discovery.py b/atomic-agents/tests/observability/test_opentelemetry_discovery.py index 0828d70d..ea18ef5c 100644 --- a/atomic-agents/tests/observability/test_opentelemetry_discovery.py +++ b/atomic-agents/tests/observability/test_opentelemetry_discovery.py @@ -111,7 +111,7 @@ def test_loader_skips_missing_opentelemetry_and_continues_without_error( @pytest.mark.parametrize("first_group", ["openinference_instrumentor", "opentelemetry_instrumentor"]) -def test_both_entry_points_keep_first_provider_and_config( +def test_both_entry_points_replace_provider_and_config( first_group: str, local_tracing: tuple[TracerProvider, InMemorySpanExporter], caplog: pytest.LogCaptureFixture ) -> None: provider, exporter = local_tracing @@ -129,12 +129,11 @@ def test_both_entry_points_keep_first_provider_and_config( with caplog.at_level(logging.WARNING): second.instrument(tracer_provider=other_provider, config=TraceConfig(hide_inputs=False), skip_dep_check=True) run_agent_and_tool() - assert "already installed" in caplog.text - assert [exported_attributes(span)[SpanAttributes.INPUT_VALUE] for span in exporter.get_finished_spans()] == [ - "__REDACTED__", - "__REDACTED__", - ] - assert other_exporter.get_finished_spans() == () + assert "tracer provider/config replaced" in caplog.text + assert exporter.get_finished_spans() == () + spans = other_exporter.get_finished_spans() + assert len(spans) == 2 + assert all(exported_attributes(span)[SpanAttributes.INPUT_VALUE] != "__REDACTED__" for span in spans) finally: second.uninstrument() other_provider.shutdown() diff --git a/atomic-agents/tests/observability/test_tracing.py b/atomic-agents/tests/observability/test_tracing.py index 6e6ef8ac..1016f9d2 100644 --- a/atomic-agents/tests/observability/test_tracing.py +++ b/atomic-agents/tests/observability/test_tracing.py @@ -189,7 +189,8 @@ def test_run_emits_agent_span_with_io_and_framework_attributes(exporter): (span,) = spans_of(exporter) assert span.name == "invoke_agent AtomicAgent" assert kind_of(span) == OpenInferenceSpanKindValues.AGENT.value - assert span.attributes[SpanAttributes.AGENT_NAME] == "AtomicAgent[QuestionSchema, AnswerSchema]" + assert span.attributes[SpanAttributes.AGENT_NAME] == "AtomicAgent" + assert span.attributes[tracing.ATTR_AGENT_CLASS] == "AtomicAgent[QuestionSchema, AnswerSchema]" assert input_json(span) == {"question": "6*7?"} assert output_json(span) == {"answer": "42"} assert span.attributes[SpanAttributes.INPUT_MIME_TYPE] == "application/json" @@ -615,15 +616,27 @@ def test_session_and_user_metadata_survive_on_agent_span(exporter): assert json.loads(span.attributes[SpanAttributes.METADATA]) == {"tenant": "acme"} -def test_second_instrument_with_other_settings_warns_and_keeps_first(exporter, caplog): +def test_second_instrument_replaces_provider_and_warns(exporter, caplog): other_provider = TracerProvider() + other_exporter = InMemorySpanExporter() + other_provider.add_span_processor(SimpleSpanProcessor(other_exporter)) + agent = make_agent(sync_client()) + agent.run(QuestionSchema(question="first")) + installed = tracing._installation + wrapped_run = AtomicAgent.run with caplog.at_level(logging.WARNING): AtomicAgentsInstrumentor().instrument(tracer_provider=other_provider) - assert "already installed" in caplog.text - make_agent(sync_client()).run(QuestionSchema(question="?")) + agent.run(QuestionSchema(question="second")) + assert tracing._installation is installed + assert AtomicAgent.run is wrapped_run assert len(spans_of(exporter)) == 1 + assert len(spans_of(other_exporter)) == 1 + assert "tracer provider/config replaced" in caplog.text + assert "in-flight spans finish on the provider they started with" in caplog.text + assert caplog.records[-1].levelno == logging.WARNING + other_provider.shutdown() def test_uninstrument_restores_originals_and_stops_tracing(): @@ -998,3 +1011,126 @@ async def final_agent_answer(agent_method, result): return [partial async for partial in result][-1].answer else: return result.answer + + +@pytest.mark.parametrize( + "name, expected", [(None, "AtomicAgent"), ("", "AtomicAgent"), ("orchestrator_planning", "orchestrator_planning")] +) +def test_stable_agent_name_is_separate_from_schema_class(exporter, name, expected): + agent = AtomicAgent[QuestionSchema, AnswerSchema](AgentConfig(client=sync_client(), name=name)) + agent.run(QuestionSchema(question="?")) + (span,) = spans_of(exporter) + assert span.name == f"invoke_agent {expected}" + assert span.attributes["gen_ai.agent.name"] == expected + assert span.attributes["agent.name"] == expected + assert span.attributes["atomic_agents.agent.class"] == "AtomicAgent[QuestionSchema, AnswerSchema]" + + +@pytest.mark.parametrize("openinference_enabled", [False, True]) +def test_conversation_context_overrides_session_on_agent_and_tool(exporter, openinference_enabled): + config = tracing.TracingConfig(openinference_attributes=openinference_enabled) + AtomicAgentsInstrumentor().instrument(tracer_provider=exporter.provider, config=config) + with using_session("session-1"), observability.using_conversation("c-1"): + make_agent(sync_client()).run(QuestionSchema(question="?")) + CalculatorTool().run(CalculationInputSchema(expression="?")) + agent_span, tool_span = spans_of(exporter) + assert agent_span.attributes["gen_ai.conversation.id"] == "c-1" + assert tool_span.attributes["gen_ai.conversation.id"] == "c-1" + assert agent_span.attributes.get("session.id") == ("c-1" if openinference_enabled else None) + assert tool_span.attributes.get("session.id") == ("c-1" if openinference_enabled else None) + + +def test_session_is_mirrored_even_with_openinference_attributes_disabled(exporter): + config = tracing.TracingConfig(openinference_attributes=False) + AtomicAgentsInstrumentor().instrument(tracer_provider=exporter.provider, config=config) + with using_session("session-1"): + make_agent(sync_client()).run(QuestionSchema(question="?")) + CalculatorTool().run(CalculationInputSchema(expression="?")) + for span in spans_of(exporter): + assert span.attributes["gen_ai.conversation.id"] == "session-1" + assert "session.id" not in span.attributes + + +def test_conversation_context_restores_after_nested_exception(exporter): + with observability.using_conversation("outer"): + with pytest.raises(ValueError), observability.using_conversation("inner"): + CalculatorTool().run(CalculationInputSchema(expression="inner")) + raise ValueError("reset context") + CalculatorTool().run(CalculationInputSchema(expression="outer")) + CalculatorTool().run(CalculationInputSchema(expression="unset")) + inner, outer, unset = spans_of(exporter) + assert inner.attributes["gen_ai.conversation.id"] == "inner" + assert outer.attributes["gen_ai.conversation.id"] == "outer" + assert "gen_ai.conversation.id" not in unset.attributes + + +@pytest.mark.asyncio +async def test_conversation_context_is_task_local(exporter): + async def run_in_conversation(conversation_id): + with observability.using_conversation(conversation_id): + await asyncio.sleep(0) + await make_agent(async_client()).run_async(QuestionSchema(question="?")) + CalculatorTool().run(CalculationInputSchema(expression="?")) + + await asyncio.gather(run_in_conversation("first"), run_in_conversation("second")) + assert [span.attributes["gen_ai.conversation.id"] for span in spans_of(exporter)] == ["first", "first", "second", "second"] + + +def test_plain_parent_span_owns_agent_and_tool_trace(exporter): + with exporter.provider.get_tracer("app").start_as_current_span("turn") as turn: + make_agent(sync_client()).run(QuestionSchema(question="?")) + CalculatorTool().run(CalculationInputSchema(expression="?")) + agent_span, tool_span, parent = spans_of(exporter) + assert agent_span.context.trace_id == turn.get_span_context().trace_id + assert tool_span.context.trace_id == turn.get_span_context().trace_id + assert agent_span.parent.span_id == parent.context.span_id + assert tool_span.parent.span_id == parent.context.span_id + + +def test_same_provider_and_config_is_a_silent_no_op(exporter, caplog): + installed = tracing._installation + tracer = installed.tracer + with caplog.at_level(logging.DEBUG, logger=tracing.__name__): + AtomicAgentsInstrumentor().instrument(tracer_provider=exporter.provider, config=tracing.TracingConfig()) + assert tracing._installation is installed + assert installed.tracer is tracer + assert not caplog.records + + +def test_reinstrument_replaces_config_without_rewrapping(exporter, caplog): + wrapped_run = AtomicAgent.run + agent = make_agent(sync_client()) + config = tracing.TracingConfig(hide_inputs=True, openinference_attributes=False) + with caplog.at_level(logging.WARNING): + AtomicAgentsInstrumentor().instrument(tracer_provider=exporter.provider, config=config) + agent.run(QuestionSchema(question="secret")) + CalculatorTool().run(CalculationInputSchema(expression="secret")) + agent_span, tool_span = spans_of(exporter) + assert agent_span.attributes["atomic_agents.input"] == "__REDACTED__" + assert tool_span.attributes["gen_ai.tool.call.arguments"] == "__REDACTED__" + assert "agent.name" not in agent_span.attributes + assert AtomicAgent.run is wrapped_run + assert "tracer provider/config replaced" in caplog.text + + +def test_in_flight_span_finishes_on_original_provider(exporter): + other_provider = TracerProvider() + other_exporter = InMemorySpanExporter() + other_provider.add_span_processor(SimpleSpanProcessor(other_exporter)) + client = sync_client() + + def replace_provider(**kwargs): + AtomicAgentsInstrumentor().instrument(tracer_provider=other_provider) + CalculatorTool().run(CalculationInputSchema(expression="?")) + return AnswerSchema(answer="42") + + client.chat.completions.create.side_effect = replace_provider + make_agent(client).run(QuestionSchema(question="?")) + assert len(spans_of(exporter)) == 1 + assert len(spans_of(other_exporter)) == 1 + (agent_span,) = spans_of(exporter) + (tool_span,) = spans_of(other_exporter) + assert agent_span.name == "invoke_agent AtomicAgent" + assert tool_span.parent.span_id == agent_span.context.span_id + assert tool_span.context.trace_id == agent_span.context.trace_id + other_provider.shutdown() From c318f75f751b47441c0a1e594bd0b486acce75fe Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 11:34:15 +0200 Subject: [PATCH 06/25] Test the OpenInference floors through the phoenix extra The lowest-versions job compiled only the observability extra, which no longer includes the OpenInference packages after the extras split, so test_tracing.py and test_opentelemetry_discovery.py failed at collection with ModuleNotFoundError: No module named 'openinference' (run 39577808157). Compile the phoenix extra too so those floors stay tested. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/code-quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 7fcb029f..67642deb 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -205,7 +205,7 @@ jobs: - name: Resolve runtime dependencies at their declared floors run: | echo 'opentelemetry-instrumentation==0.51b0' > telemetry-test-requirements.txt - uv pip compile pyproject.toml telemetry-test-requirements.txt --resolution lowest-direct --extra observability -o lowest-requirements.txt + uv pip compile pyproject.toml telemetry-test-requirements.txt --resolution lowest-direct --extra observability --extra phoenix -o lowest-requirements.txt - name: Install the floors plus current test tooling run: | From b5ffaa4117a04a0c24b3109adbf150add1a33011 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 11:48:07 +0200 Subject: [PATCH 07/25] Fix tracing identity, privacy and consumer attributes Use model-visible tool names, collect descriptive attributes independently, filter absent values, cap error/schema text, map provider names and honor Atomic Agents visibility settings. Warn about unknown launcher options. Verified 112 observability tests, Pyright, Black and Flake8. Co-Authored-By: Claude Code --- .../atomic_agents/observability/__init__.py | 12 +- .../atomic_agents/observability/tracing.py | 169 ++++++++++----- .../test_opentelemetry_discovery.py | 9 +- .../tests/observability/test_tracing.py | 193 ++++++++++++++++-- 4 files changed, 321 insertions(+), 62 deletions(-) diff --git a/atomic-agents/atomic_agents/observability/__init__.py b/atomic-agents/atomic_agents/observability/__init__.py index 7cf4fe84..d19c2e23 100644 --- a/atomic-agents/atomic_agents/observability/__init__.py +++ b/atomic-agents/atomic_agents/observability/__init__.py @@ -20,8 +20,9 @@ logger = logging.getLogger(__name__) _INSTALL_HINT = ( - "Atomic Agents observability needs the OpenInference packages. " - 'Install them with: pip install "atomic-agents[observability]"' + "Atomic Agents tracing needs opentelemetry-api. " + 'Install it with: pip install "atomic-agents[observability]" ' + '(add "atomic-agents[phoenix]" for Arize Phoenix / OpenInference attributes)' ) _OPENINFERENCE_DISTRIBUTION = "openinference-instrumentation" _MINIMUM_OPENINFERENCE_VERSION = (0, 1, 38) @@ -55,6 +56,12 @@ def __getattr__(name: str) -> object: raise AttributeError(name) +def _warn_unknown_options(options: dict[str, object]) -> None: + for key in options: + if key != "skip_dep_check": + logger.warning("Unknown Atomic Agents instrumentation option %s ignored", key) + + class AtomicAgentsInstrumentor: """ Installs AGENT and TOOL spans around ``AtomicAgent`` and ``BaseTool`` calls. @@ -84,6 +91,7 @@ def instrument( ModuleNotFoundError: When the ``observability`` extra is not installed, except during OpenTelemetry auto-discovery, which skips unavailable optional instrumentation. """ + _warn_unknown_options(options) if not _opentelemetry_is_installed(): if options.get("skip_dep_check") is True: logger.debug(_INSTALL_HINT) diff --git a/atomic-agents/atomic_agents/observability/tracing.py b/atomic-agents/atomic_agents/observability/tracing.py index f275c4fc..5d0bb8dc 100644 --- a/atomic-agents/atomic_agents/observability/tracing.py +++ b/atomic-agents/atomic_agents/observability/tracing.py @@ -12,6 +12,7 @@ import inspect import json import logging +from collections.abc import Mapping from contextlib import contextmanager from dataclasses import dataclass, replace import os @@ -22,7 +23,7 @@ from opentelemetry import trace as trace_api from opentelemetry.trace import Span, Status, StatusCode, Tracer from opentelemetry.util.types import AttributeValue -from pydantic import BaseModel +from pydantic import BaseModel, ValidationError from atomic_agents import __version__ from atomic_agents.agents.atomic_agent import AtomicAgent @@ -76,6 +77,17 @@ ATTR_OUTPUT = "atomic_agents.output" ATTR_AGENT_CLASS = "atomic_agents.agent.class" ATTR_ERROR_TYPE = "error.type" +ATTR_TOOL_CLASS = "atomic_agents.tool.class" +TRUNCATION_MARKER = "...[truncated]" +PROVIDER_NAMES = { + "azure_openai": "azure.ai.openai", + "vertexai": "gcp.vertex_ai", + "gemini": "gcp.gemini", + "genai": "gcp.gemini", + "bedrock": "aws.bedrock", + "mistral": "mistral_ai", + "xai": "x_ai", +} class PayloadVisibility(Protocol): @@ -94,8 +106,8 @@ class TracingConfig: @classmethod def from_environment(cls) -> "TracingConfig": return cls( - hide_inputs=os.environ.get("OPENINFERENCE_HIDE_INPUTS", "").lower() in ("true", "1"), - hide_outputs=os.environ.get("OPENINFERENCE_HIDE_OUTPUTS", "").lower() in ("true", "1"), + hide_inputs=_visibility_environment("INPUTS").lower() in ("true", "1"), + hide_outputs=_visibility_environment("OUTPUTS").lower() in ("true", "1"), max_attribute_length=int(os.environ.get("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", "4096")), ) @@ -108,6 +120,10 @@ def coerce(cls, config: "TracingConfig | PayloadVisibility | None") -> "TracingC return cls(hide_inputs=config.hide_inputs, hide_outputs=config.hide_outputs) +def _visibility_environment(direction: str) -> str: + return os.environ.get(f"ATOMIC_AGENTS_TRACE_HIDE_{direction}", os.environ.get(f"OPENINFERENCE_HIDE_{direction}", "")) + + def _import_context_attributes() -> Callable[[], dict[str, AttributeValue]] | None: try: from openinference.instrumentation import get_attributes_from_context @@ -301,26 +317,47 @@ def _agent_class_name(agent: AtomicAgent) -> str: return f"{type(agent).__name__}[{agent.input_schema.__name__}, {agent.output_schema.__name__}]" -def _agent_start_attributes(agent: AtomicAgent) -> Dict[str, Any]: - try: - return { - ATTR_OPERATION: "invoke_agent", - ATTR_AGENT_NAME: _agent_display_name(agent), - ATTR_AGENT_CLASS: _agent_class_name(agent), - ATTR_REQUEST_MODEL: agent.model, - **_provider_attributes(agent), - **_optional_attributes("AGENT", {ATTR_OI_AGENT_NAME: _agent_display_name(agent)}), - ATTR_MODEL: agent.model, - ATTR_MODE: getattr(agent.mode, "value", str(agent.mode)), - ATTR_INPUT_SCHEMA: agent.input_schema.__name__, - ATTR_OUTPUT_SCHEMA: agent.output_schema.__name__, - ATTR_HISTORY_MESSAGE_COUNT: len(agent.history.history), - ATTR_CONTEXT_PROVIDERS: _context_provider_names(agent), - **_system_prompt_version(agent), - } - except Exception as error: - logger.warning("Could not describe agent for tracing: %s", error) - return {} +def _best_effort_attributes( + readers: dict[str, Callable[[], AttributeValue | None]], +) -> dict[str, AttributeValue]: + attributes: dict[str, AttributeValue] = {} + for key, read in readers.items(): + try: + value = read() + if value is not None: + attributes[key] = value + except Exception as error: + logger.debug("Could not collect tracing attribute %s: %s", key, error) + return attributes + + +def _without_none(attributes: Mapping[str, AttributeValue | None]) -> dict[str, AttributeValue]: + return {key: value for key, value in attributes.items() if value is not None} + + +def _agent_start_attributes(agent: AtomicAgent) -> dict[str, AttributeValue]: + attributes: dict[str, AttributeValue] = { + ATTR_OPERATION: "invoke_agent", + ATTR_AGENT_NAME: _agent_display_name(agent), + ATTR_AGENT_CLASS: _agent_class_name(agent), + } + attributes.update( + _best_effort_attributes( + { + ATTR_REQUEST_MODEL: lambda: agent.model, + ATTR_MODEL: lambda: agent.model, + ATTR_MODE: lambda: getattr(agent.mode, "value", None), + ATTR_INPUT_SCHEMA: lambda: agent.input_schema.__name__, + ATTR_OUTPUT_SCHEMA: lambda: agent.output_schema.__name__, + ATTR_HISTORY_MESSAGE_COUNT: lambda: len(agent.history.history), + ATTR_CONTEXT_PROVIDERS: lambda: _context_provider_names(agent), + ATTR_SYSTEM_PROMPT_SHA256: lambda: _system_prompt_version(agent).get(ATTR_SYSTEM_PROMPT_SHA256), + ATTR_PROVIDER_NAME: lambda: _provider_attributes(agent).get(ATTR_PROVIDER_NAME), + } + ) + ) + attributes.update(_optional_attributes("AGENT", {ATTR_OI_AGENT_NAME: _agent_display_name(agent)})) + return _without_none(attributes) def _context_provider_names(agent: AtomicAgent) -> List[str]: @@ -345,7 +382,7 @@ def _user_input_from_call(args: tuple, kwargs: dict) -> Any: def _provider_attributes(agent: AtomicAgent) -> dict[str, str]: provider = getattr(getattr(agent.client, "provider", None), "value", None) - return {ATTR_PROVIDER_NAME: provider} if isinstance(provider, str) else {} + return {ATTR_PROVIDER_NAME: PROVIDER_NAMES.get(provider, provider)} if isinstance(provider, str) else {} def _optional_attributes(kind: str, attributes: dict[str, AttributeValue]) -> dict[str, AttributeValue]: @@ -353,7 +390,7 @@ def _optional_attributes(kind: str, attributes: dict[str, AttributeValue]) -> di conversation = _conversation_attributes(context) if not _current_installation().config.openinference_attributes: return conversation - return {ATTR_SPAN_KIND: kind, **attributes, **context, **conversation} + return _without_none({ATTR_SPAN_KIND: kind, **attributes, **context, **conversation}) def _conversation_attributes(context: dict[str, AttributeValue]) -> dict[str, AttributeValue]: @@ -368,15 +405,21 @@ def _conversation_attributes(context: dict[str, AttributeValue]) -> dict[str, At return attributes +def _serialize_payload(value: object) -> str | None: + try: + return value.model_dump_json() if isinstance(value, BaseModel) else json.dumps(value, default=str) + except Exception as error: + logger.warning("Could not serialize tracing payload: %s", error) + return None + + def _record_payload( span: Span, key_pairs: tuple[tuple[str, str | None], ...], value: object, hide: bool, config: TracingConfig ) -> None: if value is None: return - try: - serialized = value.model_dump_json() if isinstance(value, BaseModel) else json.dumps(value, default=str) - except Exception as error: - logger.warning("Could not serialize tracing payload: %s", error) + serialized = "__REDACTED__" if hide else _serialize_payload(value) + if serialized is None: return for key, mime_key in key_pairs: _write_payload(span, key, mime_key, serialized, hide, config) @@ -388,7 +431,7 @@ def _write_payload(span: Span, key: str, mime_key: str | None, serialized: str, return if len(serialized) > config.max_attribute_length: span.set_attribute(key + ".original_length", len(serialized)) - serialized = serialized[: config.max_attribute_length] + "...[truncated]" + serialized = _bounded_text(serialized, config) span.set_attribute(key, serialized) if mime_key is not None: span.set_attribute(mime_key, "application/json") @@ -421,10 +464,24 @@ def _mark_closed_early(span: Any, output: Any) -> None: _set_output_safely(span, output) -def _record_failure(span: Any, error: BaseException) -> None: +def _bounded_text(value: str, config: TracingConfig) -> str: + if len(value) > config.max_attribute_length: + return value[: config.max_attribute_length] + TRUNCATION_MARKER + return value + + +def _failure_message(error: BaseException, config: TracingConfig) -> str: + if config.hide_outputs and isinstance(error, ValidationError): + return f"ValidationError: {error.error_count()} errors (__REDACTED__)" + return str(error) + + +def _record_failure(span: Span, error: BaseException) -> None: + config = _current_installation().config + message = _bounded_text(_failure_message(error, config), config) span.set_attribute(ATTR_ERROR_TYPE, type(error).__qualname__) - span.record_exception(error) - span.set_status(Status(StatusCode.ERROR, str(error))) + span.record_exception(error, attributes={"exception.message": message, "exception.stacktrace": message}) + span.set_status(Status(StatusCode.ERROR, message)) @contextmanager @@ -595,33 +652,53 @@ async def traced(agent, *args, **kwargs): return traced +def _tool_parameter_attributes(tool: BaseTool) -> dict[str, AttributeValue]: + parameters = _best_effort_attributes({ATTR_TOOL_PARAMETERS: lambda: json.dumps(tool.input_schema.model_json_schema())}) + value = parameters.get(ATTR_TOOL_PARAMETERS) + if isinstance(value, str): + config = _current_installation().config + parameters[ATTR_TOOL_PARAMETERS] = _bounded_text(value, config) + if len(value) > config.max_attribute_length: + parameters[ATTR_TOOL_PARAMETERS + ".original_length"] = len(value) + return parameters + + +def _tool_display_name(tool: BaseTool) -> str: + try: + return tool.tool_name + except Exception as error: + logger.debug("Could not collect tracing tool name: %s", error) + return type(tool).__name__ + + def _tool_start_attributes(tool: BaseTool) -> dict[str, AttributeValue]: attributes: dict[str, AttributeValue] = { ATTR_OPERATION: "execute_tool", - ATTR_TOOL_NAME: type(tool).__name__, + ATTR_TOOL_NAME: _tool_display_name(tool), + ATTR_TOOL_CLASS: type(tool).__name__, ATTR_TOOL_TYPE: "function", **_optional_attributes("TOOL", {}), } - try: - attributes[ATTR_TOOL_DESCRIPTION] = tool.tool_description - if _current_installation().config.openinference_attributes: - attributes.update( + attributes.update(_best_effort_attributes({ATTR_TOOL_DESCRIPTION: lambda: tool.tool_description})) + if _current_installation().config.openinference_attributes: + attributes.update( + _best_effort_attributes( { - ATTR_OI_TOOL_NAME: tool.tool_name, - ATTR_OI_TOOL_DESCRIPTION: tool.tool_description, - ATTR_TOOL_PARAMETERS: json.dumps(tool.input_schema.model_json_schema()), + ATTR_OI_TOOL_NAME: lambda: attributes[ATTR_TOOL_NAME], + ATTR_OI_TOOL_DESCRIPTION: lambda: tool.tool_description, } ) - except Exception as error: - logger.warning("Could not describe tool %s for tracing: %s", type(tool).__name__, error) - return attributes + ) + attributes.update(_tool_parameter_attributes(tool)) + return _without_none(attributes) @contextmanager def _tool_span(tool: BaseTool, args: tuple, kwargs: dict) -> Iterator[Span]: + attributes = _tool_start_attributes(tool) with _current_installation().tracer.start_as_current_span( - f"execute_tool {type(tool).__name__}", - attributes=_tool_start_attributes(tool), + f"execute_tool {attributes[ATTR_TOOL_NAME]}", + attributes=attributes, record_exception=False, set_status_on_exception=False, ) as span: diff --git a/atomic-agents/tests/observability/test_opentelemetry_discovery.py b/atomic-agents/tests/observability/test_opentelemetry_discovery.py index ea18ef5c..b201c1fe 100644 --- a/atomic-agents/tests/observability/test_opentelemetry_discovery.py +++ b/atomic-agents/tests/observability/test_opentelemetry_discovery.py @@ -72,6 +72,7 @@ def test_installed_opentelemetry_entry_point_activates_real_loader( def test_unknown_launcher_options_preserve_provider_and_privacy( local_tracing: tuple[TracerProvider, InMemorySpanExporter], + caplog: pytest.LogCaptureFixture, ) -> None: provider, exporter = local_tracing AtomicAgentsInstrumentor().instrument( @@ -82,6 +83,9 @@ def test_unknown_launcher_options_preserve_provider_and_privacy( ) run_agent_and_tool() + assert "custom_launcher_option" in caplog.text + assert any(record.levelno == logging.WARNING for record in caplog.records) + assert "skip_dep_check" not in caplog.text spans = exporter.get_finished_spans() assert len(spans) == 2 assert [exported_attributes(span)[SpanAttributes.INPUT_VALUE] for span in spans] == ["__REDACTED__", "__REDACTED__"] @@ -105,7 +109,10 @@ def test_loader_skips_missing_opentelemetry_and_continues_without_error( _load._load_instrumentors(DefaultDistro()) sentinel.load.return_value.return_value.instrument.assert_called_once_with(skip_dep_check=True) - assert "atomic-agents[observability]" in caplog.text + assert ( + 'Atomic Agents tracing needs opentelemetry-api. Install it with: pip install "atomic-agents[observability]" ' + '(add "atomic-agents[phoenix]" for Arize Phoenix / OpenInference attributes)' + ) in caplog.text assert not [record for record in caplog.records if record.levelno >= logging.ERROR] assert not AtomicAgentsInstrumentor().is_instrumented_by_opentelemetry diff --git a/atomic-agents/tests/observability/test_tracing.py b/atomic-agents/tests/observability/test_tracing.py index 1016f9d2..84fe51e5 100644 --- a/atomic-agents/tests/observability/test_tracing.py +++ b/atomic-agents/tests/observability/test_tracing.py @@ -8,7 +8,7 @@ from concurrent.futures import ThreadPoolExecutor from importlib.metadata import entry_points from typing import Dict, List -from unittest.mock import Mock +from unittest.mock import Mock, PropertyMock import instructor import pytest @@ -19,7 +19,7 @@ from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace import StatusCode -from pydantic import Field +from pydantic import Field, ValidationError from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema, BaseTool from atomic_agents.context import BaseDynamicContextProvider, SystemPromptGenerator @@ -485,7 +485,7 @@ def test_tool_defined_before_instrumentation_emits_tool_span(exporter): assert result.result == "3.0" (span,) = spans_of(exporter) - assert span.name == "execute_tool CalculatorTool" + assert span.name == "execute_tool CalculationInputSchema" assert kind_of(span) == OpenInferenceSpanKindValues.TOOL.value assert span.attributes[SpanAttributes.TOOL_NAME] == "CalculationInputSchema" assert span.attributes[SpanAttributes.TOOL_DESCRIPTION] == "Expression to evaluate." @@ -511,7 +511,7 @@ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema: LateTool().run(CalculationInputSchema(expression="x")) (span,) = spans_of(exporter) - assert span.name == "execute_tool LateTool" + assert span.name == "execute_tool CalculationInputSchema" assert LateTool().input_schema is CalculationInputSchema @@ -523,14 +523,14 @@ def run(self, params: CalculationInputSchema, precision: int = 2) -> Calculation LoudCalculatorTool().run(CalculationInputSchema(expression="2*2")) (span,) = spans_of(exporter) - assert span.name == "execute_tool LoudCalculatorTool" + assert span.name == "execute_tool CalculationInputSchema" def test_sync_run_delegating_to_arun_emits_one_span(exporter): AsyncCalculatorTool().run(CalculationInputSchema(expression="2*2")) (span,) = spans_of(exporter) - assert span.name == "execute_tool AsyncCalculatorTool" + assert span.name == "execute_tool CalculationInputSchema" assert output_json(span) == {"result": "3"} @@ -539,7 +539,7 @@ async def test_run_async_delegating_to_thread_emits_one_span(exporter): await ThreadedCalculatorTool().run_async(CalculationInputSchema(expression="3*3")) (span,) = spans_of(exporter) - assert span.name == "execute_tool ThreadedCalculatorTool" + assert span.name == "execute_tool CalculationInputSchema" assert output_json(span) == {"result": "3"} @@ -547,7 +547,7 @@ def test_mixin_defined_run_is_traced(exporter): MixinCalculatorTool().run(CalculationInputSchema(expression="x")) (span,) = spans_of(exporter) - assert span.name == "execute_tool MixinCalculatorTool" + assert span.name == "execute_tool CalculationInputSchema" def test_static_run_is_left_alone(exporter): @@ -803,7 +803,7 @@ async def run_async(self, params: CalculationInputSchema) -> CalculationOutputSc ForgeStyleTool().run(CalculationInputSchema(expression="x")) tool_span, root_span = spans_of(exporter) - assert tool_span.name == "execute_tool ForgeStyleTool" + assert tool_span.name == "execute_tool CalculationInputSchema" assert tool_span.parent.span_id == root.get_span_context().span_id assert tool_span.context.trace_id == root_span.context.trace_id @@ -825,7 +825,7 @@ def run(self, params): GeneratedTool().run(CalculationInputSchema(expression="x")) (span,) = spans_of(exporter) - assert span.name == "execute_tool GeneratedTool" + assert span.name == "execute_tool CalculationInputSchema" assert output_json(span) == {"result": "mcp"} @@ -860,7 +860,7 @@ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema: finally: instrumentor.uninstrument() - assert [span.name for span in spans_of(exporter)] == ["execute_tool BetweenCyclesTool"] + assert [span.name for span in spans_of(exporter)] == ["execute_tool CalculationInputSchema"] assert BetweenCyclesTool.__dict__["run"] is original_run @@ -886,7 +886,7 @@ def test_genai_attributes_survive_without_openinference(monkeypatch, configured_ CalculatorTool().run(CalculationInputSchema(expression="1+1")) agent_span, tool_span = spans_of(exporter) - assert [agent_span.name, tool_span.name] == ["invoke_agent AtomicAgent", "execute_tool CalculatorTool"] + assert [agent_span.name, tool_span.name] == ["invoke_agent AtomicAgent", "execute_tool CalculationInputSchema"] assert agent_span.attributes["gen_ai.operation.name"] == "invoke_agent" assert agent_span.attributes["gen_ai.agent.name"] == "AtomicAgent" assert agent_span.attributes["gen_ai.request.model"] == "test-model" @@ -895,7 +895,8 @@ def test_genai_attributes_survive_without_openinference(monkeypatch, configured_ assert json.loads(agent_span.attributes["atomic_agents.input"]) == {"question": "question"} assert json.loads(agent_span.attributes["atomic_agents.output"]) == {"answer": "42"} assert tool_span.attributes["gen_ai.operation.name"] == "execute_tool" - assert tool_span.attributes["gen_ai.tool.name"] == "CalculatorTool" + assert tool_span.attributes["gen_ai.tool.name"] == "CalculationInputSchema" + assert tool_span.attributes["atomic_agents.tool.class"] == "CalculatorTool" assert tool_span.attributes["gen_ai.tool.description"] == "Expression to evaluate." assert tool_span.attributes["gen_ai.tool.type"] == "function" assert json.loads(tool_span.attributes["gen_ai.tool.call.arguments"]) == {"expression": "1+1"} @@ -1134,3 +1135,169 @@ def replace_provider(**kwargs): assert tool_span.parent.span_id == agent_span.context.span_id assert tool_span.context.trace_id == agent_span.context.trace_id other_provider.shutdown() + + +@pytest.mark.parametrize("name", ["model", "mode"]) +def test_none_descriptions_are_omitted_without_otel_warnings(exporter, caplog, name): + agent = make_agent(sync_client()) + setattr(agent, name, None) + with tracing._agent_span(agent, (), {}): + pass + (span,) = spans_of(exporter) + assert f"atomic_agents.{name}" not in span.attributes + assert "gen_ai.request.model" not in span.attributes if name == "model" else True + assert "Invalid type" not in caplog.text + + +def test_broken_history_preserves_mandatory_agent_attributes(exporter, monkeypatch, caplog): + agent = make_agent(sync_client()) + monkeypatch.setattr(type(agent), "history", PropertyMock(side_effect=RuntimeError("history unavailable")), raising=False) + with caplog.at_level(logging.DEBUG), tracing._agent_span(agent, (), {}): + pass + (span,) = spans_of(exporter) + assert span.attributes["gen_ai.operation.name"] == "invoke_agent" + assert span.attributes["gen_ai.agent.name"] == "AtomicAgent" + assert span.attributes["atomic_agents.agent.class"] == "AtomicAgent[QuestionSchema, AnswerSchema]" + assert span.attributes["gen_ai.request.model"] == "test-model" + assert "atomic_agents.history.message_count" not in span.attributes + assert sum("history.message_count" in record.message for record in caplog.records) == 1 + + +def test_broken_tool_description_preserves_other_attributes(exporter, monkeypatch): + monkeypatch.setattr(CalculatorTool, "tool_description", PropertyMock(side_effect=RuntimeError("unavailable"))) + CalculatorTool().run(CalculationInputSchema(expression="1+1")) + (span,) = spans_of(exporter) + assert span.attributes["gen_ai.tool.name"] == "CalculationInputSchema" + assert span.attributes["atomic_agents.tool.class"] == "CalculatorTool" + assert span.attributes["tool.name"] == "CalculationInputSchema" + assert "tool.parameters" in span.attributes + assert "gen_ai.tool.description" not in span.attributes + + +def test_configured_tool_name_is_the_span_join_key(exporter): + tool = CalculatorTool() + tool.config.title = "calculate" + tool.run(CalculationInputSchema(expression="1+1")) + (span,) = spans_of(exporter) + assert span.name == "execute_tool calculate" + assert span.attributes["gen_ai.tool.name"] == span.attributes["tool.name"] == "calculate" + assert span.attributes["atomic_agents.tool.class"] == "CalculatorTool" + + +@pytest.mark.parametrize("hide", [False, True]) +def test_hidden_payload_is_never_serialized(exporter, caplog, hide): + payload = UnserializableInputSchema(store=AnswerStore()) + with exporter.provider.get_tracer("test").start_as_current_span("payload") as span: + tracing._record_payload(span, (("payload", "payload.mime_type"),), payload, hide, tracing.TracingConfig()) + (span,) = spans_of(exporter) + assert ("Could not serialize tracing payload" in caplog.text) is not hide + assert span.attributes.get("payload") == ("__REDACTED__" if hide else None) + assert "payload.mime_type" not in span.attributes + + +@pytest.mark.parametrize("hide", [False, True]) +def test_hidden_non_model_payload_does_not_call_json_dumps(exporter, monkeypatch, caplog, hide): + serialize = Mock(side_effect=ValueError("serialization failed")) + monkeypatch.setattr(tracing.json, "dumps", serialize) + with exporter.provider.get_tracer("test").start_as_current_span("payload") as span: + tracing._record_payload(span, (("payload", None),), object(), hide, tracing.TracingConfig()) + assert serialize.call_count == (0 if hide else 1) + assert ("Could not serialize tracing payload" in caplog.text) is not hide + + +@pytest.mark.parametrize( + "provider,expected", + [ + ("azure_openai", "azure.ai.openai"), + ("vertexai", "gcp.vertex_ai"), + ("gemini", "gcp.gemini"), + ("genai", "gcp.gemini"), + ("bedrock", "aws.bedrock"), + ("mistral", "mistral_ai"), + ("xai", "x_ai"), + ("openai", "openai"), + ("anthropic", "anthropic"), + ("cohere", "cohere"), + ("groq", "groq"), + ("deepseek", "deepseek"), + ("perplexity", "perplexity"), + ("custom", "custom"), + ], +) +def test_instructor_provider_names_follow_genai_conventions(exporter, provider, expected): + client = sync_client() + client.provider = types.SimpleNamespace(value=provider) + make_agent(client).run(QuestionSchema(question="question")) + (span,) = spans_of(exporter) + assert span.attributes["gen_ai.provider.name"] == expected + + +def test_none_context_attributes_are_filtered(exporter, monkeypatch, caplog): + monkeypatch.setattr(tracing, "_context_attributes", lambda: {"optional": None}) + make_agent(sync_client()).run(QuestionSchema(question="question")) + (span,) = spans_of(exporter) + assert "optional" not in span.attributes + assert "gen_ai.provider.name" not in span.attributes + assert "Invalid type" not in caplog.text + + +@pytest.mark.parametrize("prefix", ["ATOMIC_AGENTS_TRACE", "OPENINFERENCE"]) +def test_visibility_environment_supports_both_spellings(monkeypatch, prefix): + monkeypatch.delenv("ATOMIC_AGENTS_TRACE_HIDE_INPUTS", raising=False) + monkeypatch.delenv("ATOMIC_AGENTS_TRACE_HIDE_OUTPUTS", raising=False) + monkeypatch.setenv(f"{prefix}_HIDE_INPUTS", "TRUE") + monkeypatch.setenv(f"{prefix}_HIDE_OUTPUTS", "1") + config = tracing.TracingConfig.from_environment() + assert config.hide_inputs is True + assert config.hide_outputs is True + + +def test_atomic_visibility_environment_overrides_openinference(monkeypatch): + monkeypatch.setenv("OPENINFERENCE_HIDE_INPUTS", "true") + monkeypatch.setenv("OPENINFERENCE_HIDE_OUTPUTS", "true") + monkeypatch.setenv("ATOMIC_AGENTS_TRACE_HIDE_INPUTS", "false") + monkeypatch.setenv("ATOMIC_AGENTS_TRACE_HIDE_OUTPUTS", "") + config = tracing.TracingConfig.from_environment() + assert config.hide_inputs is False + assert config.hide_outputs is False + + +def test_error_status_and_recorded_message_are_bounded(configured_tracing): + provider, exporter = configured_tracing + AtomicAgentsInstrumentor().instrument(tracer_provider=provider, config=tracing.TracingConfig(max_attribute_length=12)) + error = ValueError("private" * 100) + agent = make_agent(sync_client()) + with pytest.raises(ValueError) as raised, tracing._agent_span(agent, (), {}): + raise error + (span,) = spans_of(exporter) + expected = str(error)[:12] + "...[truncated]" + assert raised.value is error + assert span.status.description == expected + assert span.events[0].attributes["exception.message"] == expected + assert len(span.events[0].attributes["exception.stacktrace"]) <= 12 + len("...[truncated]") + + +def test_hidden_validation_errors_do_not_export_model_output(configured_tracing): + provider, exporter = configured_tracing + AtomicAgentsInstrumentor().instrument(tracer_provider=provider, config=tracing.TracingConfig(hide_outputs=True)) + try: + QuestionSchema.model_validate({"question": ["secret model output"]}) + except ValidationError as error: + with provider.get_tracer("test").start_as_current_span("failure") as span: + tracing._record_failure(span, error) + (span,) = spans_of(exporter) + assert span.status.description == "ValidationError: 1 errors (__REDACTED__)" + assert span.events[0].attributes["exception.message"] == span.status.description + assert "secret model output" not in str(span.events[0].attributes) + + +def test_tool_parameters_use_the_payload_size_limit(configured_tracing): + provider, exporter = configured_tracing + AtomicAgentsInstrumentor().instrument( + tracer_provider=provider, config=tracing.TracingConfig(max_attribute_length=12, openinference_attributes=True) + ) + CalculatorTool().run(CalculationInputSchema(expression="x")) + (span,) = spans_of(exporter) + parameters = json.dumps(CalculationInputSchema.model_json_schema()) + assert span.attributes["tool.parameters"] == parameters[:12] + "...[truncated]" + assert span.attributes["tool.parameters.original_length"] == len(parameters) From 9f0e0b14dd9ce53a273e52ccc2c827ba065e7581 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 12:01:07 +0200 Subject: [PATCH 08/25] Split tracing into focused observability modules Keep tracing.py as the compatibility facade and move configuration, payloads, attributes, agent spans, tool spans, and installation into their own modules. Update test module references without changing assertions. Co-Authored-By: Claude Code --- .../observability/agent_spans.py | 217 +++++ .../observability/installation.py | 127 +++ .../observability/span_attributes.py | 189 ++++ .../observability/span_payloads.py | 94 ++ .../atomic_agents/observability/tool_spans.py | 194 +++++ .../atomic_agents/observability/tracing.py | 815 +----------------- .../observability/tracing_config.py | 39 + .../tests/observability/test_tracing.py | 30 +- 8 files changed, 921 insertions(+), 784 deletions(-) create mode 100644 atomic-agents/atomic_agents/observability/agent_spans.py create mode 100644 atomic-agents/atomic_agents/observability/installation.py create mode 100644 atomic-agents/atomic_agents/observability/span_attributes.py create mode 100644 atomic-agents/atomic_agents/observability/span_payloads.py create mode 100644 atomic-agents/atomic_agents/observability/tool_spans.py create mode 100644 atomic-agents/atomic_agents/observability/tracing_config.py diff --git a/atomic-agents/atomic_agents/observability/agent_spans.py b/atomic-agents/atomic_agents/observability/agent_spans.py new file mode 100644 index 00000000..82ffdd0f --- /dev/null +++ b/atomic-agents/atomic_agents/observability/agent_spans.py @@ -0,0 +1,217 @@ +"""Agent call and stream spans.""" + +import asyncio +import contextvars +import functools +from contextlib import contextmanager +from typing import Any, AsyncGenerator, Callable, Dict, Generator, Iterator + +from opentelemetry import context as context_api +from opentelemetry import trace as trace_api +from opentelemetry.trace import Span, Status, StatusCode + +from atomic_agents.agents.atomic_agent import AtomicAgent + +from . import installation +from .span_attributes import ATTR_STREAM_CLOSED_EARLY, _agent_display_name, _agent_start_attributes, _suppressed +from .span_payloads import _record_failure, _set_input_safely, _set_output_safely + +AGENT_SYNC_METHOD = "run" +AGENT_ASYNC_METHOD = "run_async" +AGENT_SYNC_STREAM_METHOD = "run_stream" +AGENT_ASYNC_STREAM_METHOD = "run_async_stream" + + +_active_agent_span: contextvars.ContextVar[Span | None] = contextvars.ContextVar( + "atomic_agents_active_agent_span", default=None +) + + +def _user_input_from_call(args: tuple, kwargs: dict) -> Any: + return args[0] if args else kwargs.get("user_input") + + +def _finish_agent_span(span: Any, output: Any) -> None: + _set_output_safely(span, output) + span.set_status(Status(StatusCode.OK)) + + +def _mark_closed_early(span: Any, output: Any) -> None: + span.set_attribute(ATTR_STREAM_CLOSED_EARLY, True) + _set_output_safely(span, output) + + +@contextmanager +def _agent_span(agent: AtomicAgent, args: tuple, kwargs: dict) -> Iterator[Span]: + installed = installation._current_installation() + with installed.tracer.start_as_current_span( + f"invoke_agent {_agent_display_name(agent)}", + attributes=_agent_start_attributes(agent), + record_exception=False, + set_status_on_exception=False, + ) as span: + _set_input_safely(span, _user_input_from_call(args, kwargs)) + token = _active_agent_span.set(span) + try: + yield span + except BaseException as error: + _record_failure(span, error) + raise + finally: + _active_agent_span.reset(token) + + +def _trace_agent_call(method_name: str, wrapped: Callable) -> Callable: + @functools.wraps(wrapped) + def traced(agent, *args, **kwargs): + if _suppressed(): + return wrapped(agent, *args, **kwargs) + with _agent_span(agent, args, kwargs) as span: + response = wrapped(agent, *args, **kwargs) + _finish_agent_span(span, response) + return response + + return traced + + +def _trace_agent_async_call(method_name: str, wrapped: Callable) -> Callable: + @functools.wraps(wrapped) + async def traced(agent, *args, **kwargs): + if _suppressed(): + return await wrapped(agent, *args, **kwargs) + with _agent_span(agent, args, kwargs) as span: + response = await wrapped(agent, *args, **kwargs) + _finish_agent_span(span, response) + return response + + return traced + + +def _start_agent_span(agent: AtomicAgent, method_name: str, args: tuple, kwargs: dict) -> Span: + if _suppressed(): + return trace_api.INVALID_SPAN + span = installation._current_installation().tracer.start_span( + f"invoke_agent {_agent_display_name(agent)}", + attributes=_agent_start_attributes(agent), + ) + _set_input_safely(span, _user_input_from_call(args, kwargs)) + return span + + +def _call_within_span(span: Any, function: Callable, *args: Any) -> Any: + token = context_api.attach(trace_api.set_span_in_context(span)) + agent_token = _active_agent_span.set(span) + try: + return function(*args) + finally: + _active_agent_span.reset(agent_token) + context_api.detach(token) + + +async def _await_within_span(span: Any, awaitable: Any) -> Any: + token = context_api.attach(trace_api.set_span_in_context(span)) + agent_token = _active_agent_span.set(span) + try: + return await awaitable + finally: + _active_agent_span.reset(agent_token) + context_api.detach(token) + + +class _SyncStreamCursor: + def __init__(self, span: Any, inner: Generator): + self.span = span + self.inner = inner + self.last: Any = None + self.final: Any = None + + def advance(self) -> bool: + try: + self.last = _call_within_span(self.span, next, self.inner) + except StopIteration as stop: + self.final = stop.value + return False + return True + + def close_early(self) -> None: + _call_within_span(self.span, self.inner.close) + _mark_closed_early(self.span, self.last) + self.span.set_status(Status(StatusCode.OK)) + + +class _AsyncStreamCursor: + """Async twin of :class:`_SyncStreamCursor`; async generators carry no return value, so the last partial is the output.""" + + def __init__(self, span: Any, inner: AsyncGenerator): + self.span = span + self.inner = inner + self.last: Any = None + + async def advance(self) -> bool: + try: + self.last = await _await_within_span(self.span, self.inner.__anext__()) + except StopAsyncIteration: + return False + return True + + async def close_early(self) -> None: + await _await_within_span(self.span, self.inner.aclose()) + _mark_closed_early(self.span, self.last) + self.span.set_status(Status(StatusCode.OK)) + + def mark_cancelled(self) -> None: + _mark_closed_early(self.span, self.last) + + +def _trace_agent_stream(method_name: str, wrapped: Callable) -> Callable: + @functools.wraps(wrapped) + def traced(agent, *args, **kwargs): + span = _start_agent_span(agent, method_name, args, kwargs) + cursor = _SyncStreamCursor(span, wrapped(agent, *args, **kwargs)) + try: + while cursor.advance(): + yield cursor.last + _finish_agent_span(span, cursor.final) + return cursor.final + except GeneratorExit: + cursor.close_early() + raise + except BaseException as error: + _record_failure(span, error) + raise + finally: + span.end() + + return traced + + +def _trace_agent_async_stream(method_name: str, wrapped: Callable) -> Callable: + @functools.wraps(wrapped) + async def traced(agent, *args, **kwargs): + span = _start_agent_span(agent, method_name, args, kwargs) + cursor = _AsyncStreamCursor(span, wrapped(agent, *args, **kwargs)) + try: + while await cursor.advance(): + yield cursor.last + _finish_agent_span(span, cursor.last) + except GeneratorExit: + await cursor.close_early() + raise + except asyncio.CancelledError: + cursor.mark_cancelled() + raise + except BaseException as error: + _record_failure(span, error) + raise + finally: + span.end() + + return traced + + +_AGENT_WRAPPERS: Dict[str, Callable[[str, Callable], Callable]] = { + AGENT_SYNC_METHOD: _trace_agent_call, + AGENT_ASYNC_METHOD: _trace_agent_async_call, + AGENT_SYNC_STREAM_METHOD: _trace_agent_stream, + AGENT_ASYNC_STREAM_METHOD: _trace_agent_async_stream, +} diff --git a/atomic-agents/atomic_agents/observability/installation.py b/atomic-agents/atomic_agents/observability/installation.py new file mode 100644 index 00000000..326c7acc --- /dev/null +++ b/atomic-agents/atomic_agents/observability/installation.py @@ -0,0 +1,127 @@ +"""Tracing installation and method restoration.""" + +import logging +from dataclasses import replace +from typing import TYPE_CHECKING, Callable, Optional, Type + +from opentelemetry import trace as trace_api +from opentelemetry.trace import Tracer +from opentelemetry.util.types import AttributeValue + +from atomic_agents import __version__ +from atomic_agents.agents.atomic_agent import AtomicAgent +from atomic_agents.base.base_tool import BaseTool + +from . import agent_spans, span_attributes, tool_spans +from .tracing_config import PayloadVisibility, TracingConfig + +if TYPE_CHECKING: + from opentelemetry.trace import TracerProvider + + +logger = logging.getLogger("atomic_agents.observability.tracing") + + +TRACED_MARKER = "__atomic_agents_traced__" +INHERITED_MARKER = "__atomic_agents_inherited__" +TRACER_NAME = "atomic_agents" + + +def _current_installation() -> "_Installation": + assert _installation is not None + return _installation + + +class _Installation: + def __init__( + self, + tracer: Tracer, + tracer_provider: Optional["TracerProvider"], + config: TracingConfig, + original_init_subclass: classmethod, + context_attributes: Callable[[], dict[str, AttributeValue]] | None, + ): + self.context_attributes = context_attributes + self.tracer = tracer + self.tracer_provider = tracer_provider + self.config = config + self.original_init_subclass = original_init_subclass + + +_installation: Optional[_Installation] = None + + +def is_installed() -> bool: + return _installation is not None + + +def install( + *, tracer_provider: Optional["TracerProvider"] = None, config: TracingConfig | PayloadVisibility | None = None +) -> None: + global _installation + config = TracingConfig.coerce(config) + if _installation is not None: + if config.openinference_attributes is None: + config = replace(config, openinference_attributes=_installation.config.openinference_attributes) + _replace_installation(_installation, tracer_provider, config) + return + context_attributes = span_attributes._import_context_attributes() + if config.openinference_attributes is None: + config = replace(config, openinference_attributes=context_attributes is not None) + tracer = trace_api.get_tracer(TRACER_NAME, __version__, tracer_provider) + original_init_subclass = BaseTool.__dict__["__init_subclass__"] + _installation = _Installation(tracer, tracer_provider, config, original_init_subclass, context_attributes) + _instrument_agent_class() + tool_spans._instrument_existing_tool_classes() + tool_spans._hook_future_tool_classes(original_init_subclass) + + +def uninstall() -> None: + global _installation + if _installation is None: + return + setattr(BaseTool, "__init_subclass__", _installation.original_init_subclass) + for tool_class in tool_spans._all_subclasses(BaseTool): + tool_spans._restore_tool_class(tool_class) + for method_name in agent_spans._AGENT_WRAPPERS: + _restore_method(AtomicAgent, method_name) + _installation = None + + +def _replace_installation( + installed: _Installation, tracer_provider: Optional["TracerProvider"], config: TracingConfig +) -> None: + if tracer_provider is installed.tracer_provider and config == installed.config: + return + installed.tracer = trace_api.get_tracer(TRACER_NAME, __version__, tracer_provider) + installed.tracer_provider = tracer_provider + installed.config = config + installed.context_attributes = span_attributes._import_context_attributes() + logger.warning( + "AtomicAgentsInstrumentor tracer provider/config replaced; " + "in-flight spans finish on the provider they started with." + ) + + +def _instrument_agent_class() -> None: + for method_name, wrap in agent_spans._AGENT_WRAPPERS.items(): + original = AtomicAgent.__dict__[method_name] + if not getattr(original, TRACED_MARKER, False): + setattr(AtomicAgent, method_name, _mark_traced(wrap(method_name, original), original, False)) + + +def _mark_traced(wrapper: Callable, original: Callable, inherited: bool) -> Callable: + setattr(wrapper, TRACED_MARKER, True) + setattr(wrapper, INHERITED_MARKER, inherited) + setattr(wrapper, "__wrapped__", original) + return wrapper + + +def _restore_method(owner: Type, method_name: str) -> None: + current = owner.__dict__.get(method_name) + if current is None or not getattr(current, TRACED_MARKER, False): + return + if getattr(current, INHERITED_MARKER, False): + delattr(owner, method_name) + else: + setattr(owner, method_name, current.__wrapped__) diff --git a/atomic-agents/atomic_agents/observability/span_attributes.py b/atomic-agents/atomic_agents/observability/span_attributes.py new file mode 100644 index 00000000..6a2e73b1 --- /dev/null +++ b/atomic-agents/atomic_agents/observability/span_attributes.py @@ -0,0 +1,189 @@ +"""Span attributes and conversation context.""" + +import contextvars +import hashlib +import json +import logging +from collections.abc import Mapping +from contextlib import contextmanager +from typing import Callable, Dict, Iterator, List + +from opentelemetry import context as context_api +from opentelemetry.util.types import AttributeValue + +from atomic_agents.agents.atomic_agent import AtomicAgent + +from . import installation + + +logger = logging.getLogger("atomic_agents.observability.tracing") + + +ATTR_MODEL = "atomic_agents.model" +ATTR_MODE = "atomic_agents.mode" +ATTR_INPUT_SCHEMA = "atomic_agents.input_schema" +ATTR_OUTPUT_SCHEMA = "atomic_agents.output_schema" +ATTR_HISTORY_MESSAGE_COUNT = "atomic_agents.history.message_count" +ATTR_CONTEXT_PROVIDERS = "atomic_agents.context_providers" +ATTR_SYSTEM_PROMPT_SHA256 = "atomic_agents.system_prompt.sha256" +ATTR_STREAM_CLOSED_EARLY = "atomic_agents.stream.closed_early" +ATTR_OPERATION = "gen_ai.operation.name" +ATTR_AGENT_NAME = "gen_ai.agent.name" +ATTR_REQUEST_MODEL = "gen_ai.request.model" +ATTR_PROVIDER_NAME = "gen_ai.provider.name" +ATTR_CONVERSATION_ID = "gen_ai.conversation.id" +ATTR_TOOL_NAME = "gen_ai.tool.name" +ATTR_TOOL_DESCRIPTION = "gen_ai.tool.description" +ATTR_TOOL_TYPE = "gen_ai.tool.type" +ATTR_TOOL_ARGUMENTS = "gen_ai.tool.call.arguments" +ATTR_TOOL_RESULT = "gen_ai.tool.call.result" +ATTR_SPAN_KIND = "openinference.span.kind" +ATTR_INPUT_VALUE = "input.value" +ATTR_INPUT_MIME_TYPE = "input.mime_type" +ATTR_OUTPUT_VALUE = "output.value" +ATTR_OUTPUT_MIME_TYPE = "output.mime_type" +ATTR_OI_AGENT_NAME = "agent.name" +ATTR_OI_TOOL_NAME = "tool.name" +ATTR_OI_TOOL_DESCRIPTION = "tool.description" +ATTR_TOOL_PARAMETERS = "tool.parameters" +ATTR_SESSION_ID = "session.id" +ATTR_INPUT = "atomic_agents.input" +ATTR_OUTPUT = "atomic_agents.output" +ATTR_AGENT_CLASS = "atomic_agents.agent.class" +ATTR_ERROR_TYPE = "error.type" +ATTR_TOOL_CLASS = "atomic_agents.tool.class" +PROVIDER_NAMES = { + "azure_openai": "azure.ai.openai", + "vertexai": "gcp.vertex_ai", + "gemini": "gcp.gemini", + "genai": "gcp.gemini", + "bedrock": "aws.bedrock", + "mistral": "mistral_ai", + "xai": "x_ai", +} + + +def _import_context_attributes() -> Callable[[], dict[str, AttributeValue]] | None: + try: + from openinference.instrumentation import get_attributes_from_context + except ImportError: + return None + from atomic_agents.observability import _openinference_is_too_old + + if _openinference_is_too_old(): + logger.warning("OpenInference context attributes disabled: 0.1.38 or newer is required.") + return None + return lambda: dict(get_attributes_from_context()) + + +def _context_attributes() -> dict[str, AttributeValue]: + reader = installation._current_installation().context_attributes + return reader() if reader is not None else {} + + +def _suppressed() -> bool: + return bool(context_api.get_value("suppress_instrumentation")) + + +_conversation_id: contextvars.ContextVar[str | None] = contextvars.ContextVar("atomic_agents_conversation_id", default=None) + + +@contextmanager +def using_conversation(conversation_id: str) -> Iterator[None]: + token = _conversation_id.set(conversation_id) + try: + yield + finally: + _conversation_id.reset(token) + + +def _agent_display_name(agent: AtomicAgent) -> str: + return agent.name or type(agent).__name__ + + +def _agent_class_name(agent: AtomicAgent) -> str: + return f"{type(agent).__name__}[{agent.input_schema.__name__}, {agent.output_schema.__name__}]" + + +def _best_effort_attributes( + readers: dict[str, Callable[[], AttributeValue | None]], +) -> dict[str, AttributeValue]: + attributes: dict[str, AttributeValue] = {} + for key, read in readers.items(): + try: + value = read() + if value is not None: + attributes[key] = value + except Exception as error: + logger.debug("Could not collect tracing attribute %s: %s", key, error) + return attributes + + +def _without_none(attributes: Mapping[str, AttributeValue | None]) -> dict[str, AttributeValue]: + return {key: value for key, value in attributes.items() if value is not None} + + +def _agent_start_attributes(agent: AtomicAgent) -> dict[str, AttributeValue]: + attributes: dict[str, AttributeValue] = { + ATTR_OPERATION: "invoke_agent", + ATTR_AGENT_NAME: _agent_display_name(agent), + ATTR_AGENT_CLASS: _agent_class_name(agent), + } + attributes.update( + _best_effort_attributes( + { + ATTR_REQUEST_MODEL: lambda: agent.model, + ATTR_MODEL: lambda: agent.model, + ATTR_MODE: lambda: getattr(agent.mode, "value", None), + ATTR_INPUT_SCHEMA: lambda: agent.input_schema.__name__, + ATTR_OUTPUT_SCHEMA: lambda: agent.output_schema.__name__, + ATTR_HISTORY_MESSAGE_COUNT: lambda: len(agent.history.history), + ATTR_CONTEXT_PROVIDERS: lambda: _context_provider_names(agent), + ATTR_SYSTEM_PROMPT_SHA256: lambda: _system_prompt_version(agent).get(ATTR_SYSTEM_PROMPT_SHA256), + ATTR_PROVIDER_NAME: lambda: _provider_attributes(agent).get(ATTR_PROVIDER_NAME), + } + ) + ) + attributes.update(_optional_attributes("AGENT", {ATTR_OI_AGENT_NAME: _agent_display_name(agent)})) + return _without_none(attributes) + + +def _context_provider_names(agent: AtomicAgent) -> List[str]: + providers = getattr(agent.system_prompt_generator, "context_providers", None) or {} + return sorted(str(name) for name in providers) + + +def _system_prompt_version(agent: AtomicAgent) -> Dict[str, str]: + """Hashes only the static prompt sections, so the value identifies the prompt version across turns + even though context providers change the rendered prompt on every call.""" + generator = agent.system_prompt_generator + sections = {name: getattr(generator, name, None) for name in ("background", "steps", "output_instructions")} + if agent.system_role is None or all(value is None for value in sections.values()): + return {} + digest = hashlib.sha256(json.dumps(sections, sort_keys=True, default=str).encode("utf-8")).hexdigest() + return {ATTR_SYSTEM_PROMPT_SHA256: digest} + + +def _provider_attributes(agent: AtomicAgent) -> dict[str, str]: + provider = getattr(getattr(agent.client, "provider", None), "value", None) + return {ATTR_PROVIDER_NAME: PROVIDER_NAMES.get(provider, provider)} if isinstance(provider, str) else {} + + +def _optional_attributes(kind: str, attributes: dict[str, AttributeValue]) -> dict[str, AttributeValue]: + context = _context_attributes() + conversation = _conversation_attributes(context) + if not installation._current_installation().config.openinference_attributes: + return conversation + return _without_none({ATTR_SPAN_KIND: kind, **attributes, **context, **conversation}) + + +def _conversation_attributes(context: dict[str, AttributeValue]) -> dict[str, AttributeValue]: + conversation_id: AttributeValue | None = _conversation_id.get() + if conversation_id is None: + conversation_id = context.get(ATTR_SESSION_ID) + if conversation_id is None: + return {} + attributes = {ATTR_CONVERSATION_ID: conversation_id} + if installation._current_installation().config.openinference_attributes: + attributes[ATTR_SESSION_ID] = conversation_id + return attributes diff --git a/atomic-agents/atomic_agents/observability/span_payloads.py b/atomic-agents/atomic_agents/observability/span_payloads.py new file mode 100644 index 00000000..57d41001 --- /dev/null +++ b/atomic-agents/atomic_agents/observability/span_payloads.py @@ -0,0 +1,94 @@ +"""Payload serialization, redaction, and bounded failures.""" + +import json +import logging + +from opentelemetry.trace import Span, Status, StatusCode +from pydantic import BaseModel, ValidationError + +from . import installation +from .tracing_config import TracingConfig +from .span_attributes import ( + ATTR_INPUT_VALUE, + ATTR_INPUT_MIME_TYPE, + ATTR_OUTPUT_VALUE, + ATTR_OUTPUT_MIME_TYPE, + ATTR_INPUT, + ATTR_OUTPUT, + ATTR_ERROR_TYPE, +) + + +logger = logging.getLogger("atomic_agents.observability.tracing") + + +TRUNCATION_MARKER = "...[truncated]" + + +def _serialize_payload(value: object) -> str | None: + try: + return value.model_dump_json() if isinstance(value, BaseModel) else json.dumps(value, default=str) + except Exception as error: + logger.warning("Could not serialize tracing payload: %s", error) + return None + + +def _record_payload( + span: Span, key_pairs: tuple[tuple[str, str | None], ...], value: object, hide: bool, config: TracingConfig +) -> None: + if value is None: + return + serialized = "__REDACTED__" if hide else _serialize_payload(value) + if serialized is None: + return + for key, mime_key in key_pairs: + _write_payload(span, key, mime_key, serialized, hide, config) + + +def _write_payload(span: Span, key: str, mime_key: str | None, serialized: str, hide: bool, config: TracingConfig) -> None: + if hide: + span.set_attribute(key, "__REDACTED__") + return + if len(serialized) > config.max_attribute_length: + span.set_attribute(key + ".original_length", len(serialized)) + serialized = _bounded_text(serialized, config) + span.set_attribute(key, serialized) + if mime_key is not None: + span.set_attribute(mime_key, "application/json") + + +def _payload_keys(primary: str, output: bool) -> tuple[tuple[str, str | None], ...]: + keys: tuple[tuple[str, str | None], ...] = ((primary, None),) + if installation._current_installation().config.openinference_attributes: + keys += ((ATTR_OUTPUT_VALUE, ATTR_OUTPUT_MIME_TYPE),) if output else ((ATTR_INPUT_VALUE, ATTR_INPUT_MIME_TYPE),) + return keys + + +def _set_input_safely(span: Span, value: object, primary: str = ATTR_INPUT) -> None: + config = installation._current_installation().config + _record_payload(span, _payload_keys(primary, False), value, config.hide_inputs, config) + + +def _set_output_safely(span: Span, value: object, primary: str = ATTR_OUTPUT) -> None: + config = installation._current_installation().config + _record_payload(span, _payload_keys(primary, True), value, config.hide_outputs, config) + + +def _bounded_text(value: str, config: TracingConfig) -> str: + if len(value) > config.max_attribute_length: + return value[: config.max_attribute_length] + TRUNCATION_MARKER + return value + + +def _failure_message(error: BaseException, config: TracingConfig) -> str: + if config.hide_outputs and isinstance(error, ValidationError): + return f"ValidationError: {error.error_count()} errors (__REDACTED__)" + return str(error) + + +def _record_failure(span: Span, error: BaseException) -> None: + config = installation._current_installation().config + message = _bounded_text(_failure_message(error, config), config) + span.set_attribute(ATTR_ERROR_TYPE, type(error).__qualname__) + span.record_exception(error, attributes={"exception.message": message, "exception.stacktrace": message}) + span.set_status(Status(StatusCode.ERROR, message)) diff --git a/atomic-agents/atomic_agents/observability/tool_spans.py b/atomic-agents/atomic_agents/observability/tool_spans.py new file mode 100644 index 00000000..f10c2f36 --- /dev/null +++ b/atomic-agents/atomic_agents/observability/tool_spans.py @@ -0,0 +1,194 @@ +"""Tool spans and tool class instrumentation.""" + +import contextvars +import functools +import inspect +import json +import logging +from contextlib import contextmanager +from types import FunctionType +from typing import Any, Callable, Iterator, List, Optional, Type + +from opentelemetry.trace import Span, Status, StatusCode +from opentelemetry.util.types import AttributeValue + +from atomic_agents.base.base_tool import BaseTool + +from . import installation +from .span_attributes import _best_effort_attributes, _optional_attributes, _suppressed, _without_none +from .span_payloads import _bounded_text, _record_failure, _set_input_safely, _set_output_safely +from .span_attributes import ( + ATTR_OPERATION, + ATTR_TOOL_NAME, + ATTR_TOOL_DESCRIPTION, + ATTR_TOOL_TYPE, + ATTR_TOOL_ARGUMENTS, + ATTR_TOOL_RESULT, + ATTR_OI_TOOL_NAME, + ATTR_OI_TOOL_DESCRIPTION, + ATTR_TOOL_PARAMETERS, + ATTR_TOOL_CLASS, +) + + +logger = logging.getLogger("atomic_agents.observability.tracing") + + +TOOL_METHOD_NAMES = ("run", "run_async", "arun") + + +_active_tool: contextvars.ContextVar[Optional[BaseTool]] = contextvars.ContextVar("atomic_agents_active_tool", default=None) + + +def _all_subclasses(base: Type) -> List[Type]: + found: List[Type] = [] + pending = list(base.__subclasses__()) + while pending: + current = pending.pop(0) + if current not in found: + found.append(current) + pending.extend(current.__subclasses__()) + return found + + +def _instrument_existing_tool_classes() -> None: + for tool_class in _all_subclasses(BaseTool): + _instrument_tool_class(tool_class) + + +def _hook_future_tool_classes(original_init_subclass: classmethod) -> None: + def traced_init_subclass(cls, **kwargs): + original_init_subclass.__func__(cls, **kwargs) + _instrument_tool_class(cls) + + setattr(BaseTool, "__init_subclass__", classmethod(traced_init_subclass)) + + +def _instrument_tool_class(tool_class: Type) -> None: + for method_name in TOOL_METHOD_NAMES: + method = inspect.getattr_static(tool_class, method_name, None) + if _is_traceable_function(method): + inherited = method_name not in tool_class.__dict__ + setattr(tool_class, method_name, installation._mark_traced(_trace_tool_method(method), method, inherited)) + + +def _is_traceable_function(method: Any) -> bool: + if not isinstance(method, FunctionType) or inspect.isasyncgenfunction(method): + return False + if getattr(method, "__isabstractmethod__", False): + return False + return not getattr(method, installation.TRACED_MARKER, False) + + +def _restore_tool_class(tool_class: Type) -> None: + for method_name in TOOL_METHOD_NAMES: + installation._restore_method(tool_class, method_name) + + +def _tool_parameter_attributes(tool: BaseTool) -> dict[str, AttributeValue]: + parameters = _best_effort_attributes({ATTR_TOOL_PARAMETERS: lambda: json.dumps(tool.input_schema.model_json_schema())}) + value = parameters.get(ATTR_TOOL_PARAMETERS) + if isinstance(value, str): + config = installation._current_installation().config + parameters[ATTR_TOOL_PARAMETERS] = _bounded_text(value, config) + if len(value) > config.max_attribute_length: + parameters[ATTR_TOOL_PARAMETERS + ".original_length"] = len(value) + return parameters + + +def _tool_display_name(tool: BaseTool) -> str: + try: + return tool.tool_name + except Exception as error: + logger.debug("Could not collect tracing tool name: %s", error) + return type(tool).__name__ + + +def _tool_start_attributes(tool: BaseTool) -> dict[str, AttributeValue]: + attributes: dict[str, AttributeValue] = { + ATTR_OPERATION: "execute_tool", + ATTR_TOOL_NAME: _tool_display_name(tool), + ATTR_TOOL_CLASS: type(tool).__name__, + ATTR_TOOL_TYPE: "function", + **_optional_attributes("TOOL", {}), + } + attributes.update(_best_effort_attributes({ATTR_TOOL_DESCRIPTION: lambda: tool.tool_description})) + if installation._current_installation().config.openinference_attributes: + attributes.update( + _best_effort_attributes( + { + ATTR_OI_TOOL_NAME: lambda: attributes[ATTR_TOOL_NAME], + ATTR_OI_TOOL_DESCRIPTION: lambda: tool.tool_description, + } + ) + ) + attributes.update(_tool_parameter_attributes(tool)) + return _without_none(attributes) + + +@contextmanager +def _tool_span(tool: BaseTool, args: tuple, kwargs: dict) -> Iterator[Span]: + attributes = _tool_start_attributes(tool) + with installation._current_installation().tracer.start_as_current_span( + f"execute_tool {attributes[ATTR_TOOL_NAME]}", + attributes=attributes, + record_exception=False, + set_status_on_exception=False, + ) as span: + _set_input_safely(span, _tool_input_from_call(args, kwargs), ATTR_TOOL_ARGUMENTS) + try: + yield span + except BaseException as error: + _record_failure(span, error) + raise + + +def _tool_input_from_call(args: tuple, kwargs: dict) -> Any: + return args[0] if args else kwargs.get("params") + + +def _trace_tool_method(wrapped: Callable) -> Callable: + if inspect.iscoroutinefunction(wrapped): + return _trace_tool_async_method(wrapped) + + @functools.wraps(wrapped) + def traced(tool, *args, **kwargs): + if _active_tool.get() is tool or _suppressed(): + return wrapped(tool, *args, **kwargs) + with _tool_span(tool, args, kwargs) as span: + result = _run_as_active_tool(tool, wrapped, args, kwargs) + _set_output_safely(span, result, ATTR_TOOL_RESULT) + span.set_status(Status(StatusCode.OK)) + return result + + return traced + + +def _trace_tool_async_method(wrapped: Callable) -> Callable: + @functools.wraps(wrapped) + async def traced(tool, *args, **kwargs): + if _active_tool.get() is tool or _suppressed(): + return await wrapped(tool, *args, **kwargs) + with _tool_span(tool, args, kwargs) as span: + result = await _await_as_active_tool(tool, wrapped, args, kwargs) + _set_output_safely(span, result, ATTR_TOOL_RESULT) + span.set_status(Status(StatusCode.OK)) + return result + + return traced + + +def _run_as_active_tool(tool: BaseTool, wrapped: Callable, args: tuple, kwargs: dict) -> Any: + token = _active_tool.set(tool) + try: + return wrapped(tool, *args, **kwargs) + finally: + _active_tool.reset(token) + + +async def _await_as_active_tool(tool: BaseTool, wrapped: Callable, args: tuple, kwargs: dict) -> Any: + token = _active_tool.set(tool) + try: + return await wrapped(tool, *args, **kwargs) + finally: + _active_tool.reset(token) diff --git a/atomic-agents/atomic_agents/observability/tracing.py b/atomic-agents/atomic_agents/observability/tracing.py index 5d0bb8dc..f1c54e9a 100644 --- a/atomic-agents/atomic_agents/observability/tracing.py +++ b/atomic-agents/atomic_agents/observability/tracing.py @@ -1,769 +1,46 @@ -""" -OpenTelemetry span emission for ``AtomicAgent`` runs and ``BaseTool`` runs. - -Loaded lazily by :class:`atomic_agents.observability.AtomicAgentsInstrumentor`; importing this module -requires the ``observability`` extra. -""" - -import asyncio -import contextvars -import functools -import hashlib -import inspect -import json -import logging -from collections.abc import Mapping -from contextlib import contextmanager -from dataclasses import dataclass, replace -import os -from types import FunctionType -from typing import TYPE_CHECKING, Any, AsyncGenerator, Callable, Dict, Generator, List, Optional, Type, Protocol, Iterator - -from opentelemetry import context as context_api -from opentelemetry import trace as trace_api -from opentelemetry.trace import Span, Status, StatusCode, Tracer -from opentelemetry.util.types import AttributeValue -from pydantic import BaseModel, ValidationError - -from atomic_agents import __version__ -from atomic_agents.agents.atomic_agent import AtomicAgent -from atomic_agents.base.base_tool import BaseTool - -if TYPE_CHECKING: - from opentelemetry.trace import TracerProvider - -logger = logging.getLogger(__name__) - -TRACED_MARKER = "__atomic_agents_traced__" -INHERITED_MARKER = "__atomic_agents_inherited__" -TRACER_NAME = "atomic_agents" - -AGENT_SYNC_METHOD = "run" -AGENT_ASYNC_METHOD = "run_async" -AGENT_SYNC_STREAM_METHOD = "run_stream" -AGENT_ASYNC_STREAM_METHOD = "run_async_stream" -TOOL_METHOD_NAMES = ("run", "run_async", "arun") - -ATTR_MODEL = "atomic_agents.model" -ATTR_MODE = "atomic_agents.mode" -ATTR_INPUT_SCHEMA = "atomic_agents.input_schema" -ATTR_OUTPUT_SCHEMA = "atomic_agents.output_schema" -ATTR_HISTORY_MESSAGE_COUNT = "atomic_agents.history.message_count" -ATTR_CONTEXT_PROVIDERS = "atomic_agents.context_providers" -ATTR_SYSTEM_PROMPT_SHA256 = "atomic_agents.system_prompt.sha256" -ATTR_STREAM_CLOSED_EARLY = "atomic_agents.stream.closed_early" - -ATTR_OPERATION = "gen_ai.operation.name" -ATTR_AGENT_NAME = "gen_ai.agent.name" -ATTR_REQUEST_MODEL = "gen_ai.request.model" -ATTR_PROVIDER_NAME = "gen_ai.provider.name" -ATTR_CONVERSATION_ID = "gen_ai.conversation.id" -ATTR_TOOL_NAME = "gen_ai.tool.name" -ATTR_TOOL_DESCRIPTION = "gen_ai.tool.description" -ATTR_TOOL_TYPE = "gen_ai.tool.type" -ATTR_TOOL_ARGUMENTS = "gen_ai.tool.call.arguments" -ATTR_TOOL_RESULT = "gen_ai.tool.call.result" -ATTR_SPAN_KIND = "openinference.span.kind" -ATTR_INPUT_VALUE = "input.value" -ATTR_INPUT_MIME_TYPE = "input.mime_type" -ATTR_OUTPUT_VALUE = "output.value" -ATTR_OUTPUT_MIME_TYPE = "output.mime_type" -ATTR_OI_AGENT_NAME = "agent.name" -ATTR_OI_TOOL_NAME = "tool.name" -ATTR_OI_TOOL_DESCRIPTION = "tool.description" -ATTR_TOOL_PARAMETERS = "tool.parameters" -ATTR_SESSION_ID = "session.id" -ATTR_INPUT = "atomic_agents.input" -ATTR_OUTPUT = "atomic_agents.output" -ATTR_AGENT_CLASS = "atomic_agents.agent.class" -ATTR_ERROR_TYPE = "error.type" -ATTR_TOOL_CLASS = "atomic_agents.tool.class" -TRUNCATION_MARKER = "...[truncated]" -PROVIDER_NAMES = { - "azure_openai": "azure.ai.openai", - "vertexai": "gcp.vertex_ai", - "gemini": "gcp.gemini", - "genai": "gcp.gemini", - "bedrock": "aws.bedrock", - "mistral": "mistral_ai", - "xai": "x_ai", -} - - -class PayloadVisibility(Protocol): - hide_inputs: bool - hide_outputs: bool - - -@dataclass(frozen=True) -class TracingConfig: - hide_inputs: bool = False - hide_outputs: bool = False - max_attribute_length: int = 4096 - openinference_attributes: bool | None = None - llm_spans: bool = True - - @classmethod - def from_environment(cls) -> "TracingConfig": - return cls( - hide_inputs=_visibility_environment("INPUTS").lower() in ("true", "1"), - hide_outputs=_visibility_environment("OUTPUTS").lower() in ("true", "1"), - max_attribute_length=int(os.environ.get("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", "4096")), - ) - - @classmethod - def coerce(cls, config: "TracingConfig | PayloadVisibility | None") -> "TracingConfig": - if config is None: - return cls.from_environment() - if isinstance(config, cls): - return config - return cls(hide_inputs=config.hide_inputs, hide_outputs=config.hide_outputs) - - -def _visibility_environment(direction: str) -> str: - return os.environ.get(f"ATOMIC_AGENTS_TRACE_HIDE_{direction}", os.environ.get(f"OPENINFERENCE_HIDE_{direction}", "")) - - -def _import_context_attributes() -> Callable[[], dict[str, AttributeValue]] | None: - try: - from openinference.instrumentation import get_attributes_from_context - except ImportError: - return None - from atomic_agents.observability import _openinference_is_too_old - - if _openinference_is_too_old(): - logger.warning("OpenInference context attributes disabled: 0.1.38 or newer is required.") - return None - return lambda: dict(get_attributes_from_context()) - - -def _context_attributes() -> dict[str, AttributeValue]: - reader = _current_installation().context_attributes - return reader() if reader is not None else {} - - -def _current_installation() -> "_Installation": - assert _installation is not None - return _installation - - -def _suppressed() -> bool: - return bool(context_api.get_value("suppress_instrumentation")) - - -_conversation_id: contextvars.ContextVar[str | None] = contextvars.ContextVar("atomic_agents_conversation_id", default=None) -_active_agent_span: contextvars.ContextVar[Span | None] = contextvars.ContextVar( - "atomic_agents_active_agent_span", default=None -) - - -@contextmanager -def using_conversation(conversation_id: str) -> Iterator[None]: - token = _conversation_id.set(conversation_id) - try: - yield - finally: - _conversation_id.reset(token) - - -_active_tool: contextvars.ContextVar[Optional[BaseTool]] = contextvars.ContextVar("atomic_agents_active_tool", default=None) - - -class _Installation: - def __init__( - self, - tracer: Tracer, - tracer_provider: Optional["TracerProvider"], - config: TracingConfig, - original_init_subclass: classmethod, - context_attributes: Callable[[], dict[str, AttributeValue]] | None, - ): - self.context_attributes = context_attributes - self.tracer = tracer - self.tracer_provider = tracer_provider - self.config = config - self.original_init_subclass = original_init_subclass - - -_installation: Optional[_Installation] = None - - -def is_installed() -> bool: - return _installation is not None - - -def install( - *, tracer_provider: Optional["TracerProvider"] = None, config: TracingConfig | PayloadVisibility | None = None -) -> None: - global _installation - config = TracingConfig.coerce(config) - if _installation is not None: - if config.openinference_attributes is None: - config = replace(config, openinference_attributes=_installation.config.openinference_attributes) - _replace_installation(_installation, tracer_provider, config) - return - context_attributes = _import_context_attributes() - if config.openinference_attributes is None: - config = replace(config, openinference_attributes=context_attributes is not None) - tracer = trace_api.get_tracer(TRACER_NAME, __version__, tracer_provider) - original_init_subclass = BaseTool.__dict__["__init_subclass__"] - _installation = _Installation(tracer, tracer_provider, config, original_init_subclass, context_attributes) - _instrument_agent_class() - _instrument_existing_tool_classes() - _hook_future_tool_classes(original_init_subclass) - - -def uninstall() -> None: - global _installation - if _installation is None: - return - setattr(BaseTool, "__init_subclass__", _installation.original_init_subclass) - for tool_class in _all_subclasses(BaseTool): - _restore_tool_class(tool_class) - for method_name in _AGENT_WRAPPERS: - _restore_method(AtomicAgent, method_name) - _installation = None - - -def _replace_installation( - installed: _Installation, tracer_provider: Optional["TracerProvider"], config: TracingConfig -) -> None: - if tracer_provider is installed.tracer_provider and config == installed.config: - return - installed.tracer = trace_api.get_tracer(TRACER_NAME, __version__, tracer_provider) - installed.tracer_provider = tracer_provider - installed.config = config - installed.context_attributes = _import_context_attributes() - logger.warning( - "AtomicAgentsInstrumentor tracer provider/config replaced; " - "in-flight spans finish on the provider they started with." - ) - - -def _instrument_agent_class() -> None: - for method_name, wrap in _AGENT_WRAPPERS.items(): - original = AtomicAgent.__dict__[method_name] - if not getattr(original, TRACED_MARKER, False): - setattr(AtomicAgent, method_name, _mark_traced(wrap(method_name, original), original, False)) - - -def _mark_traced(wrapper: Callable, original: Callable, inherited: bool) -> Callable: - setattr(wrapper, TRACED_MARKER, True) - setattr(wrapper, INHERITED_MARKER, inherited) - setattr(wrapper, "__wrapped__", original) - return wrapper - - -def _restore_method(owner: Type, method_name: str) -> None: - current = owner.__dict__.get(method_name) - if current is None or not getattr(current, TRACED_MARKER, False): - return - if getattr(current, INHERITED_MARKER, False): - delattr(owner, method_name) - else: - setattr(owner, method_name, current.__wrapped__) - - -def _all_subclasses(base: Type) -> List[Type]: - found: List[Type] = [] - pending = list(base.__subclasses__()) - while pending: - current = pending.pop(0) - if current not in found: - found.append(current) - pending.extend(current.__subclasses__()) - return found - - -def _instrument_existing_tool_classes() -> None: - for tool_class in _all_subclasses(BaseTool): - _instrument_tool_class(tool_class) - - -def _hook_future_tool_classes(original_init_subclass: classmethod) -> None: - def traced_init_subclass(cls, **kwargs): - original_init_subclass.__func__(cls, **kwargs) - _instrument_tool_class(cls) - - setattr(BaseTool, "__init_subclass__", classmethod(traced_init_subclass)) - - -def _instrument_tool_class(tool_class: Type) -> None: - for method_name in TOOL_METHOD_NAMES: - method = inspect.getattr_static(tool_class, method_name, None) - if _is_traceable_function(method): - inherited = method_name not in tool_class.__dict__ - setattr(tool_class, method_name, _mark_traced(_trace_tool_method(method), method, inherited)) - - -def _is_traceable_function(method: Any) -> bool: - if not isinstance(method, FunctionType) or inspect.isasyncgenfunction(method): - return False - if getattr(method, "__isabstractmethod__", False): - return False - return not getattr(method, TRACED_MARKER, False) - - -def _restore_tool_class(tool_class: Type) -> None: - for method_name in TOOL_METHOD_NAMES: - _restore_method(tool_class, method_name) - - -def _agent_display_name(agent: AtomicAgent) -> str: - return agent.name or type(agent).__name__ - - -def _agent_class_name(agent: AtomicAgent) -> str: - return f"{type(agent).__name__}[{agent.input_schema.__name__}, {agent.output_schema.__name__}]" - - -def _best_effort_attributes( - readers: dict[str, Callable[[], AttributeValue | None]], -) -> dict[str, AttributeValue]: - attributes: dict[str, AttributeValue] = {} - for key, read in readers.items(): - try: - value = read() - if value is not None: - attributes[key] = value - except Exception as error: - logger.debug("Could not collect tracing attribute %s: %s", key, error) - return attributes - - -def _without_none(attributes: Mapping[str, AttributeValue | None]) -> dict[str, AttributeValue]: - return {key: value for key, value in attributes.items() if value is not None} - - -def _agent_start_attributes(agent: AtomicAgent) -> dict[str, AttributeValue]: - attributes: dict[str, AttributeValue] = { - ATTR_OPERATION: "invoke_agent", - ATTR_AGENT_NAME: _agent_display_name(agent), - ATTR_AGENT_CLASS: _agent_class_name(agent), - } - attributes.update( - _best_effort_attributes( - { - ATTR_REQUEST_MODEL: lambda: agent.model, - ATTR_MODEL: lambda: agent.model, - ATTR_MODE: lambda: getattr(agent.mode, "value", None), - ATTR_INPUT_SCHEMA: lambda: agent.input_schema.__name__, - ATTR_OUTPUT_SCHEMA: lambda: agent.output_schema.__name__, - ATTR_HISTORY_MESSAGE_COUNT: lambda: len(agent.history.history), - ATTR_CONTEXT_PROVIDERS: lambda: _context_provider_names(agent), - ATTR_SYSTEM_PROMPT_SHA256: lambda: _system_prompt_version(agent).get(ATTR_SYSTEM_PROMPT_SHA256), - ATTR_PROVIDER_NAME: lambda: _provider_attributes(agent).get(ATTR_PROVIDER_NAME), - } - ) - ) - attributes.update(_optional_attributes("AGENT", {ATTR_OI_AGENT_NAME: _agent_display_name(agent)})) - return _without_none(attributes) - - -def _context_provider_names(agent: AtomicAgent) -> List[str]: - providers = getattr(agent.system_prompt_generator, "context_providers", None) or {} - return sorted(str(name) for name in providers) - - -def _system_prompt_version(agent: AtomicAgent) -> Dict[str, str]: - """Hashes only the static prompt sections, so the value identifies the prompt version across turns - even though context providers change the rendered prompt on every call.""" - generator = agent.system_prompt_generator - sections = {name: getattr(generator, name, None) for name in ("background", "steps", "output_instructions")} - if agent.system_role is None or all(value is None for value in sections.values()): - return {} - digest = hashlib.sha256(json.dumps(sections, sort_keys=True, default=str).encode("utf-8")).hexdigest() - return {ATTR_SYSTEM_PROMPT_SHA256: digest} - - -def _user_input_from_call(args: tuple, kwargs: dict) -> Any: - return args[0] if args else kwargs.get("user_input") - - -def _provider_attributes(agent: AtomicAgent) -> dict[str, str]: - provider = getattr(getattr(agent.client, "provider", None), "value", None) - return {ATTR_PROVIDER_NAME: PROVIDER_NAMES.get(provider, provider)} if isinstance(provider, str) else {} - - -def _optional_attributes(kind: str, attributes: dict[str, AttributeValue]) -> dict[str, AttributeValue]: - context = _context_attributes() - conversation = _conversation_attributes(context) - if not _current_installation().config.openinference_attributes: - return conversation - return _without_none({ATTR_SPAN_KIND: kind, **attributes, **context, **conversation}) - - -def _conversation_attributes(context: dict[str, AttributeValue]) -> dict[str, AttributeValue]: - conversation_id: AttributeValue | None = _conversation_id.get() - if conversation_id is None: - conversation_id = context.get(ATTR_SESSION_ID) - if conversation_id is None: - return {} - attributes = {ATTR_CONVERSATION_ID: conversation_id} - if _current_installation().config.openinference_attributes: - attributes[ATTR_SESSION_ID] = conversation_id - return attributes - - -def _serialize_payload(value: object) -> str | None: - try: - return value.model_dump_json() if isinstance(value, BaseModel) else json.dumps(value, default=str) - except Exception as error: - logger.warning("Could not serialize tracing payload: %s", error) - return None - - -def _record_payload( - span: Span, key_pairs: tuple[tuple[str, str | None], ...], value: object, hide: bool, config: TracingConfig -) -> None: - if value is None: - return - serialized = "__REDACTED__" if hide else _serialize_payload(value) - if serialized is None: - return - for key, mime_key in key_pairs: - _write_payload(span, key, mime_key, serialized, hide, config) - - -def _write_payload(span: Span, key: str, mime_key: str | None, serialized: str, hide: bool, config: TracingConfig) -> None: - if hide: - span.set_attribute(key, "__REDACTED__") - return - if len(serialized) > config.max_attribute_length: - span.set_attribute(key + ".original_length", len(serialized)) - serialized = _bounded_text(serialized, config) - span.set_attribute(key, serialized) - if mime_key is not None: - span.set_attribute(mime_key, "application/json") - - -def _payload_keys(primary: str, output: bool) -> tuple[tuple[str, str | None], ...]: - keys: tuple[tuple[str, str | None], ...] = ((primary, None),) - if _current_installation().config.openinference_attributes: - keys += ((ATTR_OUTPUT_VALUE, ATTR_OUTPUT_MIME_TYPE),) if output else ((ATTR_INPUT_VALUE, ATTR_INPUT_MIME_TYPE),) - return keys - - -def _set_input_safely(span: Span, value: object, primary: str = ATTR_INPUT) -> None: - config = _current_installation().config - _record_payload(span, _payload_keys(primary, False), value, config.hide_inputs, config) - - -def _set_output_safely(span: Span, value: object, primary: str = ATTR_OUTPUT) -> None: - config = _current_installation().config - _record_payload(span, _payload_keys(primary, True), value, config.hide_outputs, config) - - -def _finish_agent_span(span: Any, output: Any) -> None: - _set_output_safely(span, output) - span.set_status(Status(StatusCode.OK)) - - -def _mark_closed_early(span: Any, output: Any) -> None: - span.set_attribute(ATTR_STREAM_CLOSED_EARLY, True) - _set_output_safely(span, output) - - -def _bounded_text(value: str, config: TracingConfig) -> str: - if len(value) > config.max_attribute_length: - return value[: config.max_attribute_length] + TRUNCATION_MARKER - return value - - -def _failure_message(error: BaseException, config: TracingConfig) -> str: - if config.hide_outputs and isinstance(error, ValidationError): - return f"ValidationError: {error.error_count()} errors (__REDACTED__)" - return str(error) - - -def _record_failure(span: Span, error: BaseException) -> None: - config = _current_installation().config - message = _bounded_text(_failure_message(error, config), config) - span.set_attribute(ATTR_ERROR_TYPE, type(error).__qualname__) - span.record_exception(error, attributes={"exception.message": message, "exception.stacktrace": message}) - span.set_status(Status(StatusCode.ERROR, message)) - - -@contextmanager -def _agent_span(agent: AtomicAgent, args: tuple, kwargs: dict) -> Iterator[Span]: - installed = _current_installation() - with installed.tracer.start_as_current_span( - f"invoke_agent {_agent_display_name(agent)}", - attributes=_agent_start_attributes(agent), - record_exception=False, - set_status_on_exception=False, - ) as span: - _set_input_safely(span, _user_input_from_call(args, kwargs)) - token = _active_agent_span.set(span) - try: - yield span - except BaseException as error: - _record_failure(span, error) - raise - finally: - _active_agent_span.reset(token) - - -def _trace_agent_call(method_name: str, wrapped: Callable) -> Callable: - @functools.wraps(wrapped) - def traced(agent, *args, **kwargs): - if _suppressed(): - return wrapped(agent, *args, **kwargs) - with _agent_span(agent, args, kwargs) as span: - response = wrapped(agent, *args, **kwargs) - _finish_agent_span(span, response) - return response - - return traced - - -def _trace_agent_async_call(method_name: str, wrapped: Callable) -> Callable: - @functools.wraps(wrapped) - async def traced(agent, *args, **kwargs): - if _suppressed(): - return await wrapped(agent, *args, **kwargs) - with _agent_span(agent, args, kwargs) as span: - response = await wrapped(agent, *args, **kwargs) - _finish_agent_span(span, response) - return response - - return traced - - -def _start_agent_span(agent: AtomicAgent, method_name: str, args: tuple, kwargs: dict) -> Span: - if _suppressed(): - return trace_api.INVALID_SPAN - span = _current_installation().tracer.start_span( - f"invoke_agent {_agent_display_name(agent)}", - attributes=_agent_start_attributes(agent), - ) - _set_input_safely(span, _user_input_from_call(args, kwargs)) - return span - - -def _call_within_span(span: Any, function: Callable, *args: Any) -> Any: - token = context_api.attach(trace_api.set_span_in_context(span)) - agent_token = _active_agent_span.set(span) - try: - return function(*args) - finally: - _active_agent_span.reset(agent_token) - context_api.detach(token) - - -async def _await_within_span(span: Any, awaitable: Any) -> Any: - token = context_api.attach(trace_api.set_span_in_context(span)) - agent_token = _active_agent_span.set(span) - try: - return await awaitable - finally: - _active_agent_span.reset(agent_token) - context_api.detach(token) - - -class _SyncStreamCursor: - def __init__(self, span: Any, inner: Generator): - self.span = span - self.inner = inner - self.last: Any = None - self.final: Any = None - - def advance(self) -> bool: - try: - self.last = _call_within_span(self.span, next, self.inner) - except StopIteration as stop: - self.final = stop.value - return False - return True - - def close_early(self) -> None: - _call_within_span(self.span, self.inner.close) - _mark_closed_early(self.span, self.last) - self.span.set_status(Status(StatusCode.OK)) - - -class _AsyncStreamCursor: - """Async twin of :class:`_SyncStreamCursor`; async generators carry no return value, so the last partial is the output.""" - - def __init__(self, span: Any, inner: AsyncGenerator): - self.span = span - self.inner = inner - self.last: Any = None - - async def advance(self) -> bool: - try: - self.last = await _await_within_span(self.span, self.inner.__anext__()) - except StopAsyncIteration: - return False - return True - - async def close_early(self) -> None: - await _await_within_span(self.span, self.inner.aclose()) - _mark_closed_early(self.span, self.last) - self.span.set_status(Status(StatusCode.OK)) - - def mark_cancelled(self) -> None: - _mark_closed_early(self.span, self.last) - - -def _trace_agent_stream(method_name: str, wrapped: Callable) -> Callable: - @functools.wraps(wrapped) - def traced(agent, *args, **kwargs): - span = _start_agent_span(agent, method_name, args, kwargs) - cursor = _SyncStreamCursor(span, wrapped(agent, *args, **kwargs)) - try: - while cursor.advance(): - yield cursor.last - _finish_agent_span(span, cursor.final) - return cursor.final - except GeneratorExit: - cursor.close_early() - raise - except BaseException as error: - _record_failure(span, error) - raise - finally: - span.end() - - return traced - - -def _trace_agent_async_stream(method_name: str, wrapped: Callable) -> Callable: - @functools.wraps(wrapped) - async def traced(agent, *args, **kwargs): - span = _start_agent_span(agent, method_name, args, kwargs) - cursor = _AsyncStreamCursor(span, wrapped(agent, *args, **kwargs)) - try: - while await cursor.advance(): - yield cursor.last - _finish_agent_span(span, cursor.last) - except GeneratorExit: - await cursor.close_early() - raise - except asyncio.CancelledError: - cursor.mark_cancelled() - raise - except BaseException as error: - _record_failure(span, error) - raise - finally: - span.end() - - return traced - - -def _tool_parameter_attributes(tool: BaseTool) -> dict[str, AttributeValue]: - parameters = _best_effort_attributes({ATTR_TOOL_PARAMETERS: lambda: json.dumps(tool.input_schema.model_json_schema())}) - value = parameters.get(ATTR_TOOL_PARAMETERS) - if isinstance(value, str): - config = _current_installation().config - parameters[ATTR_TOOL_PARAMETERS] = _bounded_text(value, config) - if len(value) > config.max_attribute_length: - parameters[ATTR_TOOL_PARAMETERS + ".original_length"] = len(value) - return parameters - - -def _tool_display_name(tool: BaseTool) -> str: - try: - return tool.tool_name - except Exception as error: - logger.debug("Could not collect tracing tool name: %s", error) - return type(tool).__name__ - - -def _tool_start_attributes(tool: BaseTool) -> dict[str, AttributeValue]: - attributes: dict[str, AttributeValue] = { - ATTR_OPERATION: "execute_tool", - ATTR_TOOL_NAME: _tool_display_name(tool), - ATTR_TOOL_CLASS: type(tool).__name__, - ATTR_TOOL_TYPE: "function", - **_optional_attributes("TOOL", {}), - } - attributes.update(_best_effort_attributes({ATTR_TOOL_DESCRIPTION: lambda: tool.tool_description})) - if _current_installation().config.openinference_attributes: - attributes.update( - _best_effort_attributes( - { - ATTR_OI_TOOL_NAME: lambda: attributes[ATTR_TOOL_NAME], - ATTR_OI_TOOL_DESCRIPTION: lambda: tool.tool_description, - } - ) - ) - attributes.update(_tool_parameter_attributes(tool)) - return _without_none(attributes) - - -@contextmanager -def _tool_span(tool: BaseTool, args: tuple, kwargs: dict) -> Iterator[Span]: - attributes = _tool_start_attributes(tool) - with _current_installation().tracer.start_as_current_span( - f"execute_tool {attributes[ATTR_TOOL_NAME]}", - attributes=attributes, - record_exception=False, - set_status_on_exception=False, - ) as span: - _set_input_safely(span, _tool_input_from_call(args, kwargs), ATTR_TOOL_ARGUMENTS) - try: - yield span - except BaseException as error: - _record_failure(span, error) - raise - - -def _tool_input_from_call(args: tuple, kwargs: dict) -> Any: - return args[0] if args else kwargs.get("params") - - -def _trace_tool_method(wrapped: Callable) -> Callable: - if inspect.iscoroutinefunction(wrapped): - return _trace_tool_async_method(wrapped) - - @functools.wraps(wrapped) - def traced(tool, *args, **kwargs): - if _active_tool.get() is tool or _suppressed(): - return wrapped(tool, *args, **kwargs) - with _tool_span(tool, args, kwargs) as span: - result = _run_as_active_tool(tool, wrapped, args, kwargs) - _set_output_safely(span, result, ATTR_TOOL_RESULT) - span.set_status(Status(StatusCode.OK)) - return result - - return traced - - -def _trace_tool_async_method(wrapped: Callable) -> Callable: - @functools.wraps(wrapped) - async def traced(tool, *args, **kwargs): - if _active_tool.get() is tool or _suppressed(): - return await wrapped(tool, *args, **kwargs) - with _tool_span(tool, args, kwargs) as span: - result = await _await_as_active_tool(tool, wrapped, args, kwargs) - _set_output_safely(span, result, ATTR_TOOL_RESULT) - span.set_status(Status(StatusCode.OK)) - return result - - return traced - - -def _run_as_active_tool(tool: BaseTool, wrapped: Callable, args: tuple, kwargs: dict) -> Any: - token = _active_tool.set(tool) - try: - return wrapped(tool, *args, **kwargs) - finally: - _active_tool.reset(token) - - -async def _await_as_active_tool(tool: BaseTool, wrapped: Callable, args: tuple, kwargs: dict) -> Any: - token = _active_tool.set(tool) - try: - return await wrapped(tool, *args, **kwargs) - finally: - _active_tool.reset(token) - - -_AGENT_WRAPPERS: Dict[str, Callable[[str, Callable], Callable]] = { - AGENT_SYNC_METHOD: _trace_agent_call, - AGENT_ASYNC_METHOD: _trace_agent_async_call, - AGENT_SYNC_STREAM_METHOD: _trace_agent_stream, - AGENT_ASYNC_STREAM_METHOD: _trace_agent_async_stream, -} +"""Compatibility exports for Atomic Agents tracing.""" + +from . import installation, span_attributes, tracing_config + +TRACER_NAME = installation.TRACER_NAME +TRACED_MARKER = installation.TRACED_MARKER +install = installation.install +is_installed = installation.is_installed +uninstall = installation.uninstall +PayloadVisibility = tracing_config.PayloadVisibility +TracingConfig = tracing_config.TracingConfig +using_conversation = span_attributes.using_conversation + +ATTR_MODEL = span_attributes.ATTR_MODEL +ATTR_MODE = span_attributes.ATTR_MODE +ATTR_INPUT_SCHEMA = span_attributes.ATTR_INPUT_SCHEMA +ATTR_OUTPUT_SCHEMA = span_attributes.ATTR_OUTPUT_SCHEMA +ATTR_HISTORY_MESSAGE_COUNT = span_attributes.ATTR_HISTORY_MESSAGE_COUNT +ATTR_CONTEXT_PROVIDERS = span_attributes.ATTR_CONTEXT_PROVIDERS +ATTR_SYSTEM_PROMPT_SHA256 = span_attributes.ATTR_SYSTEM_PROMPT_SHA256 +ATTR_STREAM_CLOSED_EARLY = span_attributes.ATTR_STREAM_CLOSED_EARLY +ATTR_OPERATION = span_attributes.ATTR_OPERATION +ATTR_AGENT_NAME = span_attributes.ATTR_AGENT_NAME +ATTR_REQUEST_MODEL = span_attributes.ATTR_REQUEST_MODEL +ATTR_PROVIDER_NAME = span_attributes.ATTR_PROVIDER_NAME +ATTR_CONVERSATION_ID = span_attributes.ATTR_CONVERSATION_ID +ATTR_TOOL_NAME = span_attributes.ATTR_TOOL_NAME +ATTR_TOOL_DESCRIPTION = span_attributes.ATTR_TOOL_DESCRIPTION +ATTR_TOOL_TYPE = span_attributes.ATTR_TOOL_TYPE +ATTR_TOOL_ARGUMENTS = span_attributes.ATTR_TOOL_ARGUMENTS +ATTR_TOOL_RESULT = span_attributes.ATTR_TOOL_RESULT +ATTR_SPAN_KIND = span_attributes.ATTR_SPAN_KIND +ATTR_INPUT_VALUE = span_attributes.ATTR_INPUT_VALUE +ATTR_INPUT_MIME_TYPE = span_attributes.ATTR_INPUT_MIME_TYPE +ATTR_OUTPUT_VALUE = span_attributes.ATTR_OUTPUT_VALUE +ATTR_OUTPUT_MIME_TYPE = span_attributes.ATTR_OUTPUT_MIME_TYPE +ATTR_OI_AGENT_NAME = span_attributes.ATTR_OI_AGENT_NAME +ATTR_OI_TOOL_NAME = span_attributes.ATTR_OI_TOOL_NAME +ATTR_OI_TOOL_DESCRIPTION = span_attributes.ATTR_OI_TOOL_DESCRIPTION +ATTR_TOOL_PARAMETERS = span_attributes.ATTR_TOOL_PARAMETERS +ATTR_SESSION_ID = span_attributes.ATTR_SESSION_ID +ATTR_INPUT = span_attributes.ATTR_INPUT +ATTR_OUTPUT = span_attributes.ATTR_OUTPUT +ATTR_AGENT_CLASS = span_attributes.ATTR_AGENT_CLASS +ATTR_ERROR_TYPE = span_attributes.ATTR_ERROR_TYPE +ATTR_TOOL_CLASS = span_attributes.ATTR_TOOL_CLASS diff --git a/atomic-agents/atomic_agents/observability/tracing_config.py b/atomic-agents/atomic_agents/observability/tracing_config.py new file mode 100644 index 00000000..8c25224f --- /dev/null +++ b/atomic-agents/atomic_agents/observability/tracing_config.py @@ -0,0 +1,39 @@ +"""Tracing configuration and payload visibility.""" + +import os +from dataclasses import dataclass +from typing import Protocol + + +class PayloadVisibility(Protocol): + hide_inputs: bool + hide_outputs: bool + + +@dataclass(frozen=True) +class TracingConfig: + hide_inputs: bool = False + hide_outputs: bool = False + max_attribute_length: int = 4096 + openinference_attributes: bool | None = None + llm_spans: bool = True + + @classmethod + def from_environment(cls) -> "TracingConfig": + return cls( + hide_inputs=_visibility_environment("INPUTS").lower() in ("true", "1"), + hide_outputs=_visibility_environment("OUTPUTS").lower() in ("true", "1"), + max_attribute_length=int(os.environ.get("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", "4096")), + ) + + @classmethod + def coerce(cls, config: "TracingConfig | PayloadVisibility | None") -> "TracingConfig": + if config is None: + return cls.from_environment() + if isinstance(config, cls): + return config + return cls(hide_inputs=config.hide_inputs, hide_outputs=config.hide_outputs) + + +def _visibility_environment(direction: str) -> str: + return os.environ.get(f"ATOMIC_AGENTS_TRACE_HIDE_{direction}", os.environ.get(f"OPENINFERENCE_HIDE_{direction}", "")) diff --git a/atomic-agents/tests/observability/test_tracing.py b/atomic-agents/tests/observability/test_tracing.py index 84fe51e5..66e6036a 100644 --- a/atomic-agents/tests/observability/test_tracing.py +++ b/atomic-agents/tests/observability/test_tracing.py @@ -25,7 +25,7 @@ from atomic_agents.context import BaseDynamicContextProvider, SystemPromptGenerator from atomic_agents.observability import AtomicAgentsInstrumentor import atomic_agents.observability as observability -from atomic_agents.observability import tracing as tracing +from atomic_agents.observability import agent_spans, installation, span_attributes, span_payloads, tracing as tracing class QuestionSchema(BaseIOSchema): @@ -622,14 +622,14 @@ def test_second_instrument_replaces_provider_and_warns(exporter, caplog): other_provider.add_span_processor(SimpleSpanProcessor(other_exporter)) agent = make_agent(sync_client()) agent.run(QuestionSchema(question="first")) - installed = tracing._installation + installed = installation._installation wrapped_run = AtomicAgent.run with caplog.at_level(logging.WARNING): AtomicAgentsInstrumentor().instrument(tracer_provider=other_provider) agent.run(QuestionSchema(question="second")) - assert tracing._installation is installed + assert installation._installation is installed assert AtomicAgent.run is wrapped_run assert len(spans_of(exporter)) == 1 assert len(spans_of(other_exporter)) == 1 @@ -710,7 +710,7 @@ def test_openinference_below_the_floor_only_disables_context_with_a_warning(monk assert "0.1.38 or newer is required" in caplog.text assert AtomicAgentsInstrumentor().is_instrumented_by_opentelemetry - assert tracing._context_attributes() == {} + assert span_attributes._context_attributes() == {} AtomicAgentsInstrumentor().uninstrument() @@ -877,7 +877,7 @@ def configured_tracing(): @pytest.mark.parametrize("openinference_enabled", [False, True]) def test_genai_attributes_survive_without_openinference(monkeypatch, configured_tracing, openinference_enabled): provider, exporter = configured_tracing - monkeypatch.setattr(tracing, "_import_context_attributes", lambda: None) + monkeypatch.setattr(span_attributes, "_import_context_attributes", lambda: None) config = tracing.TracingConfig(openinference_attributes=True if openinference_enabled else None) AtomicAgentsInstrumentor().instrument(tracer_provider=provider, config=config) client = sync_client() @@ -1089,11 +1089,11 @@ def test_plain_parent_span_owns_agent_and_tool_trace(exporter): def test_same_provider_and_config_is_a_silent_no_op(exporter, caplog): - installed = tracing._installation + installed = installation._installation tracer = installed.tracer with caplog.at_level(logging.DEBUG, logger=tracing.__name__): AtomicAgentsInstrumentor().instrument(tracer_provider=exporter.provider, config=tracing.TracingConfig()) - assert tracing._installation is installed + assert installation._installation is installed assert installed.tracer is tracer assert not caplog.records @@ -1141,7 +1141,7 @@ def replace_provider(**kwargs): def test_none_descriptions_are_omitted_without_otel_warnings(exporter, caplog, name): agent = make_agent(sync_client()) setattr(agent, name, None) - with tracing._agent_span(agent, (), {}): + with agent_spans._agent_span(agent, (), {}): pass (span,) = spans_of(exporter) assert f"atomic_agents.{name}" not in span.attributes @@ -1152,7 +1152,7 @@ def test_none_descriptions_are_omitted_without_otel_warnings(exporter, caplog, n def test_broken_history_preserves_mandatory_agent_attributes(exporter, monkeypatch, caplog): agent = make_agent(sync_client()) monkeypatch.setattr(type(agent), "history", PropertyMock(side_effect=RuntimeError("history unavailable")), raising=False) - with caplog.at_level(logging.DEBUG), tracing._agent_span(agent, (), {}): + with caplog.at_level(logging.DEBUG), agent_spans._agent_span(agent, (), {}): pass (span,) = spans_of(exporter) assert span.attributes["gen_ai.operation.name"] == "invoke_agent" @@ -1188,7 +1188,7 @@ def test_configured_tool_name_is_the_span_join_key(exporter): def test_hidden_payload_is_never_serialized(exporter, caplog, hide): payload = UnserializableInputSchema(store=AnswerStore()) with exporter.provider.get_tracer("test").start_as_current_span("payload") as span: - tracing._record_payload(span, (("payload", "payload.mime_type"),), payload, hide, tracing.TracingConfig()) + span_payloads._record_payload(span, (("payload", "payload.mime_type"),), payload, hide, tracing.TracingConfig()) (span,) = spans_of(exporter) assert ("Could not serialize tracing payload" in caplog.text) is not hide assert span.attributes.get("payload") == ("__REDACTED__" if hide else None) @@ -1198,9 +1198,9 @@ def test_hidden_payload_is_never_serialized(exporter, caplog, hide): @pytest.mark.parametrize("hide", [False, True]) def test_hidden_non_model_payload_does_not_call_json_dumps(exporter, monkeypatch, caplog, hide): serialize = Mock(side_effect=ValueError("serialization failed")) - monkeypatch.setattr(tracing.json, "dumps", serialize) + monkeypatch.setattr(span_payloads.json, "dumps", serialize) with exporter.provider.get_tracer("test").start_as_current_span("payload") as span: - tracing._record_payload(span, (("payload", None),), object(), hide, tracing.TracingConfig()) + span_payloads._record_payload(span, (("payload", None),), object(), hide, tracing.TracingConfig()) assert serialize.call_count == (0 if hide else 1) assert ("Could not serialize tracing payload" in caplog.text) is not hide @@ -1233,7 +1233,7 @@ def test_instructor_provider_names_follow_genai_conventions(exporter, provider, def test_none_context_attributes_are_filtered(exporter, monkeypatch, caplog): - monkeypatch.setattr(tracing, "_context_attributes", lambda: {"optional": None}) + monkeypatch.setattr(span_attributes, "_context_attributes", lambda: {"optional": None}) make_agent(sync_client()).run(QuestionSchema(question="question")) (span,) = spans_of(exporter) assert "optional" not in span.attributes @@ -1267,7 +1267,7 @@ def test_error_status_and_recorded_message_are_bounded(configured_tracing): AtomicAgentsInstrumentor().instrument(tracer_provider=provider, config=tracing.TracingConfig(max_attribute_length=12)) error = ValueError("private" * 100) agent = make_agent(sync_client()) - with pytest.raises(ValueError) as raised, tracing._agent_span(agent, (), {}): + with pytest.raises(ValueError) as raised, agent_spans._agent_span(agent, (), {}): raise error (span,) = spans_of(exporter) expected = str(error)[:12] + "...[truncated]" @@ -1284,7 +1284,7 @@ def test_hidden_validation_errors_do_not_export_model_output(configured_tracing) QuestionSchema.model_validate({"question": ["secret model output"]}) except ValidationError as error: with provider.get_tracer("test").start_as_current_span("failure") as span: - tracing._record_failure(span, error) + span_payloads._record_failure(span, error) (span,) = spans_of(exporter) assert span.status.description == "ValidationError: 1 errors (__REDACTED__)" assert span.events[0].attributes["exception.message"] == span.status.description From 88d74b2b2b276325b86e2fc4f8fb3566d9156922 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 12:09:11 +0200 Subject: [PATCH 09/25] Key the fresh-install smoke on operation name after the tool identity change SQ-149 made gen_ai.tool.name and the execute_tool span name use tool.tool_name (the input schema title), so the smoke script's lookup of "execute_tool SmokeTool" raised KeyError in run 39582225418. Look spans up by gen_ai.operation.name, assert the name against SmokeTool().tool_name and check the class under atomic_agents.tool.class. The extracted script passes locally with OpenInference keys forced off. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/code-quality.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 67642deb..fa41b319 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -152,12 +152,13 @@ jobs: spans = exporter.get_finished_spans() for span in spans: print(f"{span.name}: {sorted(span.attributes)}") - spans_by_name = {span.name: span for span in spans} - tool_span = spans_by_name["execute_tool SmokeTool"] - agent_span = spans_by_name["invoke_agent SmokeAgent"] - assert tool_span.attributes["gen_ai.operation.name"] == "execute_tool" - assert tool_span.attributes["gen_ai.tool.name"] == "SmokeTool" - assert agent_span.attributes["gen_ai.operation.name"] == "invoke_agent" + spans_by_operation = {span.attributes["gen_ai.operation.name"]: span for span in spans} + tool_span = spans_by_operation["execute_tool"] + agent_span = spans_by_operation["invoke_agent"] + assert tool_span.name == f"execute_tool {SmokeTool().tool_name}" + assert agent_span.name == "invoke_agent SmokeAgent" + assert tool_span.attributes["gen_ai.tool.name"] == SmokeTool().tool_name + assert tool_span.attributes["atomic_agents.tool.class"] == "SmokeTool" assert agent_span.attributes["atomic_agents.input"] == "__REDACTED__" for span in (tool_span, agent_span): assert not any(key.startswith("openinference.") for key in span.attributes) From 57711f242869d0f55e9ec739d12669674e96624e Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 12:18:43 +0200 Subject: [PATCH 10/25] Trace Instructor attempts and cumulative agent token usage Exercise retry parentage, shared-client isolation and stream cleanup. Co-Authored-By: Claude Code --- .../observability/agent_spans.py | 45 ++- .../observability/installation.py | 3 +- .../observability/llm_attempt_spans.py | 219 ++++++++++++ .../observability/test_llm_attempt_spans.py | 321 ++++++++++++++++++ 4 files changed, 571 insertions(+), 17 deletions(-) create mode 100644 atomic-agents/atomic_agents/observability/llm_attempt_spans.py create mode 100644 atomic-agents/tests/observability/test_llm_attempt_spans.py diff --git a/atomic-agents/atomic_agents/observability/agent_spans.py b/atomic-agents/atomic_agents/observability/agent_spans.py index 82ffdd0f..37fda75c 100644 --- a/atomic-agents/atomic_agents/observability/agent_spans.py +++ b/atomic-agents/atomic_agents/observability/agent_spans.py @@ -12,7 +12,7 @@ from atomic_agents.agents.atomic_agent import AtomicAgent -from . import installation +from . import installation, llm_attempt_spans from .span_attributes import ATTR_STREAM_CLOSED_EARLY, _agent_display_name, _agent_start_attributes, _suppressed from .span_payloads import _record_failure, _set_input_safely, _set_output_safely @@ -51,13 +51,16 @@ def _agent_span(agent: AtomicAgent, args: tuple, kwargs: dict) -> Iterator[Span] set_status_on_exception=False, ) as span: _set_input_safely(span, _user_input_from_call(args, kwargs)) + run = llm_attempt_spans.new_run(span, agent) token = _active_agent_span.set(span) try: - yield span + with llm_attempt_spans.active_run(run): + yield span except BaseException as error: _record_failure(span, error) raise finally: + llm_attempt_spans.finish_attempt(run) _active_agent_span.reset(token) @@ -98,43 +101,48 @@ def _start_agent_span(agent: AtomicAgent, method_name: str, args: tuple, kwargs: return span -def _call_within_span(span: Any, function: Callable, *args: Any) -> Any: +def _call_within_span(run: llm_attempt_spans._AgentRun, function: Callable, *args: Any) -> Any: + span = run.span token = context_api.attach(trace_api.set_span_in_context(span)) agent_token = _active_agent_span.set(span) try: - return function(*args) + with llm_attempt_spans.active_run(run): + return function(*args) finally: _active_agent_span.reset(agent_token) context_api.detach(token) -async def _await_within_span(span: Any, awaitable: Any) -> Any: +async def _await_within_span(run: llm_attempt_spans._AgentRun, awaitable: Any) -> Any: + span = run.span token = context_api.attach(trace_api.set_span_in_context(span)) agent_token = _active_agent_span.set(span) try: - return await awaitable + with llm_attempt_spans.active_run(run): + return await awaitable finally: _active_agent_span.reset(agent_token) context_api.detach(token) class _SyncStreamCursor: - def __init__(self, span: Any, inner: Generator): - self.span = span + def __init__(self, run: llm_attempt_spans._AgentRun, inner: Generator): + self.run = run + self.span = run.span self.inner = inner self.last: Any = None self.final: Any = None def advance(self) -> bool: try: - self.last = _call_within_span(self.span, next, self.inner) + self.last = _call_within_span(self.run, next, self.inner) except StopIteration as stop: self.final = stop.value return False return True def close_early(self) -> None: - _call_within_span(self.span, self.inner.close) + _call_within_span(self.run, self.inner.close) _mark_closed_early(self.span, self.last) self.span.set_status(Status(StatusCode.OK)) @@ -142,20 +150,21 @@ def close_early(self) -> None: class _AsyncStreamCursor: """Async twin of :class:`_SyncStreamCursor`; async generators carry no return value, so the last partial is the output.""" - def __init__(self, span: Any, inner: AsyncGenerator): - self.span = span + def __init__(self, run: llm_attempt_spans._AgentRun, inner: AsyncGenerator): + self.run = run + self.span = run.span self.inner = inner self.last: Any = None async def advance(self) -> bool: try: - self.last = await _await_within_span(self.span, self.inner.__anext__()) + self.last = await _await_within_span(self.run, self.inner.__anext__()) except StopAsyncIteration: return False return True async def close_early(self) -> None: - await _await_within_span(self.span, self.inner.aclose()) + await _await_within_span(self.run, self.inner.aclose()) _mark_closed_early(self.span, self.last) self.span.set_status(Status(StatusCode.OK)) @@ -167,7 +176,8 @@ def _trace_agent_stream(method_name: str, wrapped: Callable) -> Callable: @functools.wraps(wrapped) def traced(agent, *args, **kwargs): span = _start_agent_span(agent, method_name, args, kwargs) - cursor = _SyncStreamCursor(span, wrapped(agent, *args, **kwargs)) + run = llm_attempt_spans.new_run(span, agent) + cursor = _SyncStreamCursor(run, wrapped(agent, *args, **kwargs)) try: while cursor.advance(): yield cursor.last @@ -180,6 +190,7 @@ def traced(agent, *args, **kwargs): _record_failure(span, error) raise finally: + llm_attempt_spans.finish_attempt(run) span.end() return traced @@ -189,7 +200,8 @@ def _trace_agent_async_stream(method_name: str, wrapped: Callable) -> Callable: @functools.wraps(wrapped) async def traced(agent, *args, **kwargs): span = _start_agent_span(agent, method_name, args, kwargs) - cursor = _AsyncStreamCursor(span, wrapped(agent, *args, **kwargs)) + run = llm_attempt_spans.new_run(span, agent) + cursor = _AsyncStreamCursor(run, wrapped(agent, *args, **kwargs)) try: while await cursor.advance(): yield cursor.last @@ -204,6 +216,7 @@ async def traced(agent, *args, **kwargs): _record_failure(span, error) raise finally: + llm_attempt_spans.finish_attempt(run) span.end() return traced diff --git a/atomic-agents/atomic_agents/observability/installation.py b/atomic-agents/atomic_agents/observability/installation.py index 326c7acc..981ce2d3 100644 --- a/atomic-agents/atomic_agents/observability/installation.py +++ b/atomic-agents/atomic_agents/observability/installation.py @@ -12,7 +12,7 @@ from atomic_agents.agents.atomic_agent import AtomicAgent from atomic_agents.base.base_tool import BaseTool -from . import agent_spans, span_attributes, tool_spans +from . import agent_spans, llm_attempt_spans, span_attributes, tool_spans from .tracing_config import PayloadVisibility, TracingConfig if TYPE_CHECKING: @@ -85,6 +85,7 @@ def uninstall() -> None: tool_spans._restore_tool_class(tool_class) for method_name in agent_spans._AGENT_WRAPPERS: _restore_method(AtomicAgent, method_name) + llm_attempt_spans.unregister_clients() _installation = None diff --git a/atomic-agents/atomic_agents/observability/llm_attempt_spans.py b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py new file mode 100644 index 00000000..3dc7f4c2 --- /dev/null +++ b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py @@ -0,0 +1,219 @@ +"""Instructor attempt spans and cumulative agent usage.""" + +import contextvars +import functools +import logging +import weakref +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Callable, Iterator, ParamSpec, Protocol + +from opentelemetry import context as context_api, trace as trace_api +from opentelemetry.trace import Span, Status, StatusCode, Tracer +from opentelemetry.util.types import AttributeValue + +from atomic_agents.agents.atomic_agent import AtomicAgent + +from . import installation +from .span_attributes import _best_effort_attributes, _optional_attributes, _provider_attributes, _without_none +from .span_payloads import _bounded_text, _failure_message, _write_payload +from .tracing_config import TracingConfig + +logger = logging.getLogger("atomic_agents.observability.tracing") +_PARAMETERS = ParamSpec("_PARAMETERS") +_RESPONSE_MODEL = "gen_ai.response.model" +_ATTEMPT = "atomic_agents.llm.attempt" +_VALIDATION_ERROR = "atomic_agents.llm.last_validation_error" + + +class _Usage(Protocol): + input_tokens: int + output_tokens: int + + +class _Response(Protocol): + model: str + usage: _Usage + + +class _HookClient(Protocol): + def on(self, event: str, handler: Callable[..., None]) -> object: ... + + def off(self, event: str, handler: Callable[..., None]) -> object: ... + + +@dataclass +class _AgentRun: + span: Span + tracer: Tracer + config: TracingConfig + provider: dict[str, AttributeValue] + attempt_count: int = 0 + open_attempt: Span | None = None + context_token: contextvars.Token | None = None + + +_active_run: contextvars.ContextVar[_AgentRun | None] = contextvars.ContextVar("atomic_agents_llm_run", default=None) +_clients: weakref.WeakSet[_HookClient] = weakref.WeakSet() + + +def _never_raise(handler: Callable[_PARAMETERS, None]) -> Callable[_PARAMETERS, None]: + @functools.wraps(handler) + def guarded(*args: _PARAMETERS.args, **kwargs: _PARAMETERS.kwargs) -> None: + try: + handler(*args, **kwargs) + except Exception: + logger.warning("Instructor tracing hook failed: %s", handler.__name__, exc_info=True) + + return guarded + + +@_never_raise +def register_client(client: _HookClient) -> None: + if not hasattr(client, "on") or client in _clients: + return + for event, handler in _HANDLERS.items(): + client.on(event, handler) + _clients.add(client) + + +@_never_raise +def unregister_clients() -> None: + for client in list(_clients): + _unregister_client(client) + _clients.clear() + + +@_never_raise +def _unregister_client(client: _HookClient) -> None: + for event, handler in _HANDLERS.items(): + client.off(event, handler) + + +def new_run(span: Span, agent: AtomicAgent) -> _AgentRun: + installed = installation._current_installation() + register_client(getattr(agent, "client")) + provider = _best_effort_attributes( + {"gen_ai.provider.name": lambda: _provider_attributes(agent).get("gen_ai.provider.name")} + ) + return _AgentRun(span, installed.tracer, installed.config, provider) + + +@contextmanager +def active_run(run: _AgentRun) -> Iterator[None]: + token = _active_run.set(None if run.span is trace_api.INVALID_SPAN else run) + _attach_attempt(run) + try: + yield + finally: + _detach_attempt(run) + _active_run.reset(token) + + +def _attach_attempt(run: _AgentRun) -> None: + if run.open_attempt is not None: + run.context_token = context_api.attach(trace_api.set_span_in_context(run.open_attempt)) + + +def _detach_attempt(run: _AgentRun) -> None: + if run.context_token is not None: + context_api.detach(run.context_token) + run.context_token = None + + +@_never_raise +def finish_attempt(run: _AgentRun, status: Status = Status(StatusCode.UNSET)) -> None: + if run.open_attempt is None: + return + run.open_attempt.set_status(status) + run.open_attempt.end() + run.open_attempt = None + _detach_attempt(run) + + +@_never_raise +def _completion_kwargs(*args: object, **kwargs: object) -> None: + run = _active_run.get() + if run is None: + return + finish_attempt(run) + run.attempt_count += 1 + if not run.config.llm_spans: + return + model = str(kwargs["model"]) if kwargs.get("model") is not None else None + attributes: dict[str, AttributeValue | None] = { + "gen_ai.operation.name": "chat", + "gen_ai.request.model": model, + _ATTEMPT: run.attempt_count, + } + attributes.update(run.provider) + attributes.update(_optional_attributes("LLM", {})) + run.open_attempt = run.tracer.start_span(f"chat {model}", attributes=_without_none(attributes)) + _attach_attempt(run) + + +def _usage_attributes(usage: _Usage | None) -> dict[str, int]: + readers = { + "input_tokens": getattr(usage, "input_tokens", getattr(usage, "prompt_tokens", None)), + "output_tokens": getattr(usage, "output_tokens", getattr(usage, "completion_tokens", None)), + "cache_read.input_tokens": getattr(getattr(usage, "prompt_tokens_details", None), "cached_tokens", None), + "reasoning.output_tokens": getattr(getattr(usage, "completion_tokens_details", None), "reasoning_tokens", None), + } + return {"gen_ai.usage." + key: value for key, value in readers.items() if isinstance(value, int)} + + +@_never_raise +def _completion_response(response: _Response, **kwargs: object) -> None: + run = _active_run.get() + if run is None: + return + attributes = _without_none({_RESPONSE_MODEL: getattr(response, "model", None)}) + run.span.set_attributes(attributes) + _complete_response_attempt(run, response, attributes) + + +def _complete_response_attempt(run: _AgentRun, response: _Response, attributes: dict[str, AttributeValue]) -> None: + if run.open_attempt is None: + return + run.open_attempt.set_attributes(attributes) + run.open_attempt.set_attributes(_usage_attributes(getattr(response, "usage", None))) + finish_attempt(run, Status(StatusCode.OK)) + + +@_never_raise +def _completion_error(error: BaseException, **kwargs: object) -> None: + run = _active_run.get() + if run is None or run.open_attempt is None: + return + message = _bounded_text(_failure_message(error, run.config), run.config) + run.open_attempt.set_attribute("error.type", type(error).__qualname__) + run.open_attempt.record_exception(error, attributes={"exception.message": message, "exception.stacktrace": message}) + finish_attempt(run, Status(StatusCode.ERROR, message)) + + +@_never_raise +def _parse_error(error: BaseException, **kwargs: object) -> None: + run = _active_run.get() + if run is None: + return + text = "__REDACTED__" if run.config.hide_outputs else _failure_message(error, run.config) + _write_payload(run.span, _VALIDATION_ERROR, None, text, run.config.hide_outputs, run.config) + run.span.add_event("validation_error", {_ATTEMPT: run.attempt_count, _VALIDATION_ERROR: _bounded_text(text, run.config)}) + + +@_never_raise +def _completion_usage(usage: _Usage, *, attempt_number: int, **kwargs: object) -> None: + run = _active_run.get() + if run is None: + return + run.span.set_attributes(_usage_attributes(usage)) + run.span.set_attribute("atomic_agents.llm.attempts", attempt_number) + + +_HANDLERS: dict[str, Callable[..., None]] = { + "completion:kwargs": _completion_kwargs, + "completion:response": _completion_response, + "completion:error": _completion_error, + "parse:error": _parse_error, + "completion:usage": _completion_usage, +} diff --git a/atomic-agents/tests/observability/test_llm_attempt_spans.py b/atomic-agents/tests/observability/test_llm_attempt_spans.py new file mode 100644 index 00000000..abd59d5d --- /dev/null +++ b/atomic-agents/tests/observability/test_llm_attempt_spans.py @@ -0,0 +1,321 @@ +import asyncio +from types import SimpleNamespace +from unittest.mock import Mock + +import instructor +import pytest +from instructor.core.hooks import Hooks +from opentelemetry import trace as trace_api +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema +from atomic_agents.observability import TracingConfig, using_conversation +from atomic_agents.observability import agent_spans, installation, llm_attempt_spans + + +class Question(BaseIOSchema): + """A test question.""" + + question: str + + +class Answer(BaseIOSchema): + """A test answer.""" + + answer: str + + +@pytest.fixture +def exporter(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + installation.install(tracer_provider=provider, config=TracingConfig(openinference_attributes=False)) + yield exporter + installation.uninstall() + + +def usage(input_tokens=3, output_tokens=2): + return SimpleNamespace( + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + prompt_tokens_details=SimpleNamespace(cached_tokens=1), + completion_tokens_details=SimpleNamespace(reasoning_tokens=2), + ) + + +def finished_spans(exporter, count): + spans = exporter.get_finished_spans() + assert len(spans) == count + return spans + + +def response(): + return SimpleNamespace(model="response-model", usage=usage()) + + +def scripted_client(script): + client = Mock(spec=instructor.Instructor) + client.provider = instructor.Provider.OPENAI + client.hooks = Hooks() + client.on.side_effect = client.hooks.on + client.off.side_effect = client.hooks.off + client.chat = Mock() + client.chat.completions = Mock() + + def create(**kwargs): + script(client.hooks, kwargs) + return Answer(answer="42") + + client.chat.completions.create.side_effect = create + return client + + +def agent(client): + return AtomicAgent[Question, Answer](AgentConfig(client=client, model="test-model")) + + +def retry_script(hooks, kwargs): + hooks.emit_completion_arguments(**kwargs) + hooks.emit_completion_response(response()) + hooks.emit_completion_usage(usage(), attempt_number=1) + hooks.emit_parse_error(ValueError("invalid output")) + hooks.emit_completion_arguments(**kwargs) + hooks.emit_completion_response(response()) + hooks.emit_completion_usage(usage(6, 4), attempt_number=2) + + +def run_agent(client): + return agent(client).run(Question(question="test")) + + +def test_retry_spans_usage_validation_and_parentage(exporter): + client = scripted_client(retry_script) + with using_conversation("conversation"): + run_agent(client) + first, second, parent = finished_spans(exporter, 3) + assert [first.name, second.name] == ["chat test-model", "chat test-model"] + assert [first.attributes["atomic_agents.llm.attempt"], second.attributes["atomic_agents.llm.attempt"]] == [1, 2] + assert first.parent.span_id == second.parent.span_id == parent.context.span_id + assert first.status.status_code == second.status.status_code == StatusCode.OK + assert first.attributes["gen_ai.provider.name"] == "openai" + assert first.attributes["gen_ai.conversation.id"] == "conversation" + assert first.attributes["gen_ai.usage.input_tokens"] == second.attributes["gen_ai.usage.input_tokens"] == 3 + assert first.attributes["gen_ai.usage.output_tokens"] == 2 + assert first.attributes["gen_ai.usage.cache_read.input_tokens"] == 1 + assert first.attributes["gen_ai.usage.reasoning.output_tokens"] == 2 + assert parent.attributes["gen_ai.usage.input_tokens"] == 6 + assert parent.attributes["gen_ai.usage.output_tokens"] == 4 + assert parent.attributes.get("atomic_agents.llm.attempts") == 2 + assert parent.attributes["atomic_agents.llm.last_validation_error"] == "invalid output" + assert parent.attributes["gen_ai.response.model"] == first.attributes["gen_ai.response.model"] == "response-model" + event = parent.events[0] + assert event.name == "validation_error" + assert event.attributes["atomic_agents.llm.attempt"] == 1 + + +def test_disabled_attempt_spans_keep_totals(exporter): + installed = installation._current_installation() + installation.install(tracer_provider=installed.tracer_provider, config=TracingConfig(llm_spans=False)) + run_agent(scripted_client(retry_script)) + (parent,) = finished_spans(exporter, 1) + assert parent.attributes.get("atomic_agents.llm.attempts") == 2 + assert parent.attributes["gen_ai.usage.input_tokens"] == 6 + + +def test_completion_error_is_bounded_and_ends_attempt(exporter): + def script(hooks, kwargs): + hooks.emit_completion_arguments(**kwargs) + hooks.emit_completion_error(ValueError("failed"), attempt_number=1, max_attempts=1, is_last_attempt=True) + + run_agent(scripted_client(script)) + child, parent = finished_spans(exporter, 2) + assert child.status.status_code == StatusCode.ERROR + assert child.attributes["error.type"] == "ValueError" + assert child.events[0].attributes["exception.message"] == "failed" + assert parent.status.status_code == StatusCode.OK + + +def test_registration_once_and_uninstall_preserves_application_hooks(exporter): + client = scripted_client(retry_script) + application_hook = Mock() + client.hooks.on("completion:response", application_hook) + for _ in range(3): + run_agent(client) + assert client.on.call_count == 5 + assert application_hook.call_count == 6 + installation.uninstall() + assert client.off.call_count == 5 + client.hooks.emit_completion_response(response()) + assert application_hook.call_count == 7 + assert not llm_attempt_spans._clients + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_concurrent_shared_client_runs_are_isolated(exporter, asynchronous): + client = scripted_client(retry_script) + + async def invoke(number): + async def complete(agent, *args, **kwargs): + client.hooks.emit_completion_arguments(model="test-model") + await asyncio.sleep(0) + client.hooks.emit_completion_response(response()) + client.hooks.emit_completion_usage(usage(number), attempt_number=1) + return Answer(answer="ok") + + if asynchronous: + return await agent_spans._trace_agent_async_call("run_async", complete)(agent(client)) + return await asyncio.to_thread(run_agent, client) + + async def gather(): + await asyncio.gather(invoke(10), invoke(20)) + + asyncio.run(gather()) + parents = [span for span in exporter.get_finished_spans() if span.name.startswith("invoke_agent")] + expected = [10, 20] if asynchronous else [6, 6] + assert sorted(span.attributes.get("gen_ai.usage.input_tokens", -1) for span in parents) == expected + assert all(span.attributes["atomic_agents.llm.attempts"] == (1 if asynchronous else 2) for span in parents) + + +@pytest.mark.parametrize("kind", ["sync", "async", "stream", "async_stream"]) +@pytest.mark.parametrize("fails", [False, True]) +def test_open_attempt_ends_at_agent_end(exporter, kind, fails): + client = scripted_client(retry_script) + + def start(): + client.hooks.emit_completion_arguments(model="test-model") + if fails: + raise ValueError("agent failed") + return Answer(answer="partial") + + def call(*args, **kwargs): + return start() + + async def async_call(*args, **kwargs): + return start() + + def stream(*args, **kwargs): + yield start() + yield Answer(answer="final") + + async def async_stream(*args, **kwargs): + yield start() + yield Answer(answer="final") + + def invoke(): + if kind == "sync": + return agent_spans._trace_agent_call("run", call)(agent(client)) + if kind == "async": + return asyncio.run(agent_spans._trace_agent_async_call("run_async", async_call)(agent(client))) + if kind == "stream": + iterator = agent_spans._trace_agent_stream("run_stream", stream)(agent(client)) + next(iterator) + assert trace_api.get_current_span() is trace_api.INVALID_SPAN + iterator.close() + return None + return asyncio.run(consume_async_stream()) + + async def consume_async_stream(): + iterator = agent_spans._trace_agent_async_stream("run_async_stream", async_stream)(agent(client)) + await anext(iterator) + assert trace_api.get_current_span() is trace_api.INVALID_SPAN + await iterator.aclose() + + if fails: + with pytest.raises(ValueError, match="agent failed"): + invoke() + else: + invoke() + child, parent = finished_spans(exporter, 2) + assert child.status.status_code == StatusCode.UNSET + assert child.parent.span_id == parent.context.span_id + assert child.end_time <= parent.end_time + + +def test_validation_text_redacted_and_anthropic_usage(exporter): + installed = installation._current_installation() + installation.install(tracer_provider=installed.tracer_provider, config=TracingConfig(hide_outputs=True)) + + def script(hooks, kwargs): + hooks.emit_completion_arguments(**kwargs) + hooks.emit_parse_error(ValueError("secret")) + hooks.emit_completion_usage(SimpleNamespace(input_tokens=9, output_tokens=4), attempt_number=1) + + run_agent(scripted_client(script)) + child, parent = finished_spans(exporter, 2) + assert parent.attributes["atomic_agents.llm.last_validation_error"] == "__REDACTED__" + assert parent.events[0].attributes["atomic_agents.llm.last_validation_error"] == "__REDACTED__" + assert parent.attributes["gen_ai.usage.input_tokens"] == 9 + + +def test_hook_failure_warns_without_breaking_agent(exporter, caplog): + class BrokenResponse: + @property + def model(self): + raise RuntimeError("broken metadata") + + def script(hooks, kwargs): + hooks.emit_completion_arguments(**kwargs) + hooks.emit_completion_response(BrokenResponse()) + + assert run_agent(scripted_client(script)).answer == "42" + child, parent = finished_spans(exporter, 2) + assert child.status.status_code == StatusCode.UNSET + assert "Instructor tracing hook failed" in caplog.text + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_stream_attempt_context_survives_cursor_advances(exporter, asynchronous): + client = scripted_client(retry_script) + + def begin(): + client.hooks.emit_completion_arguments(model="test-model") + return trace_api.get_current_span().get_span_context().span_id + + def complete(expected): + assert trace_api.get_current_span().get_span_context().span_id == expected + client.hooks.emit_completion_response(response()) + return Answer(answer="final") + + def stream(*args, **kwargs): + expected = begin() + yield Answer(answer="partial") + yield complete(expected) + + async def async_stream(*args, **kwargs): + expected = begin() + yield Answer(answer="partial") + await asyncio.sleep(0) + yield complete(expected) + + async def consume(): + return [ + partial async for partial in agent_spans._trace_agent_async_stream("run_async_stream", async_stream)(agent(client)) + ] + + if asynchronous: + assert len(asyncio.run(consume())) == 2 + else: + assert len(list(agent_spans._trace_agent_stream("run_stream", stream)(agent(client)))) == 2 + child, parent = finished_spans(exporter, 2) + assert child.status.status_code == StatusCode.OK + assert parent.status.status_code == StatusCode.OK + + +def test_validation_text_truncation_and_extra_hook_keywords(exporter): + installed = installation._current_installation() + installation.install(tracer_provider=installed.tracer_provider, config=TracingConfig(max_attribute_length=4)) + + def script(hooks, kwargs): + hooks.emit_completion_arguments(**kwargs) + llm_attempt_spans._parse_error(ValueError("long error"), future_option=True) + llm_attempt_spans._completion_response(response(), future_option=True) + llm_attempt_spans._completion_usage(usage(), attempt_number=1, future_option=True) + + run_agent(scripted_client(script)) + child, parent = finished_spans(exporter, 2) + assert parent.attributes["atomic_agents.llm.last_validation_error"] == "long...[truncated]" + assert parent.attributes["atomic_agents.llm.last_validation_error.original_length"] == 10 From 0d05928a2226db494413c8fd0d3e3e0f35c1190d Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 12:20:31 +0200 Subject: [PATCH 11/25] Keep hook Protocol declarations compatible with Black and Flake8 Both style checks pass for the hook module. Co-Authored-By: Claude Code --- .../atomic_agents/observability/llm_attempt_spans.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/atomic-agents/atomic_agents/observability/llm_attempt_spans.py b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py index 3dc7f4c2..05bda66b 100644 --- a/atomic-agents/atomic_agents/observability/llm_attempt_spans.py +++ b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py @@ -37,9 +37,11 @@ class _Response(Protocol): class _HookClient(Protocol): - def on(self, event: str, handler: Callable[..., None]) -> object: ... + def on(self, event: str, handler: Callable[..., None]) -> object: + """Append a handler without replacing application hooks.""" - def off(self, event: str, handler: Callable[..., None]) -> object: ... + def off(self, event: str, handler: Callable[..., None]) -> object: + """Remove only this handler, preserving application hooks.""" @dataclass From 26c47cb5caf7d7e6b341e4443213647040cecdcf Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 12:27:29 +0200 Subject: [PATCH 12/25] Rewrite the observability guide around generic OpenTelemetry Phoenix becomes one backend section. Adds a backends list, attribute tables (GenAI, atomic_agents, optional OpenInference), LLM attempt spans, naming and conversations, privacy and size, and re-instrumentation. README gets an Observability section and the phoenix example README an install line. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 6 +- atomic-examples/phoenix-tracing/README.md | 19 +- docs/guides/observability.md | 476 ++++++++++++++++------ 3 files changed, 378 insertions(+), 123 deletions(-) diff --git a/README.md b/README.md index 1db7d87e..408bc5e0 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,7 @@ For full, runnable examples, please refer to the following files in the `atomic- In addition to the quickstart examples, we have more complex examples demonstrating the power of Atomic Agents: - [Hooks System](/atomic-examples/hooks-example/README.md): Comprehensive demonstration of the AtomicAgent hook system for monitoring, error handling, and performance metrics with intelligent retry mechanisms. -- [Phoenix Tracing](/atomic-examples/phoenix-tracing/README.md): Traces agents and tools in Arize Phoenix through the optional OpenInference instrumentor (`pip install "atomic-agents[observability]"`). +- [Phoenix Tracing](/atomic-examples/phoenix-tracing/README.md): Traces agents and tools in Arize Phoenix with the OpenTelemetry instrumentor (`pip install "atomic-agents[phoenix]"`). - [Basic Multimodal](/atomic-examples/basic-multimodal/README.md): Demonstrates how to analyze images with text, focusing on extracting structured information from nutrition labels using GPT-4 Vision capabilities. - [Deep Research](/atomic-examples/deep-research/README.md): An advanced example showing how to perform deep research tasks. - [Orchestration Agent](/atomic-examples/orchestration-agent/README.md): Shows how to create an Orchestrator Agent that intelligently decides between using different tools (search or calculator) based on user input. @@ -298,6 +298,10 @@ In addition to the quickstart examples, we have more complex examples demonstrat For a complete list of examples, see the [examples directory](/atomic-examples/). +## Observability + +Every agent and tool run is an OpenTelemetry span. Works with any OpenTelemetry backend (Grafana, Sentry, Phoenix, Langfuse, ...), including Datadog, Honeycomb, Jaeger or a plain OTLP collector. Install it with `pip install "atomic-agents[observability]"` and see the [Observability guide](docs/guides/observability.md). + ## AI-Assisted Development Building with an AI coding assistant? Atomic Agents ships first-class support so your assistant knows the framework's current API and conventions instead of guessing. diff --git a/atomic-examples/phoenix-tracing/README.md b/atomic-examples/phoenix-tracing/README.md index 077801da..24bc3da5 100644 --- a/atomic-examples/phoenix-tracing/README.md +++ b/atomic-examples/phoenix-tracing/README.md @@ -6,17 +6,30 @@ produces this tree in Phoenix: ``` CHAIN answer_question -├── AGENT AtomicAgent[BasicChatInputSchema, CalculationPlanSchema].run +├── AGENT invoke_agent {agent name} +│ ├── LLM chat {model} (Atomic Agents attempt span, one per Instructor attempt) │ └── LLM ChatCompletion (from openinference-instrumentation-openai) -├── TOOL CalculatorTool -└── AGENT AtomicAgent[CalculationResultSchema, BasicChatOutputSchema].run +├── TOOL execute_tool {tool name} +└── AGENT invoke_agent {agent name} + ├── LLM chat {model} └── LLM ChatCompletion ``` +Running the OpenAI instrumentor next to the Atomic Agents attempt spans gives two LLM layers. To keep one, pass +`TracingConfig(llm_spans=False)` when you call `AtomicAgentsInstrumentor().instrument(...)` yourself. + The AGENT and TOOL spans come from `atomic_agents.observability`, which Phoenix activates automatically through `register(auto_instrument=True)`. The LLM spans come from the OpenAI instrumentor. The CHAIN span is the only manual tracing in `main.py`; without it every agent and tool call would be its own trace. +## Install + +Inside this repository `uv sync` installs everything. In your own project: + +```bash +pip install "atomic-agents[phoenix]" arize-phoenix-otel openinference-instrumentation-openai +``` + ## Run it 1. Start Phoenix locally (or point `PHOENIX_COLLECTOR_ENDPOINT` at a hosted instance): diff --git a/docs/guides/observability.md b/docs/guides/observability.md index 1bcff648..cc1ebf84 100644 --- a/docs/guides/observability.md +++ b/docs/guides/observability.md @@ -1,69 +1,148 @@ -# Observability with Arize Phoenix and OpenTelemetry +# Observability with OpenTelemetry -Atomic Agents ships an optional [OpenInference](https://github.com/Arize-ai/openinference) instrumentor. Once it -is installed, every `AtomicAgent` run and every `BaseTool` run becomes a span that -[Arize Phoenix](https://github.com/Arize-ai/phoenix), Arize AX, Langfuse, or any other OpenTelemetry backend can -display. Nothing changes for applications that do not opt in. +Atomic Agents can turn every agent run, tool run and LLM attempt into an OpenTelemetry span. The spans follow the +OpenTelemetry GenAI semantic conventions, so they show up on any OpenTelemetry backend: a plain OTLP collector, +Grafana Tempo, Sentry, Jaeger, Honeycomb, Datadog, Arize Phoenix, Langfuse and so on. Only `opentelemetry-api` is +required. Nothing changes for applications that do not opt in. -## What you get +You get three kinds of spans: -| Call | Span | Attributes | -|------|------|------------| -| `agent.run`, `run_async`, `run_stream`, `run_async_stream` | `AGENT`, named `AtomicAgent[InputSchema, OutputSchema].run` (a subclass reports its own name) | `agent.name`, `input.value` / `output.value` as JSON, `atomic_agents.model`, `atomic_agents.mode`, `atomic_agents.input_schema`, `atomic_agents.output_schema`, `atomic_agents.history.message_count` (history size entering the turn), `atomic_agents.context_providers`, `atomic_agents.system_prompt.sha256` (hash of the static prompt sections: background, steps, output instructions) | -| `tool.run`, `run_async`, `arun` on any `BaseTool` subclass, including MCP tools | `TOOL`, named after the tool class | `tool.name`, `tool.description`, `tool.parameters` (the input JSON schema), `input.value` / `output.value` as JSON | +- **AGENT**: one per `AtomicAgent` run (`run`, `run_async`, `run_stream`, `run_async_stream`). +- **TOOL**: one per `BaseTool` run, including MCP tools. +- **LLM attempt**: one per call Instructor makes to the model, so retries after a validation error are visible. -Spans carry `session.id`, `user.id`, `metadata` and tags when you wrap calls in the OpenInference context -managers (`using_session`, `using_user`, `using_metadata`, `using_attributes`). Exceptions set the span status to -`ERROR` and are recorded as span events. Streaming spans record the final response when the stream completes, -or the last partial plus `atomic_agents.stream.closed_early = true` when the consumer stops early or the task -is cancelled (a cancelled stream is not reported as an error). +## Install -LLM spans are not produced by Atomic Agents. Install exactly one LLM instrumentor for the provider SDK you use -(`openinference-instrumentation-openai`, `-anthropic`, `-litellm`, ...). Installing both a provider instrumentor and -`openinference-instrumentation-instructor` gives you two layers of LLM spans. Because the agent span is the current -span while the provider call runs, the LLM span (with prompts, tokens and the model name) nests under it. Instructor -retries show up as several LLM spans under one agent span. +```bash +pip install "atomic-agents[observability]" opentelemetry-sdk opentelemetry-exporter-otlp-proto-http +``` -## Install +The `observability` extra installs `opentelemetry-api` and nothing else. You bring the SDK and an exporter for your +backend. The `phoenix` extra adds the OpenInference packages: ```bash -pip install "atomic-agents[observability]" arize-phoenix-otel openinference-instrumentation-openai +pip install "atomic-agents[phoenix]" ``` -The `observability` extra installs `openinference-instrumentation`, `openinference-semantic-conventions` and -`opentelemetry-api`. The Phoenix path below works without the extra because `arize-phoenix-otel` already depends on -those packages; the manual path needs the extra plus an OpenTelemetry SDK, an exporter, and the instrumentor for -your LLM client (the example below uses the OpenAI one): +With those installed, Atomic Agents also writes the OpenInference attribute family (turned on automatically) and the +OpenInference context managers `using_session`, `using_user` and `using_metadata` start to work. Backends that read +OpenInference attributes, such as Arize Phoenix, use them for their own views. Every other backend works from the GenAI +attributes alone. + +## Setup with any OpenTelemetry backend (manual) + +```python +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + +from atomic_agents.observability import AtomicAgentsInstrumentor, TracingConfig + +tracer_provider = TracerProvider() +tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + +AtomicAgentsInstrumentor().instrument(tracer_provider=tracer_provider, config=TracingConfig()) +``` + +`OTLPSpanExporter()` reads the standard `OTEL_EXPORTER_OTLP_*` environment variables, so this code stays the same for +every backend below. Only the environment changes. `tracer_provider` and `config` are both optional: without a provider, +the global one is used, and without a config you get the defaults described under "Privacy and size". + +Call `instrument()` once, at import time or in your service's startup hook, before any agent runs. + +## Backends + +The GenAI attributes are the OpenTelemetry standard, and backends with LLM views (Datadog LLM Observability, Langfuse, +Phoenix, Sentry AI monitoring, PostHog) read them. Every recipe below uses the setup above. Only the exporter +configuration differs. + +What has actually been run: the span output is tested offline against an in-memory exporter, and the Phoenix recipe is +the [phoenix-tracing example](https://github.com/eigenwise/atomic-agents/tree/main/atomic-examples/phoenix-tracing). +Everything marked "untested" follows the vendor's documented OTLP settings but has not been run against a live account. +Check the vendor's page before relying on it. + +**Plain OTLP collector** (untested against a live collector): ```bash -pip install "atomic-agents[observability]" opentelemetry-sdk opentelemetry-exporter-otlp-proto-http openinference-instrumentation-openai +export OTEL_SERVICE_NAME=my-app +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf ``` -## Setup with Phoenix (automatic) +**Grafana Tempo** (untested): send OTLP to Tempo directly or through a collector or Grafana Alloy. Self-hosted, point the +endpoint at Tempo's OTLP HTTP port (4318 by default). Grafana Cloud uses its OTLP gateway plus a basic-auth header built +from your instance ID and an access token: + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp-gateway-.grafana.net/otlp +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic%20" +``` + +**Jaeger** (untested): current Jaeger releases accept OTLP directly. + +```bash +docker run --rm -p 16686:16686 -p 4318:4318 jaegertracing/all-in-one +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +``` + +**Sentry** (follows Sentry's Python OTLP integration docs, untested here). The integration sets up the exporter itself, +so you do not build a `TracerProvider` and do not set `traces_sample_rate`: + +```bash +pip install "atomic-agents[observability]" "sentry-sdk[opentelemetry-otlp]" +``` ```python -from phoenix.otel import register +import sentry_sdk +from sentry_sdk.integrations.otlp import OTLPIntegration -tracer_provider = register(project_name="my-app", auto_instrument=True) +from atomic_agents.observability import AtomicAgentsInstrumentor + +sentry_sdk.init(dsn="https://@o.ingest.sentry.io/", integrations=[OTLPIntegration()]) +AtomicAgentsInstrumentor().instrument() ``` -`register(auto_instrument=True)` discovers the Atomic Agents instrumentor through the `openinference_instrumentor` -entry point and activates it together with every other OpenInference instrumentor in the environment. Do this once, -at import time or in your service's startup hook, before any agent runs. Phoenix reads `PHOENIX_COLLECTOR_ENDPOINT`, -`PHOENIX_PROJECT` (alias `PHOENIX_PROJECT_NAME`) and `PHOENIX_API_KEY` from the environment. With no endpoint set, -`register()` exports over gRPC to `localhost:4317`; set `PHOENIX_COLLECTOR_ENDPOINT=http://localhost:6006` to send -HTTP to the port the Phoenix UI serves on. +Sentry's OTLP ingest drops span events. The `validation_error` event and recorded exceptions do not arrive there, while +`error.type`, the span status and `atomic_agents.llm.last_validation_error` do. Whether Sentry's AI Agents dashboard +fills from these spans is unverified. -Two things to know about the automatic path: +**Honeycomb** (untested): use `https://api.eu1.honeycomb.io` for the EU region. Honeycomb's Agent Timeline groups by +`gen_ai.conversation.id`, so wrap calls in `using_conversation` (see "Naming and conversations"). -- Phoenix imports `atomic_agents` to load the entry point. That import is the same as any `import atomic_agents` in - your application, including the Instructor streaming patch the framework applies at import. -- Existing `auto_instrument=True` users start exporting agent and tool payloads when they upgrade to a version with the - instrumentor. Use the privacy controls below if that is not wanted. +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=https://api.honeycomb.io +export OTEL_EXPORTER_OTLP_HEADERS="x-honeycomb-team=" +``` + +**Datadog** (untested): send OTLP to a Datadog Agent with OTLP ingestion enabled (HTTP on 4318 by default) and set +`OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318`. Datadog LLM Observability maps `invoke_agent` spans to agents, +`execute_tool` to tools and `chat` to LLM calls, and reads the `gen_ai.usage.*` tokens and `gen_ai.conversation.id`. +It reads prompt and response text only from `gen_ai.input.messages` and `gen_ai.output.messages`, which Atomic Agents +does not emit, so you see spans, tokens and sessions but not the agent text. PostHog behaves the same way. + +**Arize Phoenix**: see "Setup with Phoenix" below. + +**Langfuse** (untested): Langfuse takes OTLP at `/api/public/otel` with basic auth built from your public and secret +key. Its default filter keeps spans that carry `gen_ai.*` attributes, which these spans do. + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=https://cloud.langfuse.com/api/public/otel +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic%20" +``` + +**PostHog** (untested): PostHog AI Observability ingests OTLP over HTTP only (protobuf or JSON, no gRPC). Use the +`TRACES` variables, because the generic endpoint variable appends `/v1/traces` and gets a 404. Use your project's client +API host in the URL, see [PostHog's OpenTelemetry page](https://posthog.com/docs/ai-observability/installation/opentelemetry). +`chat` spans become `$ai_generation` events and the others become `$ai_span`. + +```bash +export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="/i/v0/ai/otel" +export OTEL_EXPORTER_OTLP_TRACES_HEADERS="Authorization=Bearer%20" +``` ## Setup with the OpenTelemetry launcher (automatic) -Install the optional tracing adapter, launcher, SDK, and exporter explicitly: +Install the tracing adapter, launcher, SDK and exporter explicitly: ```bash pip install "atomic-agents[observability]" opentelemetry-distro opentelemetry-exporter-otlp-proto-http @@ -75,115 +154,271 @@ Run your existing application with environment-only configuration: export OTEL_SERVICE_NAME=my-app export OTEL_TRACES_EXPORTER=otlp export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:6006 +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 export OTEL_METRICS_EXPORTER=none export OTEL_LOGS_EXPORTER=none -export OPENINFERENCE_HIDE_INPUTS=true -export OPENINFERENCE_HIDE_OUTPUTS=true +export ATOMIC_AGENTS_TRACE_HIDE_INPUTS=true +export ATOMIC_AGENTS_TRACE_HIDE_OUTPUTS=true opentelemetry-instrument python app.py ``` -The launcher discovers `atomic_agents` through the `opentelemetry_instrumentor` entry point and activates the -same adapter used by Phoenix and manual setup. Without the OpenInference dependencies, the loader skips the adapter. -`opentelemetry-bootstrap` does not automatically install Atomic Agents or its observability extra. +The launcher discovers `atomic_agents` through the `opentelemetry_instrumentor` entry point and activates the same +adapter as the manual setup. Without `opentelemetry-api` installed, the loader skips the adapter. +`opentelemetry-bootstrap` does not automatically install Atomic Agents or its extras. -To disable this adapter, set `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=atomic_agents` (or add `atomic_agents` to an -existing comma-separated list). Ordinary imports without a launcher or explicit setup remain uninstrumented. +To disable the adapter, set `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=atomic_agents` (or add `atomic_agents` to an +existing comma-separated list). Ordinary imports without a launcher or explicit setup stay uninstrumented. -Choose one provider configuration path: the launcher, Phoenix registration, or manual setup. Both discovery entry -points share one installation, so activating both does not duplicate Atomic Agents spans. The first provider/config -wins; a later call with different settings warns and keeps the existing installation. OpenTelemetry also keeps its -first global provider. Calling Phoenix registration after launcher startup therefore does not switch the export -destination. To combine explicit activations, configure a shared provider, or uninstrument before changing the -Atomic Agents provider/config as described below. +Choose one provider configuration path: the launcher, Phoenix registration, or manual setup. The discovery entry points +share one installation, so activating several does not duplicate spans. OpenTelemetry itself keeps its first global +provider, so calling Phoenix registration after launcher startup does not switch where spans are exported. See +"Re-instrumentation" for what a second `instrument()` call does. -The launcher adds AGENT and TOOL spans, with no parent CHAIN or provider LLM spans of its own. Install exactly one -instrumentor for your LLM provider and activate it through its supported setup. Use the grouping example below when -you want several agent/tool calls in one trace. +The launcher adds AGENT, TOOL and LLM attempt spans and no parent span. Use the grouping example below when you want +several agent and tool calls in one trace. -## Setup with any OpenTelemetry backend (manual) +## Setup with Phoenix (automatic) ```python -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -from openinference.instrumentation import OITracer, TraceConfig -from openinference.instrumentation.openai import OpenAIInstrumentor +from phoenix.otel import register -from atomic_agents.observability import AtomicAgentsInstrumentor +tracer_provider = register(project_name="my-app", auto_instrument=True) +``` -tracer_provider = TracerProvider() -tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter("http://localhost:6006/v1/traces"))) -trace_config = TraceConfig() +`register(auto_instrument=True)` discovers the Atomic Agents instrumentor through the `openinference_instrumentor` entry +point and activates it together with every other OpenInference instrumentor in the environment. Install +`atomic-agents[phoenix]` (or `arize-phoenix-otel`, which brings the same packages) and do this once, at import time or in +your startup hook, before any agent runs. Phoenix reads `PHOENIX_COLLECTOR_ENDPOINT`, `PHOENIX_PROJECT` (alias +`PHOENIX_PROJECT_NAME`) and `PHOENIX_API_KEY` from the environment. With no endpoint set, `register()` exports over gRPC +to `localhost:4317`; set `PHOENIX_COLLECTOR_ENDPOINT=http://localhost:6006` to send HTTP to the port the Phoenix UI +serves on. -AtomicAgentsInstrumentor().instrument(tracer_provider=tracer_provider, config=trace_config) -OpenAIInstrumentor().instrument(tracer_provider=tracer_provider, config=trace_config) -tracer = OITracer(tracer_provider.get_tracer(__name__), config=trace_config) -``` +Two things to know about the automatic path: -The last line matters for the grouping section below: a plain SDK provider hands out a plain OpenTelemetry tracer, -and the `openinference_span_kind` argument, `set_input` and `set_output` only exist on an OpenInference `OITracer`. -Phoenix's `register()` returns a provider whose tracers already are `OITracer`s, so the automatic path skips this -step. Pass the same `TraceConfig` to every layer; the config given to `instrument()` configures only the Atomic Agents -spans. +- Phoenix imports `atomic_agents` to load the entry point. That import is the same as any `import atomic_agents` in + your application, including the Instructor streaming patch the framework applies at import. +- Existing `auto_instrument=True` users start exporting agent and tool payloads when they upgrade to a version with the + instrumentor. Use the privacy controls below if that is not wanted. -`instrument()` is idempotent. Calling it again with a different provider or config logs a warning and keeps the first -installation; call `uninstrument()` first to change it. `uninstrument()` restores the original methods on -`AtomicAgent` and on every tool class. +Langfuse and other OpenInference-aware backends use the same OpenInference attributes, so the Phoenix-style setup, or +the manual setup with the `phoenix` extra installed, works for them too. -## Grouping agent and tool calls into one trace +## What you get -Atomic Agents has no tool-calling loop: your code calls `agent.run(...)`, then `tool.run(...)`, then another agent. -Each of those is a root span, so without a parent they become separate traces. Open one parent span per unit of work: +| Call | Span name | Kind | +|------|-----------|------| +| `AtomicAgent.run`, `run_async`, `run_stream`, `run_async_stream` | `invoke_agent {agent name}` | AGENT | +| `run`, `run_async`, `arun` on any `BaseTool` subclass | `execute_tool {tool name}` | TOOL | +| Each model call Instructor makes during an agent run | `chat {request model}` | LLM attempt | + +The agent name is `AgentConfig.name`, or the class name when you do not set one. The tool name is the name the model +sees (`tool_name` on the tool). Spans attach to the current context and never start a new trace when a parent span +exists, so your own request or job span becomes their parent. An agent span is the parent of its LLM attempt spans, and +a tool span nests under whatever span is current when the tool runs. A span that fails gets +status `ERROR`, the exception recorded, and `error.type` set to the exception class name. + +Attributes, GenAI keys first. All of them are written whether or not the OpenInference packages are installed. + +| Key | On | Value | +|-----|----|-------| +| `gen_ai.operation.name` | all | `invoke_agent`, `execute_tool` or `chat` | +| `gen_ai.agent.name` | AGENT | Agent name | +| `gen_ai.tool.name` | TOOL | Tool name the model sees | +| `gen_ai.tool.description` | TOOL | Tool description | +| `gen_ai.tool.type` | TOOL | `function` | +| `gen_ai.tool.call.arguments` | TOOL | Tool input as JSON (hidden by `hide_inputs`, size-capped) | +| `gen_ai.tool.call.result` | TOOL | Tool output as JSON (hidden by `hide_outputs`, size-capped) | +| `gen_ai.request.model` | AGENT, LLM | Model the agent was configured with, or the model in the completion call | +| `gen_ai.provider.name` | AGENT, LLM | Instructor provider mapped to the OpenTelemetry value, for example `azure.ai.openai`, `gcp.vertex_ai`, `aws.bedrock`; `openai` and `anthropic` pass through | +| `gen_ai.response.model` | AGENT, LLM | Model the provider reported; on the AGENT span, the last response's model | +| `gen_ai.usage.input_tokens` | AGENT, LLM | Input tokens (AGENT: total across attempts) | +| `gen_ai.usage.output_tokens` | AGENT, LLM | Output tokens | +| `gen_ai.usage.cache_read.input_tokens` | AGENT, LLM | Cached input tokens, when the provider reports them | +| `gen_ai.usage.reasoning.output_tokens` | AGENT, LLM | Reasoning tokens, when the provider reports them | +| `gen_ai.conversation.id` | all | Set by `using_conversation` | +| `error.type` | failed spans | Exception class name | + +Atomic Agents' own keys: + +| Key | On | Value | +|-----|----|-------| +| `atomic_agents.agent.class` | AGENT | For example `AtomicAgent[InputSchema, OutputSchema]` (a subclass reports its own class) | +| `atomic_agents.tool.class` | TOOL | Tool class name | +| `atomic_agents.model` | AGENT | Same as the request model | +| `atomic_agents.mode` | AGENT | Instructor mode | +| `atomic_agents.input_schema`, `atomic_agents.output_schema` | AGENT | Schema class names | +| `atomic_agents.history.message_count` | AGENT | History size entering the turn | +| `atomic_agents.context_providers` | AGENT | Registered context provider names | +| `atomic_agents.system_prompt.sha256` | AGENT | Hash of the static prompt sections (background, steps, output instructions). Context provider output is not included, so it identifies the prompt version even when inputs are hidden | +| `atomic_agents.input`, `atomic_agents.output` | AGENT | Agent input and output as JSON (hidden and size-capped as described under "Privacy and size"). A stream records the final response, or the last partial one | +| `atomic_agents.stream.closed_early` | AGENT | `true` when a stream consumer stopped early or the task was cancelled (not reported as an error) | +| `atomic_agents.llm.attempts` | AGENT | Number of model calls in the run | +| `atomic_agents.llm.last_validation_error` | AGENT | Text of the last validation error | +| `atomic_agents.llm.attempt` | LLM | 1-based attempt number | + +OpenInference keys, optional. These are written only when the OpenInference packages are installed (the `phoenix` +extra), or when you force them with `TracingConfig(openinference_attributes=True)`; set it to `False` to drop them even +when the packages are present. + +| Key | On | Value | +|-----|----|-------| +| `openinference.span.kind` | all | `AGENT`, `TOOL` or `LLM` | +| `input.value`, `input.mime_type` | AGENT, TOOL | Same payload as `atomic_agents.input` or `gen_ai.tool.call.arguments` | +| `output.value`, `output.mime_type` | AGENT, TOOL | Same payload as `atomic_agents.output` or `gen_ai.tool.call.result` | +| `agent.name` | AGENT | Agent name | +| `tool.name`, `tool.description` | TOOL | Same as the GenAI tool keys | +| `tool.parameters` | TOOL | The input JSON schema (size-capped, not hidden) | +| `session.id` | all | The conversation id, plus anything from `using_session`, `using_user` and `using_metadata` | + +## LLM attempt spans + +Instructor drives the model call and its retry loop, so the detail of each attempt only exists at the Atomic Agents +layer. The instrumentor registers hooks on the agent's Instructor client (once per client, on the first instrumented +run) and removes them again in `uninstrument()`: + +- `completion:kwargs` starts the `chat {model}` span as a child of the agent span and counts the attempt. +- `completion:response` sets `gen_ai.response.model` and the token usage on that span and ends it with status OK. +- `completion:error` records the exception, sets `error.type` and ends the span with status `ERROR`. +- `parse:error` adds a `validation_error` event (with `atomic_agents.llm.attempt`) to the AGENT span and sets + `atomic_agents.llm.last_validation_error`. Instructor then retries, which opens the next attempt span. +- `completion:usage` writes the running totals to the AGENT span: the four `gen_ai.usage.*` keys and + `atomic_agents.llm.attempts`. + +An attempt is one call to the model. A retry is any attempt after the first, normally because the response failed +validation, so two attempt spans under one agent span means one retry. Hooks you registered on the same client yourself +keep running next to these, because Instructor keeps a list of handlers per event. If the agent span ends while an +attempt is still open, the attempt span is ended with an unset status. The hooks never raise into your code. + +OpenTelemetry's GenAI guidance says an agent framework should not emit model-call spans by default when the LLM library +already does. Atomic Agents emits them by default because the attempt and validation detail comes from Instructor, which +no provider instrumentor sees. If you also run a provider instrumentor (`openinference-instrumentation-openai` and +friends), turn the attempt spans off or you get two LLM layers: ```python -with tracer.start_as_current_span("answer_question", openinference_span_kind="chain") as turn: - turn.set_input(question) - plan = planner.run(BasicChatInputSchema(chat_message=question)) - calculation = calculator.run(CalculatorToolInputSchema(expression=plan.expression)) - answer = answerer.run(CalculationResultSchema(question=question, expression=plan.expression, result=calculation.result)) - turn.set_output(answer.chat_message) +AtomicAgentsInstrumentor().instrument(tracer_provider=tracer_provider, config=TracingConfig(llm_spans=False)) +``` + +With `llm_spans=False` the usage totals and attempt count still land on the AGENT span, and the `validation_error` event +still fires. + +## Naming and conversations + +Give each agent a stable name, so spans group the same way across deployments and class renames: + +```python +agent = AtomicAgent[InputSchema, OutputSchema]( + config=AgentConfig(client=client, model="gpt-5-mini", name="orchestrator_planning") +) ``` -`tracer` is the `OITracer` from the manual setup, or `tracer_provider.get_tracer(__name__)` on the Phoenix path. The -[phoenix-tracing example](https://github.com/eigenwise/atomic-agents/tree/main/atomic-examples/phoenix-tracing) -is exactly this flow, and the test suite runs the manual setup plus this grouping offline with an in-memory exporter. +The span becomes `invoke_agent orchestrator_planning`. Without a name, the class name is used. -## Sessions, users and metadata +Group related runs into a conversation with `using_conversation`: ```python -from openinference.instrumentation import using_session, using_user, using_metadata +from atomic_agents.observability import using_conversation -with using_session(session_id=conversation_id), using_user(user_id=user.id), using_metadata({"tenant": tenant}): +with using_conversation(conversation_id): response = agent.run(user_message) ``` -Phoenix groups spans that share a `session.id` into a conversation view. For a streaming endpoint, put the context -manager inside the generator that yields the chunks, not around the endpoint function; the generator runs after the -endpoint returned, so a context manager around the endpoint body no longer applies. +Every AGENT, TOOL and LLM attempt span started inside the block gets `gen_ai.conversation.id` (and `session.id` when +OpenInference keys are on). With the `phoenix` extra installed, `using_session`, `using_user` and `using_metadata` from +`openinference.instrumentation` keep working, and a `using_session` value is copied to `gen_ai.conversation.id` when you +have not set one with `using_conversation`. + +For a streaming endpoint, put the context manager inside the generator that yields the chunks, not around the endpoint +function. The generator runs after the endpoint returned, so a context manager around the endpoint body no longer +applies. + +## Privacy and size -## Privacy +By default, agent and tool payloads are exported. `TracingConfig` controls that: -Agent and tool payloads are exported as `input.value` / `output.value`. To redact them: +```python +TracingConfig(hide_inputs=True, hide_outputs=True, max_attribute_length=2048) +``` -- On the automatic Phoenix or OpenTelemetry launcher path, set `OPENINFERENCE_HIDE_INPUTS=true` and/or - `OPENINFERENCE_HIDE_OUTPUTS=true`. `TraceConfig()` reads these variables before the first installation. -- On the manual path, pass `TraceConfig(hide_inputs=True, hide_outputs=True)` to `instrument()`. +| Field | Default | Effect | +|-------|---------|--------| +| `hide_inputs` | `False` | Input payload values become `__REDACTED__` | +| `hide_outputs` | `False` | Output payload values become `__REDACTED__` | +| `max_attribute_length` | `4096` | Longer values are cut to this many characters and end in `...[truncated]` | +| `openinference_attributes` | auto | See "What you get" | +| `llm_spans` | `True` | See "LLM attempt spans" | + +When no config is passed to `instrument()`, the settings come from the environment. `ATOMIC_AGENTS_TRACE_HIDE_INPUTS` +and `ATOMIC_AGENTS_TRACE_HIDE_OUTPUTS` are read first, then `OPENINFERENCE_HIDE_INPUTS` and `OPENINFERENCE_HIDE_OUTPUTS` +as a fallback (`true` or `1`). `ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH` sets the size cap. If you pass your own +`TracingConfig(...)`, the environment is not read; use `TracingConfig.from_environment()` to start from it. An +OpenInference `TraceConfig` is also accepted for `config`, for its `hide_inputs` and `hide_outputs` settings. + +A truncated value gets a companion attribute with the original size, for example `atomic_agents.input.original_length`. +Hiding wins over truncation, and a hidden payload is never serialized in the first place. + +Covered by hiding and truncation: `atomic_agents.input`, `atomic_agents.output`, `input.value`, `output.value`, +`gen_ai.tool.call.arguments`, `gen_ai.tool.call.result` and `atomic_agents.llm.last_validation_error`. `tool.parameters` +is truncated but never hidden. Status descriptions and recorded exception messages are truncated too, and under +`hide_outputs` a Pydantic validation error message is replaced by `__REDACTED__` plus the error count, because the raw +model output is quoted inside it. + +Not covered: tool names and descriptions, the system prompt hash, and whatever you put in `using_metadata`. A provider +instrumentor and your own tracer apply their own settings, so configure them the same way if you want matching masking. +Multimodal inputs (images, audio, PDFs) are serialized into the input payload in full, so `max_attribute_length` is the +control for those. + +## Grouping agent and tool calls into one trace -Redacted values are exported as `__REDACTED__`. Masking covers the `input.value` and `output.value` attributes of -Atomic Agents spans and nothing else: exception events and status descriptions (a validation error can quote input -values), tool names and descriptions, and whatever you put in `using_metadata` are exported as they are. The LLM -instrumentor and your own `OITracer` each take their own `TraceConfig`, so pass the same one to all three if you want -the same masking everywhere. `atomic_agents.system_prompt.sha256` stays available either way. It -hashes the static sections of a `SystemPromptGenerator` (background, steps, output instructions) and ignores what -context providers add per call, so it identifies the prompt version even when inputs are hidden. Multimodal inputs (images, audio, PDFs) are serialized into -`input.value` in full; cap attribute size with `OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT` if that is too much. +Atomic Agents has no tool-calling loop: your code calls `agent.run(...)`, then `tool.run(...)`, then another agent. +Each of those is a root span, so without a parent they become separate traces. Open one parent span per unit of work +with a plain OpenTelemetry tracer. It also answers "where do the seconds go": add child spans for retrieval or database +work next to the agent and tool calls. + +```python +tracer = tracer_provider.get_tracer(__name__) + +with tracer.start_as_current_span("answer_question"): + with tracer.start_as_current_span("retrieve_documents"): + documents = search_index(question) + plan = planner.run(BasicChatInputSchema(chat_message=question)) + calculation = calculator.run(CalculatorToolInputSchema(expression=plan.expression)) + answer = answerer.run(CalculationResultSchema(question=question, expression=plan.expression, result=calculation.result)) +``` + +The agent, tool and attempt spans become children of `answer_question`, and the slowest step stands out in the +waterfall. The [phoenix-tracing example](https://github.com/eigenwise/atomic-agents/tree/main/atomic-examples/phoenix-tracing) +is this flow. On the Phoenix path the provider's tracers are OpenInference `OITracer`s, which also take +`openinference_span_kind="chain"` and `set_input`/`set_output` if you want Phoenix to label the parent as a CHAIN. + +## Re-instrumentation + +`instrument()` with the same provider and config as the active installation does nothing. With a different provider or +config it replaces them in place and logs a WARNING that says so. Spans already in flight finish on the provider they +started with; new spans use the new one. `uninstrument()` followed by `instrument()` is the clean way to change +settings, and `uninstrument()` restores the original methods on `AtomicAgent` and on every tool class. + +In a test suite, call `instrument(tracer_provider=provider)` with a fresh in-memory provider in each test. The replace +behavior means you do not need to uninstrument between tests, and the warning is expected there: + +```python +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +exporter = InMemorySpanExporter() +provider = TracerProvider() +provider.add_span_processor(SimpleSpanProcessor(exporter)) +AtomicAgentsInstrumentor().instrument(tracer_provider=provider) +agent.run(user_input) +assert [span.name for span in exporter.get_finished_spans()][-1] == "invoke_agent my_agent" +``` ## Services and async -- Call `register(...)` (or `instrument(...)`) at import or in the startup hook so every worker process is instrumented. -- Use `register(batch=True)` (or a `BatchSpanProcessor`) in services so span export does not block request handling. -- Concurrent `run_async` calls, including `asyncio.gather` over several agents, get independent spans. +- Call `instrument(...)` (or `register(...)`) at import or in the startup hook so every worker process is instrumented. +- Use a `BatchSpanProcessor` (or `register(batch=True)`) in services so span export does not block request handling. +- Concurrent `run_async` calls, including `asyncio.gather` over several agents sharing one client, get independent + spans and their own attempt spans. - Streaming token counts need `model_api_parameters={"stream_options": {"include_usage": True}}` for OpenAI. ## Limitations @@ -191,12 +426,15 @@ context providers add per call, so it identifies the prompt version even when in - Methods bound before instrumentation stay untraced: `callback = agent.run` taken before `instrument()` keeps calling the original. - An `AtomicAgent` subclass that overrides `run` (or a stream method) without calling `super()` bypasses the wrapper - and emits no span; add `@tracer.agent` to such overrides yourself. -- Tool methods that are async generators are left untraced. + and emits no span. Open a span in the override yourself, with `tracer.start_as_current_span(...)` or `@tracer.agent` + on an `OITracer`. +- Tool methods that are async generators are left untraced. Open a span inside the generator body with + `tracer.start_as_current_span(...)`, or use `@tracer.tool` on an `OITracer`. - A plain `ThreadPoolExecutor` starts its worker with an empty context, so a tool that submits work to one loses the current span and the re-entrancy guard inside the pool. Submit through a copied context instead, as the forge tools do: `executor.submit(contextvars.copy_context().run, asyncio.run, self.run_async(params))`. `asyncio.to_thread` and `asyncio.run` already copy the context. - The AGENT span records the final (or last partial) response of a stream, not every chunk. -- The system prompt text is not on the AGENT span; it is the first message on the LLM span. The AGENT span carries its - SHA-256 instead. +- The system prompt text is not on any span. The AGENT span carries its SHA-256 instead. +- Datadog and PostHog read prompt and response text from `gen_ai.input.messages` and `gen_ai.output.messages`, which + are not emitted. From 6ecf30aaaf98b200dfbdc335b04e8bd2a0f2cccc Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 12:33:22 +0200 Subject: [PATCH 13/25] Serialize Instructor hook registration and roll back partial handlers Co-Authored-By: Claude Code --- .../observability/llm_attempt_spans.py | 31 +++++++--- .../observability/test_llm_attempt_spans.py | 58 +++++++++++++++++++ 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/atomic-agents/atomic_agents/observability/llm_attempt_spans.py b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py index 05bda66b..3d1dd7c8 100644 --- a/atomic-agents/atomic_agents/observability/llm_attempt_spans.py +++ b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py @@ -3,6 +3,7 @@ import contextvars import functools import logging +import threading import weakref from contextlib import contextmanager from dataclasses import dataclass @@ -57,6 +58,7 @@ class _AgentRun: _active_run: contextvars.ContextVar[_AgentRun | None] = contextvars.ContextVar("atomic_agents_llm_run", default=None) _clients: weakref.WeakSet[_HookClient] = weakref.WeakSet() +_registration_lock = threading.Lock() def _never_raise(handler: Callable[_PARAMETERS, None]) -> Callable[_PARAMETERS, None]: @@ -72,18 +74,33 @@ def guarded(*args: _PARAMETERS.args, **kwargs: _PARAMETERS.kwargs) -> None: @_never_raise def register_client(client: _HookClient) -> None: - if not hasattr(client, "on") or client in _clients: + if not hasattr(client, "on"): return - for event, handler in _HANDLERS.items(): - client.on(event, handler) - _clients.add(client) + # Shared clients must acquire all handlers once before another run can use them. + with _registration_lock: + if client not in _clients: + _register_handlers(client) + _clients.add(client) + + +def _register_handlers(client: _HookClient) -> None: + registered: list[tuple[str, Callable[..., None]]] = [] + try: + for event, handler in _HANDLERS.items(): + client.on(event, handler) + registered.append((event, handler)) + except Exception: + for event, handler in registered: + client.off(event, handler) + raise @_never_raise def unregister_clients() -> None: - for client in list(_clients): - _unregister_client(client) - _clients.clear() + with _registration_lock: + for client in list(_clients): + _unregister_client(client) + _clients.clear() @_never_raise diff --git a/atomic-agents/tests/observability/test_llm_attempt_spans.py b/atomic-agents/tests/observability/test_llm_attempt_spans.py index abd59d5d..d3f77a9f 100644 --- a/atomic-agents/tests/observability/test_llm_attempt_spans.py +++ b/atomic-agents/tests/observability/test_llm_attempt_spans.py @@ -1,4 +1,6 @@ import asyncio +import threading +from concurrent.futures import ThreadPoolExecutor, TimeoutError from types import SimpleNamespace from unittest.mock import Mock @@ -319,3 +321,59 @@ def script(hooks, kwargs): child, parent = finished_spans(exporter, 2) assert parent.attributes["atomic_agents.llm.last_validation_error"] == "long...[truncated]" assert parent.attributes["atomic_agents.llm.last_validation_error.original_length"] == 10 + + +def test_threaded_first_use_registers_once_and_uninstalls(exporter, monkeypatch): + client = scripted_client(retry_script) + first_registration = threading.Barrier(2, timeout=5) + release_registration = threading.Event() + original_on = Hooks.on + + def blocked_on(hooks, event, handler): + if event == "completion:kwargs": + first_registration.wait() + assert release_registration.wait(timeout=5) + return original_on(hooks, event, handler) + + monkeypatch.setattr(Hooks, "on", blocked_on) + client.on.side_effect = client.hooks.on + with ThreadPoolExecutor(max_workers=2) as executor: + first = executor.submit(run_agent, client) + first_registration.wait() + second_started = threading.Event() + + def second_run(): + second_started.set() + return run_agent(client) + + second = executor.submit(second_run) + assert second_started.wait(timeout=5) + try: + with pytest.raises(TimeoutError): + second.result(timeout=0.1) + assert client.on.call_count == 1 + finally: + release_registration.set() + first.result(timeout=5) + second.result(timeout=5) + assert len(client.hooks._handlers) == 5 + assert all(len(handlers) == 1 for handlers in client.hooks._handlers.values()) + installation.uninstall() + assert all(not handlers for handlers in client.hooks._handlers.values()) + + +def test_partial_registration_is_rolled_back(exporter): + client = scripted_client(retry_script) + + def fail_registration(event, handler): + if event == "parse:error": + raise RuntimeError("registration failed") + client.hooks.on(event, handler) + + client.on.side_effect = fail_registration + llm_attempt_spans.register_client(client) + assert client not in llm_attempt_spans._clients + assert all(not handlers for handlers in client.hooks._handlers.values()) + client.on.side_effect = client.hooks.on + llm_attempt_spans.register_client(client) + assert all(len(handlers) == 1 for handlers in client.hooks._handlers.values()) From d24a24e2326ec1246ab9913f5ad120c514fdd896 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 12:33:53 +0200 Subject: [PATCH 14/25] Use Phoenix extra and disable duplicate LLM spans Co-Authored-By: Claude Code --- atomic-examples/phoenix-tracing/phoenix_tracing/main.py | 2 ++ atomic-examples/phoenix-tracing/pyproject.toml | 2 +- uv.lock | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/atomic-examples/phoenix-tracing/phoenix_tracing/main.py b/atomic-examples/phoenix-tracing/phoenix_tracing/main.py index ae1f9a3d..de99b8eb 100644 --- a/atomic-examples/phoenix-tracing/phoenix_tracing/main.py +++ b/atomic-examples/phoenix-tracing/phoenix_tracing/main.py @@ -10,12 +10,14 @@ from rich.console import Console from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema, BasicChatInputSchema, BasicChatOutputSchema +from atomic_agents.observability import AtomicAgentsInstrumentor, TracingConfig from atomic_agents.context import SystemPromptGenerator from phoenix_tracing.tools.calculator import CalculatorTool, CalculatorToolInputSchema load_dotenv() tracer_provider = register(project_name="atomic-agents-example", auto_instrument=True) +AtomicAgentsInstrumentor().instrument(config=TracingConfig(llm_spans=False)) tracer = tracer_provider.get_tracer(__name__) console = Console() diff --git a/atomic-examples/phoenix-tracing/pyproject.toml b/atomic-examples/phoenix-tracing/pyproject.toml index c626725f..e09a5bf6 100644 --- a/atomic-examples/phoenix-tracing/pyproject.toml +++ b/atomic-examples/phoenix-tracing/pyproject.toml @@ -15,7 +15,7 @@ authors = [ ] requires-python = ">=3.12" dependencies = [ - "atomic-agents[observability]", + "atomic-agents[phoenix]", "arize-phoenix-otel>=0.17.0,<1.0.0", "openinference-instrumentation-openai>=0.1.30,<0.2.0", "instructor>=1.17.0,<2", diff --git a/uv.lock b/uv.lock index 40e2514b..8ced42d6 100644 --- a/uv.lock +++ b/uv.lock @@ -3165,7 +3165,7 @@ version = "1.0.0" source = { editable = "atomic-examples/phoenix-tracing" } dependencies = [ { name = "arize-phoenix-otel" }, - { name = "atomic-agents", extra = ["observability"] }, + { name = "atomic-agents", extra = ["phoenix"] }, { name = "instructor" }, { name = "openai" }, { name = "openinference-instrumentation-openai" }, @@ -3176,7 +3176,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "arize-phoenix-otel", specifier = ">=0.17.0,<1.0.0" }, - { name = "atomic-agents", extras = ["observability"], editable = "." }, + { name = "atomic-agents", extras = ["phoenix"], editable = "." }, { name = "instructor", specifier = ">=1.17.0,<2" }, { name = "openai", specifier = ">=2.0.0,<3.0.0" }, { name = "openinference-instrumentation-openai", specifier = ">=0.1.30,<0.2.0" }, From 5ad2a52f262ad89dad05be18310270134c9db7d4 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 12:34:26 +0200 Subject: [PATCH 15/25] Ignore suppressed Instructor attempts and their later hook events Co-Authored-By: Claude Code --- .../observability/llm_attempt_spans.py | 15 ++++++--- .../observability/test_llm_attempt_spans.py | 32 ++++++++++++++++++- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/atomic-agents/atomic_agents/observability/llm_attempt_spans.py b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py index 3d1dd7c8..3c873d35 100644 --- a/atomic-agents/atomic_agents/observability/llm_attempt_spans.py +++ b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py @@ -52,6 +52,7 @@ class _AgentRun: config: TracingConfig provider: dict[str, AttributeValue] attempt_count: int = 0 + suppressed_attempt: bool = False open_attempt: Span | None = None context_token: contextvars.Token | None = None @@ -156,6 +157,10 @@ def _completion_kwargs(*args: object, **kwargs: object) -> None: if run is None: return finish_attempt(run) + # Suppressed kwargs must also swallow later response and usage outside that context. + run.suppressed_attempt = bool(context_api.get_value("suppress_instrumentation")) + if run.suppressed_attempt: + return run.attempt_count += 1 if not run.config.llm_spans: return @@ -184,7 +189,7 @@ def _usage_attributes(usage: _Usage | None) -> dict[str, int]: @_never_raise def _completion_response(response: _Response, **kwargs: object) -> None: run = _active_run.get() - if run is None: + if run is None or run.suppressed_attempt: return attributes = _without_none({_RESPONSE_MODEL: getattr(response, "model", None)}) run.span.set_attributes(attributes) @@ -202,7 +207,7 @@ def _complete_response_attempt(run: _AgentRun, response: _Response, attributes: @_never_raise def _completion_error(error: BaseException, **kwargs: object) -> None: run = _active_run.get() - if run is None or run.open_attempt is None: + if run is None or run.suppressed_attempt or run.open_attempt is None: return message = _bounded_text(_failure_message(error, run.config), run.config) run.open_attempt.set_attribute("error.type", type(error).__qualname__) @@ -213,7 +218,7 @@ def _completion_error(error: BaseException, **kwargs: object) -> None: @_never_raise def _parse_error(error: BaseException, **kwargs: object) -> None: run = _active_run.get() - if run is None: + if run is None or run.suppressed_attempt: return text = "__REDACTED__" if run.config.hide_outputs else _failure_message(error, run.config) _write_payload(run.span, _VALIDATION_ERROR, None, text, run.config.hide_outputs, run.config) @@ -223,10 +228,10 @@ def _parse_error(error: BaseException, **kwargs: object) -> None: @_never_raise def _completion_usage(usage: _Usage, *, attempt_number: int, **kwargs: object) -> None: run = _active_run.get() - if run is None: + if run is None or run.suppressed_attempt: return run.span.set_attributes(_usage_attributes(usage)) - run.span.set_attribute("atomic_agents.llm.attempts", attempt_number) + run.span.set_attribute("atomic_agents.llm.attempts", run.attempt_count) _HANDLERS: dict[str, Callable[..., None]] = { diff --git a/atomic-agents/tests/observability/test_llm_attempt_spans.py b/atomic-agents/tests/observability/test_llm_attempt_spans.py index d3f77a9f..ee30e9fb 100644 --- a/atomic-agents/tests/observability/test_llm_attempt_spans.py +++ b/atomic-agents/tests/observability/test_llm_attempt_spans.py @@ -7,7 +7,7 @@ import instructor import pytest from instructor.core.hooks import Hooks -from opentelemetry import trace as trace_api +from opentelemetry import context as context_api, trace as trace_api from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter @@ -377,3 +377,33 @@ def fail_registration(event, handler): client.on.side_effect = client.hooks.on llm_attempt_spans.register_client(client) assert all(len(handlers) == 1 for handlers in client.hooks._handlers.values()) + + +@pytest.mark.parametrize("completion", ["response", "error"]) +def test_suppressed_attempt_events_are_ignored_then_normal_attempt_works(exporter, completion): + def script(hooks, kwargs): + run = llm_attempt_spans._active_run.get() + token = context_api.attach(context_api.set_value("suppress_instrumentation", True)) + try: + hooks.emit_completion_arguments(**kwargs) + finally: + context_api.detach(token) + if completion == "response": + hooks.emit_completion_response(response()) + else: + hooks.emit_completion_error(ValueError("suppressed"), attempt_number=1, max_attempts=2, is_last_attempt=False) + hooks.emit_parse_error(ValueError("suppressed")) + hooks.emit_completion_usage(usage(99, 99), attempt_number=1) + assert run.attempt_count == 0 + assert run.open_attempt is None + assert not exporter.get_finished_spans() + hooks.emit_completion_arguments(**kwargs) + hooks.emit_completion_response(response()) + hooks.emit_completion_usage(usage(), attempt_number=2) + + run_agent(scripted_client(script)) + child, parent = finished_spans(exporter, 2) + assert child.attributes["atomic_agents.llm.attempt"] == 1 + assert parent.attributes["atomic_agents.llm.attempts"] == 1 + assert parent.attributes["gen_ai.usage.input_tokens"] == 3 + assert not parent.events From 933f9775b14e9b4e497997ea2e93b62c6f33e3d1 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 12:39:41 +0200 Subject: [PATCH 16/25] Type span wrappers and harden attempt error assertions Defer run annotations to allow direct attempt-module imports. Remove unused method names from wrapper factories. Co-Authored-By: Claude Code --- .../observability/agent_spans.py | 75 +++++++++++-------- .../observability/installation.py | 2 +- .../observability/llm_attempt_spans.py | 22 +++++- .../atomic_agents/observability/tool_spans.py | 30 +++++--- .../observability/test_llm_attempt_spans.py | 41 +++++++--- 5 files changed, 116 insertions(+), 54 deletions(-) diff --git a/atomic-agents/atomic_agents/observability/agent_spans.py b/atomic-agents/atomic_agents/observability/agent_spans.py index 37fda75c..8d85dcec 100644 --- a/atomic-agents/atomic_agents/observability/agent_spans.py +++ b/atomic-agents/atomic_agents/observability/agent_spans.py @@ -1,10 +1,12 @@ """Agent call and stream spans.""" +from __future__ import annotations + import asyncio import contextvars import functools from contextlib import contextmanager -from typing import Any, AsyncGenerator, Callable, Dict, Generator, Iterator +from typing import AsyncGenerator, Awaitable, Callable, Coroutine, Generator, Generic, Iterator, TypeVar from opentelemetry import context as context_api from opentelemetry import trace as trace_api @@ -16,6 +18,9 @@ from .span_attributes import ATTR_STREAM_CLOSED_EARLY, _agent_display_name, _agent_start_attributes, _suppressed from .span_payloads import _record_failure, _set_input_safely, _set_output_safely +OutputSchema = TypeVar("OutputSchema") +CallResult = TypeVar("CallResult") + AGENT_SYNC_METHOD = "run" AGENT_ASYNC_METHOD = "run_async" AGENT_SYNC_STREAM_METHOD = "run_stream" @@ -27,22 +32,22 @@ ) -def _user_input_from_call(args: tuple, kwargs: dict) -> Any: +def _user_input_from_call(args: tuple[object, ...], kwargs: dict[str, object]) -> object: return args[0] if args else kwargs.get("user_input") -def _finish_agent_span(span: Any, output: Any) -> None: +def _finish_agent_span(span: Span, output: object) -> None: _set_output_safely(span, output) span.set_status(Status(StatusCode.OK)) -def _mark_closed_early(span: Any, output: Any) -> None: +def _mark_closed_early(span: Span, output: object) -> None: span.set_attribute(ATTR_STREAM_CLOSED_EARLY, True) _set_output_safely(span, output) @contextmanager -def _agent_span(agent: AtomicAgent, args: tuple, kwargs: dict) -> Iterator[Span]: +def _agent_span(agent: AtomicAgent, args: tuple[object, ...], kwargs: dict[str, object]) -> Iterator[Span]: installed = installation._current_installation() with installed.tracer.start_as_current_span( f"invoke_agent {_agent_display_name(agent)}", @@ -64,9 +69,9 @@ def _agent_span(agent: AtomicAgent, args: tuple, kwargs: dict) -> Iterator[Span] _active_agent_span.reset(token) -def _trace_agent_call(method_name: str, wrapped: Callable) -> Callable: +def _trace_agent_call(wrapped: Callable[..., OutputSchema]) -> Callable[..., OutputSchema]: @functools.wraps(wrapped) - def traced(agent, *args, **kwargs): + def traced(agent: AtomicAgent, *args: object, **kwargs: object) -> OutputSchema: if _suppressed(): return wrapped(agent, *args, **kwargs) with _agent_span(agent, args, kwargs) as span: @@ -77,9 +82,11 @@ def traced(agent, *args, **kwargs): return traced -def _trace_agent_async_call(method_name: str, wrapped: Callable) -> Callable: +def _trace_agent_async_call( + wrapped: Callable[..., Awaitable[OutputSchema]], +) -> Callable[..., Coroutine[object, object, OutputSchema]]: @functools.wraps(wrapped) - async def traced(agent, *args, **kwargs): + async def traced(agent: AtomicAgent, *args: object, **kwargs: object) -> OutputSchema: if _suppressed(): return await wrapped(agent, *args, **kwargs) with _agent_span(agent, args, kwargs) as span: @@ -90,7 +97,7 @@ async def traced(agent, *args, **kwargs): return traced -def _start_agent_span(agent: AtomicAgent, method_name: str, args: tuple, kwargs: dict) -> Span: +def _start_agent_span(agent: AtomicAgent, args: tuple[object, ...], kwargs: dict[str, object]) -> Span: if _suppressed(): return trace_api.INVALID_SPAN span = installation._current_installation().tracer.start_span( @@ -101,7 +108,7 @@ def _start_agent_span(agent: AtomicAgent, method_name: str, args: tuple, kwargs: return span -def _call_within_span(run: llm_attempt_spans._AgentRun, function: Callable, *args: Any) -> Any: +def _call_within_span(run: llm_attempt_spans._AgentRun, function: Callable[..., CallResult], *args: object) -> CallResult: span = run.span token = context_api.attach(trace_api.set_span_in_context(span)) agent_token = _active_agent_span.set(span) @@ -113,7 +120,7 @@ def _call_within_span(run: llm_attempt_spans._AgentRun, function: Callable, *arg context_api.detach(token) -async def _await_within_span(run: llm_attempt_spans._AgentRun, awaitable: Any) -> Any: +async def _await_within_span(run: llm_attempt_spans._AgentRun, awaitable: Awaitable[CallResult]) -> CallResult: span = run.span token = context_api.attach(trace_api.set_span_in_context(span)) agent_token = _active_agent_span.set(span) @@ -125,17 +132,19 @@ async def _await_within_span(run: llm_attempt_spans._AgentRun, awaitable: Any) - context_api.detach(token) -class _SyncStreamCursor: - def __init__(self, run: llm_attempt_spans._AgentRun, inner: Generator): +class _SyncStreamCursor(Generic[OutputSchema]): + def __init__(self, run: llm_attempt_spans._AgentRun, inner: Generator[OutputSchema, None, OutputSchema | None]): self.run = run self.span = run.span self.inner = inner - self.last: Any = None - self.final: Any = None + self.last: OutputSchema + self.output: OutputSchema | None = None + self.final: OutputSchema | None = None def advance(self) -> bool: try: self.last = _call_within_span(self.run, next, self.inner) + self.output = self.last except StopIteration as stop: self.final = stop.value return False @@ -143,39 +152,43 @@ def advance(self) -> bool: def close_early(self) -> None: _call_within_span(self.run, self.inner.close) - _mark_closed_early(self.span, self.last) + _mark_closed_early(self.span, self.output) self.span.set_status(Status(StatusCode.OK)) -class _AsyncStreamCursor: +class _AsyncStreamCursor(Generic[OutputSchema]): """Async twin of :class:`_SyncStreamCursor`; async generators carry no return value, so the last partial is the output.""" - def __init__(self, run: llm_attempt_spans._AgentRun, inner: AsyncGenerator): + def __init__(self, run: llm_attempt_spans._AgentRun, inner: AsyncGenerator[OutputSchema, None]): self.run = run self.span = run.span self.inner = inner - self.last: Any = None + self.last: OutputSchema + self.output: OutputSchema | None = None async def advance(self) -> bool: try: self.last = await _await_within_span(self.run, self.inner.__anext__()) + self.output = self.last except StopAsyncIteration: return False return True async def close_early(self) -> None: await _await_within_span(self.run, self.inner.aclose()) - _mark_closed_early(self.span, self.last) + _mark_closed_early(self.span, self.output) self.span.set_status(Status(StatusCode.OK)) def mark_cancelled(self) -> None: - _mark_closed_early(self.span, self.last) + _mark_closed_early(self.span, self.output) -def _trace_agent_stream(method_name: str, wrapped: Callable) -> Callable: +def _trace_agent_stream( + wrapped: Callable[..., Generator[OutputSchema, None, OutputSchema | None]], +) -> Callable[..., Generator[OutputSchema, None, OutputSchema | None]]: @functools.wraps(wrapped) - def traced(agent, *args, **kwargs): - span = _start_agent_span(agent, method_name, args, kwargs) + def traced(agent: AtomicAgent, *args: object, **kwargs: object) -> Generator[OutputSchema, None, OutputSchema | None]: + span = _start_agent_span(agent, args, kwargs) run = llm_attempt_spans.new_run(span, agent) cursor = _SyncStreamCursor(run, wrapped(agent, *args, **kwargs)) try: @@ -196,16 +209,18 @@ def traced(agent, *args, **kwargs): return traced -def _trace_agent_async_stream(method_name: str, wrapped: Callable) -> Callable: +def _trace_agent_async_stream( + wrapped: Callable[..., AsyncGenerator[OutputSchema, None]], +) -> Callable[..., AsyncGenerator[OutputSchema, None]]: @functools.wraps(wrapped) - async def traced(agent, *args, **kwargs): - span = _start_agent_span(agent, method_name, args, kwargs) + async def traced(agent: AtomicAgent, *args: object, **kwargs: object) -> AsyncGenerator[OutputSchema, None]: + span = _start_agent_span(agent, args, kwargs) run = llm_attempt_spans.new_run(span, agent) cursor = _AsyncStreamCursor(run, wrapped(agent, *args, **kwargs)) try: while await cursor.advance(): yield cursor.last - _finish_agent_span(span, cursor.last) + _finish_agent_span(span, cursor.output) except GeneratorExit: await cursor.close_early() raise @@ -222,7 +237,7 @@ async def traced(agent, *args, **kwargs): return traced -_AGENT_WRAPPERS: Dict[str, Callable[[str, Callable], Callable]] = { +_AGENT_WRAPPERS = { AGENT_SYNC_METHOD: _trace_agent_call, AGENT_ASYNC_METHOD: _trace_agent_async_call, AGENT_SYNC_STREAM_METHOD: _trace_agent_stream, diff --git a/atomic-agents/atomic_agents/observability/installation.py b/atomic-agents/atomic_agents/observability/installation.py index 981ce2d3..f8f6d6e8 100644 --- a/atomic-agents/atomic_agents/observability/installation.py +++ b/atomic-agents/atomic_agents/observability/installation.py @@ -108,7 +108,7 @@ def _instrument_agent_class() -> None: for method_name, wrap in agent_spans._AGENT_WRAPPERS.items(): original = AtomicAgent.__dict__[method_name] if not getattr(original, TRACED_MARKER, False): - setattr(AtomicAgent, method_name, _mark_traced(wrap(method_name, original), original, False)) + setattr(AtomicAgent, method_name, _mark_traced(wrap(original), original, False)) def _mark_traced(wrapper: Callable, original: Callable, inherited: bool) -> Callable: diff --git a/atomic-agents/atomic_agents/observability/llm_attempt_spans.py b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py index 3c873d35..5e484409 100644 --- a/atomic-agents/atomic_agents/observability/llm_attempt_spans.py +++ b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py @@ -1,5 +1,7 @@ """Instructor attempt spans and cumulative agent usage.""" +from __future__ import annotations + import contextvars import functools import logging @@ -27,11 +29,29 @@ _VALIDATION_ERROR = "atomic_agents.llm.last_validation_error" -class _Usage(Protocol): +class _PromptTokenDetails(Protocol): + cached_tokens: int | None + + +class _CompletionTokenDetails(Protocol): + reasoning_tokens: int | None + + +class _InputOutputUsage(Protocol): input_tokens: int output_tokens: int +class _PromptCompletionUsage(Protocol): + prompt_tokens: int + completion_tokens: int + prompt_tokens_details: _PromptTokenDetails | None + completion_tokens_details: _CompletionTokenDetails | None + + +_Usage = _InputOutputUsage | _PromptCompletionUsage + + class _Response(Protocol): model: str usage: _Usage diff --git a/atomic-agents/atomic_agents/observability/tool_spans.py b/atomic-agents/atomic_agents/observability/tool_spans.py index f10c2f36..59b3f16d 100644 --- a/atomic-agents/atomic_agents/observability/tool_spans.py +++ b/atomic-agents/atomic_agents/observability/tool_spans.py @@ -7,7 +7,7 @@ import logging from contextlib import contextmanager from types import FunctionType -from typing import Any, Callable, Iterator, List, Optional, Type +from typing import Awaitable, Callable, Coroutine, Iterator, List, Optional, Type, TypeVar from opentelemetry.trace import Span, Status, StatusCode from opentelemetry.util.types import AttributeValue @@ -34,6 +34,8 @@ logger = logging.getLogger("atomic_agents.observability.tracing") +OutputSchema = TypeVar("OutputSchema") + TOOL_METHOD_NAMES = ("run", "run_async", "arun") @@ -72,7 +74,7 @@ def _instrument_tool_class(tool_class: Type) -> None: setattr(tool_class, method_name, installation._mark_traced(_trace_tool_method(method), method, inherited)) -def _is_traceable_function(method: Any) -> bool: +def _is_traceable_function(method: object) -> bool: if not isinstance(method, FunctionType) or inspect.isasyncgenfunction(method): return False if getattr(method, "__isabstractmethod__", False): @@ -127,7 +129,7 @@ def _tool_start_attributes(tool: BaseTool) -> dict[str, AttributeValue]: @contextmanager -def _tool_span(tool: BaseTool, args: tuple, kwargs: dict) -> Iterator[Span]: +def _tool_span(tool: BaseTool, args: tuple[object, ...], kwargs: dict[str, object]) -> Iterator[Span]: attributes = _tool_start_attributes(tool) with installation._current_installation().tracer.start_as_current_span( f"execute_tool {attributes[ATTR_TOOL_NAME]}", @@ -143,16 +145,18 @@ def _tool_span(tool: BaseTool, args: tuple, kwargs: dict) -> Iterator[Span]: raise -def _tool_input_from_call(args: tuple, kwargs: dict) -> Any: +def _tool_input_from_call(args: tuple[object, ...], kwargs: dict[str, object]) -> object: return args[0] if args else kwargs.get("params") -def _trace_tool_method(wrapped: Callable) -> Callable: +def _trace_tool_method( + wrapped: Callable[..., OutputSchema], +) -> Callable[..., OutputSchema] | Callable[..., Coroutine[object, object, object]]: if inspect.iscoroutinefunction(wrapped): return _trace_tool_async_method(wrapped) @functools.wraps(wrapped) - def traced(tool, *args, **kwargs): + def traced(tool: BaseTool, *args: object, **kwargs: object) -> OutputSchema: if _active_tool.get() is tool or _suppressed(): return wrapped(tool, *args, **kwargs) with _tool_span(tool, args, kwargs) as span: @@ -164,9 +168,11 @@ def traced(tool, *args, **kwargs): return traced -def _trace_tool_async_method(wrapped: Callable) -> Callable: +def _trace_tool_async_method( + wrapped: Callable[..., Awaitable[OutputSchema]], +) -> Callable[..., Coroutine[object, object, OutputSchema]]: @functools.wraps(wrapped) - async def traced(tool, *args, **kwargs): + async def traced(tool: BaseTool, *args: object, **kwargs: object) -> OutputSchema: if _active_tool.get() is tool or _suppressed(): return await wrapped(tool, *args, **kwargs) with _tool_span(tool, args, kwargs) as span: @@ -178,7 +184,9 @@ async def traced(tool, *args, **kwargs): return traced -def _run_as_active_tool(tool: BaseTool, wrapped: Callable, args: tuple, kwargs: dict) -> Any: +def _run_as_active_tool( + tool: BaseTool, wrapped: Callable[..., OutputSchema], args: tuple[object, ...], kwargs: dict[str, object] +) -> OutputSchema: token = _active_tool.set(tool) try: return wrapped(tool, *args, **kwargs) @@ -186,7 +194,9 @@ def _run_as_active_tool(tool: BaseTool, wrapped: Callable, args: tuple, kwargs: _active_tool.reset(token) -async def _await_as_active_tool(tool: BaseTool, wrapped: Callable, args: tuple, kwargs: dict) -> Any: +async def _await_as_active_tool( + tool: BaseTool, wrapped: Callable[..., Awaitable[OutputSchema]], args: tuple[object, ...], kwargs: dict[str, object] +) -> OutputSchema: token = _active_tool.set(tool) try: return await wrapped(tool, *args, **kwargs) diff --git a/atomic-agents/tests/observability/test_llm_attempt_spans.py b/atomic-agents/tests/observability/test_llm_attempt_spans.py index ee30e9fb..68ac6c97 100644 --- a/atomic-agents/tests/observability/test_llm_attempt_spans.py +++ b/atomic-agents/tests/observability/test_llm_attempt_spans.py @@ -12,6 +12,7 @@ from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace import StatusCode +from pydantic import ValidationError from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema from atomic_agents.observability import TracingConfig, using_conversation @@ -129,15 +130,26 @@ def test_disabled_attempt_spans_keep_totals(exporter): def test_completion_error_is_bounded_and_ends_attempt(exporter): + installed = installation._current_installation() + installation.install(tracer_provider=installed.tracer_provider, config=TracingConfig(max_attribute_length=32)) + error = ValueError("failed " * 20) + expected = str(error)[:32] + "...[truncated]" + def script(hooks, kwargs): hooks.emit_completion_arguments(**kwargs) - hooks.emit_completion_error(ValueError("failed"), attempt_number=1, max_attempts=1, is_last_attempt=True) + hooks.emit_parse_error(error) + hooks.emit_completion_error(error, attempt_number=1, max_attempts=1, is_last_attempt=True) run_agent(scripted_client(script)) child, parent = finished_spans(exporter, 2) assert child.status.status_code == StatusCode.ERROR assert child.attributes["error.type"] == "ValueError" - assert child.events[0].attributes["exception.message"] == "failed" + assert child.events[0].attributes["exception.message"] == expected + assert child.events[0].attributes["exception.stacktrace"] == expected + assert child.status.description == expected + assert len(child.status.description) == 32 + len("...[truncated]") + assert parent.attributes["atomic_agents.llm.last_validation_error"] == expected + assert parent.attributes["atomic_agents.llm.last_validation_error.original_length"] == len(str(error)) assert parent.status.status_code == StatusCode.OK @@ -169,7 +181,7 @@ async def complete(agent, *args, **kwargs): return Answer(answer="ok") if asynchronous: - return await agent_spans._trace_agent_async_call("run_async", complete)(agent(client)) + return await agent_spans._trace_agent_async_call(complete)(agent(client)) return await asyncio.to_thread(run_agent, client) async def gather(): @@ -209,11 +221,11 @@ async def async_stream(*args, **kwargs): def invoke(): if kind == "sync": - return agent_spans._trace_agent_call("run", call)(agent(client)) + return agent_spans._trace_agent_call(call)(agent(client)) if kind == "async": - return asyncio.run(agent_spans._trace_agent_async_call("run_async", async_call)(agent(client))) + return asyncio.run(agent_spans._trace_agent_async_call(async_call)(agent(client))) if kind == "stream": - iterator = agent_spans._trace_agent_stream("run_stream", stream)(agent(client)) + iterator = agent_spans._trace_agent_stream(stream)(agent(client)) next(iterator) assert trace_api.get_current_span() is trace_api.INVALID_SPAN iterator.close() @@ -221,7 +233,7 @@ def invoke(): return asyncio.run(consume_async_stream()) async def consume_async_stream(): - iterator = agent_spans._trace_agent_async_stream("run_async_stream", async_stream)(agent(client)) + iterator = agent_spans._trace_agent_async_stream(async_stream)(agent(client)) await anext(iterator) assert trace_api.get_current_span() is trace_api.INVALID_SPAN await iterator.aclose() @@ -243,7 +255,10 @@ def test_validation_text_redacted_and_anthropic_usage(exporter): def script(hooks, kwargs): hooks.emit_completion_arguments(**kwargs) - hooks.emit_parse_error(ValueError("secret")) + with pytest.raises(ValidationError) as invalid: + Answer.model_validate({"answer": {"secret": "private output"}}) + hooks.emit_parse_error(invalid.value) + hooks.emit_completion_error(invalid.value, attempt_number=1, max_attempts=1, is_last_attempt=True) hooks.emit_completion_usage(SimpleNamespace(input_tokens=9, output_tokens=4), attempt_number=1) run_agent(scripted_client(script)) @@ -251,6 +266,10 @@ def script(hooks, kwargs): assert parent.attributes["atomic_agents.llm.last_validation_error"] == "__REDACTED__" assert parent.events[0].attributes["atomic_agents.llm.last_validation_error"] == "__REDACTED__" assert parent.attributes["gen_ai.usage.input_tokens"] == 9 + assert child.status.description == "ValidationError: 1 errors (__REDACTED__)" + assert child.events[0].attributes["exception.message"] == child.status.description + assert child.events[0].attributes["exception.stacktrace"] == child.status.description + assert "private output" not in str(child.events) def test_hook_failure_warns_without_breaking_agent(exporter, caplog): @@ -294,14 +313,12 @@ async def async_stream(*args, **kwargs): yield complete(expected) async def consume(): - return [ - partial async for partial in agent_spans._trace_agent_async_stream("run_async_stream", async_stream)(agent(client)) - ] + return [partial async for partial in agent_spans._trace_agent_async_stream(async_stream)(agent(client))] if asynchronous: assert len(asyncio.run(consume())) == 2 else: - assert len(list(agent_spans._trace_agent_stream("run_stream", stream)(agent(client)))) == 2 + assert len(list(agent_spans._trace_agent_stream(stream)(agent(client)))) == 2 child, parent = finished_spans(exporter, 2) assert child.status.status_code == StatusCode.OK assert parent.status.status_code == StatusCode.OK From 34331747b5a993aa47c80754e484270274ec40cf Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 12:47:36 +0200 Subject: [PATCH 17/25] Allow automatic tracing to disable LLM attempt spans from environment Add ATOMIC_AGENTS_TRACE_LLM_SPANS and use it in the Phoenix example instead of replacing the instrumentor configuration. Cover false, mixed-case false, zero and enabled defaults. Co-Authored-By: Claude Code --- .../observability/tracing_config.py | 1 + .../tests/observability/test_tracing.py | 19 +++++++++++++++++++ .../phoenix-tracing/phoenix_tracing/main.py | 3 +-- docs/guides/observability.md | 17 ++++++++++++++--- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/atomic-agents/atomic_agents/observability/tracing_config.py b/atomic-agents/atomic_agents/observability/tracing_config.py index 8c25224f..f0200874 100644 --- a/atomic-agents/atomic_agents/observability/tracing_config.py +++ b/atomic-agents/atomic_agents/observability/tracing_config.py @@ -24,6 +24,7 @@ def from_environment(cls) -> "TracingConfig": hide_inputs=_visibility_environment("INPUTS").lower() in ("true", "1"), hide_outputs=_visibility_environment("OUTPUTS").lower() in ("true", "1"), max_attribute_length=int(os.environ.get("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", "4096")), + llm_spans=os.environ.get("ATOMIC_AGENTS_TRACE_LLM_SPANS", "true").lower() not in ("false", "0"), ) @classmethod diff --git a/atomic-agents/tests/observability/test_tracing.py b/atomic-agents/tests/observability/test_tracing.py index 66e6036a..3c7b1d8c 100644 --- a/atomic-agents/tests/observability/test_tracing.py +++ b/atomic-agents/tests/observability/test_tracing.py @@ -995,6 +995,25 @@ def test_tracing_config_environment_and_coercion(monkeypatch): assert TracingConfig.coerce(TraceConfig(hide_inputs=True)) == TracingConfig(hide_inputs=True) +@pytest.mark.parametrize( + "value, expected", [("false", False), ("FaLsE", False), ("0", False), ("true", True), ("1", True), ("", True)] +) +def test_tracing_config_llm_spans_environment(monkeypatch, value, expected): + from atomic_agents.observability import TracingConfig + + monkeypatch.setenv("ATOMIC_AGENTS_TRACE_LLM_SPANS", value) + assert TracingConfig.from_environment().llm_spans is expected + assert TracingConfig.coerce(None).llm_spans is expected + + +def test_tracing_config_llm_spans_defaults_to_enabled(monkeypatch): + from atomic_agents.observability import TracingConfig + + monkeypatch.delenv("ATOMIC_AGENTS_TRACE_LLM_SPANS", raising=False) + assert TracingConfig.from_environment().llm_spans is True + assert TracingConfig().llm_spans is True + + def test_session_is_mirrored_to_genai_conversation(exporter): with using_session("session-1"): make_agent(sync_client()).run(QuestionSchema(question="?")) diff --git a/atomic-examples/phoenix-tracing/phoenix_tracing/main.py b/atomic-examples/phoenix-tracing/phoenix_tracing/main.py index de99b8eb..13d119c8 100644 --- a/atomic-examples/phoenix-tracing/phoenix_tracing/main.py +++ b/atomic-examples/phoenix-tracing/phoenix_tracing/main.py @@ -10,14 +10,13 @@ from rich.console import Console from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema, BasicChatInputSchema, BasicChatOutputSchema -from atomic_agents.observability import AtomicAgentsInstrumentor, TracingConfig from atomic_agents.context import SystemPromptGenerator from phoenix_tracing.tools.calculator import CalculatorTool, CalculatorToolInputSchema load_dotenv() +os.environ.setdefault("ATOMIC_AGENTS_TRACE_LLM_SPANS", "false") tracer_provider = register(project_name="atomic-agents-example", auto_instrument=True) -AtomicAgentsInstrumentor().instrument(config=TracingConfig(llm_spans=False)) tracer = tracer_provider.get_tracer(__name__) console = Console() diff --git a/docs/guides/observability.md b/docs/guides/observability.md index cc1ebf84..a0062cc7 100644 --- a/docs/guides/observability.md +++ b/docs/guides/observability.md @@ -159,6 +159,7 @@ export OTEL_METRICS_EXPORTER=none export OTEL_LOGS_EXPORTER=none export ATOMIC_AGENTS_TRACE_HIDE_INPUTS=true export ATOMIC_AGENTS_TRACE_HIDE_OUTPUTS=true +export ATOMIC_AGENTS_TRACE_LLM_SPANS=false opentelemetry-instrument python app.py ``` @@ -174,12 +175,20 @@ share one installation, so activating several does not duplicate spans. OpenTele provider, so calling Phoenix registration after launcher startup does not switch where spans are exported. See "Re-instrumentation" for what a second `instrument()` call does. -The launcher adds AGENT, TOOL and LLM attempt spans and no parent span. Use the grouping example below when you want -several agent and tool calls in one trace. +The launcher adds AGENT and TOOL spans and no parent span. The example above disables Atomic Agents' LLM attempt +spans for applications that also use a provider instrumentor. Leave `ATOMIC_AGENTS_TRACE_LLM_SPANS` unset to keep +attempt spans enabled. Use the grouping example below when you want several agent and tool calls in one trace. ## Setup with Phoenix (automatic) +`register(auto_instrument=True)` also activates the OpenAI instrumentor, so set +`ATOMIC_AGENTS_TRACE_LLM_SPANS=false` (before import) to avoid two LLM layers: + ```python +import os + +os.environ.setdefault("ATOMIC_AGENTS_TRACE_LLM_SPANS", "false") + from phoenix.otel import register tracer_provider = register(project_name="my-app", auto_instrument=True) @@ -349,7 +358,9 @@ TracingConfig(hide_inputs=True, hide_outputs=True, max_attribute_length=2048) When no config is passed to `instrument()`, the settings come from the environment. `ATOMIC_AGENTS_TRACE_HIDE_INPUTS` and `ATOMIC_AGENTS_TRACE_HIDE_OUTPUTS` are read first, then `OPENINFERENCE_HIDE_INPUTS` and `OPENINFERENCE_HIDE_OUTPUTS` -as a fallback (`true` or `1`). `ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH` sets the size cap. If you pass your own +as a fallback (`true` or `1`). `ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH` sets the size cap. +`ATOMIC_AGENTS_TRACE_LLM_SPANS` disables attempt spans for `false` or `0` (case-insensitive); any other value or unset +keeps them enabled. If you pass your own `TracingConfig(...)`, the environment is not read; use `TracingConfig.from_environment()` to start from it. An OpenInference `TraceConfig` is also accepted for `config`, for its `hide_inputs` and `hide_outputs` settings. From 8f67f5bd6b40d0f29d69bca7eec1a87b540fe793 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 12:56:28 +0200 Subject: [PATCH 18/25] Redact tracing failures and accumulate real Instructor response usage Exercise retry exhaustion, Responses usage, validation attempt status, and bounded traceback payloads through offline Instructor transports. Co-Authored-By: Claude Code --- .../observability/agent_spans.py | 6 +- .../observability/llm_attempt_spans.py | 82 ++++++--- .../observability/span_payloads.py | 65 ++++++- .../observability/instructor_transport.py | 93 ++++++++++ .../observability/test_llm_attempt_spans.py | 169 +++++++++++++++--- .../tests/observability/test_tracing.py | 8 +- 6 files changed, 355 insertions(+), 68 deletions(-) create mode 100644 atomic-agents/tests/observability/instructor_transport.py diff --git a/atomic-agents/atomic_agents/observability/agent_spans.py b/atomic-agents/atomic_agents/observability/agent_spans.py index 8d85dcec..cbfe21d1 100644 --- a/atomic-agents/atomic_agents/observability/agent_spans.py +++ b/atomic-agents/atomic_agents/observability/agent_spans.py @@ -65,7 +65,7 @@ def _agent_span(agent: AtomicAgent, args: tuple[object, ...], kwargs: dict[str, _record_failure(span, error) raise finally: - llm_attempt_spans.finish_attempt(run) + llm_attempt_spans.finish_run(run) _active_agent_span.reset(token) @@ -203,7 +203,7 @@ def traced(agent: AtomicAgent, *args: object, **kwargs: object) -> Generator[Out _record_failure(span, error) raise finally: - llm_attempt_spans.finish_attempt(run) + llm_attempt_spans.finish_run(run) span.end() return traced @@ -231,7 +231,7 @@ async def traced(agent: AtomicAgent, *args: object, **kwargs: object) -> AsyncGe _record_failure(span, error) raise finally: - llm_attempt_spans.finish_attempt(run) + llm_attempt_spans.finish_run(run) span.end() return traced diff --git a/atomic-agents/atomic_agents/observability/llm_attempt_spans.py b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py index 5e484409..05e4debc 100644 --- a/atomic-agents/atomic_agents/observability/llm_attempt_spans.py +++ b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py @@ -8,18 +8,18 @@ import threading import weakref from contextlib import contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Callable, Iterator, ParamSpec, Protocol from opentelemetry import context as context_api, trace as trace_api -from opentelemetry.trace import Span, Status, StatusCode, Tracer +from opentelemetry.trace import Span, SpanKind, Status, StatusCode, Tracer from opentelemetry.util.types import AttributeValue from atomic_agents.agents.atomic_agent import AtomicAgent from . import installation from .span_attributes import _best_effort_attributes, _optional_attributes, _provider_attributes, _without_none -from .span_payloads import _bounded_text, _failure_message, _write_payload +from .span_payloads import _bounded_text, _failure_message, _record_failure, _write_payload from .tracing_config import TracingConfig logger = logging.getLogger("atomic_agents.observability.tracing") @@ -40,6 +40,8 @@ class _CompletionTokenDetails(Protocol): class _InputOutputUsage(Protocol): input_tokens: int output_tokens: int + input_tokens_details: _PromptTokenDetails | None + output_tokens_details: _CompletionTokenDetails | None class _PromptCompletionUsage(Protocol): @@ -71,8 +73,10 @@ class _AgentRun: tracer: Tracer config: TracingConfig provider: dict[str, AttributeValue] + usage_totals: dict[str, int] = field(default_factory=dict) attempt_count: int = 0 suppressed_attempt: bool = False + response_received: bool = False open_attempt: Span | None = None context_token: contextvars.Token | None = None @@ -162,10 +166,14 @@ def _detach_attempt(run: _AgentRun) -> None: @_never_raise -def finish_attempt(run: _AgentRun, status: Status = Status(StatusCode.UNSET)) -> None: +def finish_attempt(run: _AgentRun, status: Status | None = None) -> None: if run.open_attempt is None: return - run.open_attempt.set_status(status) + if run.response_received: + status = Status(StatusCode.OK) + run.response_received = False + if status is not None: + run.open_attempt.set_status(status) run.open_attempt.end() run.open_attempt = None _detach_attempt(run) @@ -177,14 +185,14 @@ def _completion_kwargs(*args: object, **kwargs: object) -> None: if run is None: return finish_attempt(run) - # Suppressed kwargs must also swallow later response and usage outside that context. + # Suppressed kwargs must also swallow later response events outside that context. run.suppressed_attempt = bool(context_api.get_value("suppress_instrumentation")) if run.suppressed_attempt: return run.attempt_count += 1 if not run.config.llm_spans: return - model = str(kwargs["model"]) if kwargs.get("model") is not None else None + model = _request_model(kwargs) attributes: dict[str, AttributeValue | None] = { "gen_ai.operation.name": "chat", "gen_ai.request.model": model, @@ -192,16 +200,29 @@ def _completion_kwargs(*args: object, **kwargs: object) -> None: } attributes.update(run.provider) attributes.update(_optional_attributes("LLM", {})) - run.open_attempt = run.tracer.start_span(f"chat {model}", attributes=_without_none(attributes)) + run.open_attempt = run.tracer.start_span( + f"chat {model}" if model else "chat", kind=SpanKind.CLIENT, attributes=_without_none(attributes) + ) _attach_attempt(run) +def _request_model(kwargs: dict[str, object]) -> str | None: + model = kwargs.get("model") + return str(model) if model is not None else None + + def _usage_attributes(usage: _Usage | None) -> dict[str, int]: readers = { "input_tokens": getattr(usage, "input_tokens", getattr(usage, "prompt_tokens", None)), "output_tokens": getattr(usage, "output_tokens", getattr(usage, "completion_tokens", None)), - "cache_read.input_tokens": getattr(getattr(usage, "prompt_tokens_details", None), "cached_tokens", None), - "reasoning.output_tokens": getattr(getattr(usage, "completion_tokens_details", None), "reasoning_tokens", None), + "cache_read.input_tokens": getattr( + getattr(usage, "input_tokens_details", getattr(usage, "prompt_tokens_details", None)), "cached_tokens", None + ), + "reasoning.output_tokens": getattr( + getattr(usage, "output_tokens_details", getattr(usage, "completion_tokens_details", None)), + "reasoning_tokens", + None, + ), } return {"gen_ai.usage." + key: value for key, value in readers.items() if isinstance(value, int)} @@ -213,15 +234,18 @@ def _completion_response(response: _Response, **kwargs: object) -> None: return attributes = _without_none({_RESPONSE_MODEL: getattr(response, "model", None)}) run.span.set_attributes(attributes) - _complete_response_attempt(run, response, attributes) + usage = _usage_attributes(getattr(response, "usage", None)) + for key, value in usage.items(): + run.usage_totals[key] = run.usage_totals.get(key, 0) + value + _complete_response_attempt(run, usage, attributes) -def _complete_response_attempt(run: _AgentRun, response: _Response, attributes: dict[str, AttributeValue]) -> None: +def _complete_response_attempt(run: _AgentRun, usage: dict[str, int], attributes: dict[str, AttributeValue]) -> None: if run.open_attempt is None: return run.open_attempt.set_attributes(attributes) - run.open_attempt.set_attributes(_usage_attributes(getattr(response, "usage", None))) - finish_attempt(run, Status(StatusCode.OK)) + run.open_attempt.set_attributes(usage) + run.response_received = True @_never_raise @@ -229,10 +253,9 @@ def _completion_error(error: BaseException, **kwargs: object) -> None: run = _active_run.get() if run is None or run.suppressed_attempt or run.open_attempt is None: return - message = _bounded_text(_failure_message(error, run.config), run.config) - run.open_attempt.set_attribute("error.type", type(error).__qualname__) - run.open_attempt.record_exception(error, attributes={"exception.message": message, "exception.stacktrace": message}) - finish_attempt(run, Status(StatusCode.ERROR, message)) + run.response_received = False + _record_failure(run.open_attempt, error, run.config) + finish_attempt(run) @_never_raise @@ -242,15 +265,25 @@ def _parse_error(error: BaseException, **kwargs: object) -> None: return text = "__REDACTED__" if run.config.hide_outputs else _failure_message(error, run.config) _write_payload(run.span, _VALIDATION_ERROR, None, text, run.config.hide_outputs, run.config) - run.span.add_event("validation_error", {_ATTEMPT: run.attempt_count, _VALIDATION_ERROR: _bounded_text(text, run.config)}) + event = {_ATTEMPT: run.attempt_count, _VALIDATION_ERROR: _bounded_text(text, run.config)} + run.span.add_event("validation_error", event) + _mark_attempt_validation_error(run, error) -@_never_raise -def _completion_usage(usage: _Usage, *, attempt_number: int, **kwargs: object) -> None: - run = _active_run.get() - if run is None or run.suppressed_attempt: +def _mark_attempt_validation_error(run: _AgentRun, error: BaseException) -> None: + if run.open_attempt is None: return - run.span.set_attributes(_usage_attributes(usage)) + run.response_received = False + message = _bounded_text(_failure_message(error, run.config), run.config) + run.open_attempt.add_event("validation_error", {_ATTEMPT: run.attempt_count, _VALIDATION_ERROR: message}) + run.open_attempt.set_attribute("error.type", type(error).__qualname__) + run.open_attempt.set_status(Status(StatusCode.ERROR, message)) + + +@_never_raise +def finish_run(run: _AgentRun) -> None: + finish_attempt(run) + run.span.set_attributes(run.usage_totals) run.span.set_attribute("atomic_agents.llm.attempts", run.attempt_count) @@ -259,5 +292,4 @@ def _completion_usage(usage: _Usage, *, attempt_number: int, **kwargs: object) - "completion:response": _completion_response, "completion:error": _completion_error, "parse:error": _parse_error, - "completion:usage": _completion_usage, } diff --git a/atomic-agents/atomic_agents/observability/span_payloads.py b/atomic-agents/atomic_agents/observability/span_payloads.py index 57d41001..62317f29 100644 --- a/atomic-agents/atomic_agents/observability/span_payloads.py +++ b/atomic-agents/atomic_agents/observability/span_payloads.py @@ -2,6 +2,10 @@ import json import logging +import traceback +from collections.abc import Iterator + +from instructor.core.exceptions import InstructorError from opentelemetry.trace import Span, Status, StatusCode from pydantic import BaseModel, ValidationError @@ -75,20 +79,65 @@ def _set_output_safely(span: Span, value: object, primary: str = ATTR_OUTPUT) -> def _bounded_text(value: str, config: TracingConfig) -> str: - if len(value) > config.max_attribute_length: - return value[: config.max_attribute_length] + TRUNCATION_MARKER + limit = config.max_attribute_length + if len(value) > limit: + marker = TRUNCATION_MARKER[:limit] + return value[: limit - len(marker)] + marker return value +def _exception_chain(error: BaseException) -> Iterator[BaseException]: + seen: set[int] = set() + while id(error) not in seen: + seen.add(id(error)) + yield error + following = error.__cause__ or error.__context__ + if following is None: + return + error = following + + +def _instructor_attempt_count(error: InstructorError) -> int: + if error.failed_attempts: + return len(error.failed_attempts) + return getattr(error, "n_attempts", 0) + + +def _failure_count(error: BaseException) -> str | None: + if isinstance(error, InstructorError): + return f"{_instructor_attempt_count(error)} attempts" + if isinstance(error, ValidationError): + return f"{error.error_count()} errors" + return None + + def _failure_message(error: BaseException, config: TracingConfig) -> str: - if config.hide_outputs and isinstance(error, ValidationError): - return f"ValidationError: {error.error_count()} errors (__REDACTED__)" - return str(error) + if not config.hide_outputs: + return str(error) + count = next((count for cause in _exception_chain(error) if (count := _failure_count(cause)) is not None), "1 errors") + return f"{type(error).__qualname__}: {count} (__REDACTED__)" -def _record_failure(span: Span, error: BaseException) -> None: - config = installation._current_installation().config +def _failure_stacktrace(error: BaseException, config: TracingConfig) -> str: + lines = traceback.format_exception(error, chain=not config.hide_outputs) + if config.hide_outputs: + lines = _redacted_traceback(error, config) + return _bounded_text("".join(lines), config) + + +def _redacted_traceback(error: BaseException, config: TracingConfig) -> list[str]: + frames = [ + traceback.FrameSummary(frame.filename, frame.lineno, frame.name, line="") + for frame in traceback.extract_tb(error.__traceback__) + ] + return ["Traceback (most recent call last):\n", *traceback.format_list(frames), _failure_message(error, config) + "\n"] + + +def _record_failure(span: Span, error: BaseException, config: TracingConfig | None = None) -> None: + config = config or installation._current_installation().config message = _bounded_text(_failure_message(error, config), config) span.set_attribute(ATTR_ERROR_TYPE, type(error).__qualname__) - span.record_exception(error, attributes={"exception.message": message, "exception.stacktrace": message}) + span.record_exception( + error, attributes={"exception.message": message, "exception.stacktrace": _failure_stacktrace(error, config)} + ) span.set_status(Status(StatusCode.ERROR, message)) diff --git a/atomic-agents/tests/observability/instructor_transport.py b/atomic-agents/tests/observability/instructor_transport.py new file mode 100644 index 00000000..b9ef9cea --- /dev/null +++ b/atomic-agents/tests/observability/instructor_transport.py @@ -0,0 +1,93 @@ +"""Offline Instructor clients using real OpenAI request and validation paths.""" + +from dataclasses import dataclass + +import httpx +import instructor +import openai +from openai.types import CompletionUsage +from openai.types.completion_usage import CompletionTokensDetails, PromptTokensDetails + + +@dataclass(frozen=True) +class ScriptedResponse: + arguments: str = '{"answer": "42"}' + input_tokens: int = 3 + output_tokens: int = 2 + cached_tokens: int = 1 + reasoning_tokens: int = 2 + schema_name: str = "Answer" + + def chat_usage(self) -> CompletionUsage: + return CompletionUsage( + prompt_tokens=self.input_tokens, + completion_tokens=self.output_tokens, + total_tokens=self.input_tokens + self.output_tokens, + prompt_tokens_details=PromptTokensDetails(cached_tokens=self.cached_tokens), + completion_tokens_details=CompletionTokensDetails(reasoning_tokens=self.reasoning_tokens), + ) + + def responses_body(self) -> dict[str, object]: + return { + "id": "resp_test", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "response-model", + "output": [ + { + "type": "function_call", + "id": "fc_test", + "call_id": "call_test", + "name": self.schema_name, + "arguments": self.arguments, + "status": "completed", + } + ], + "usage": { + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + "total_tokens": self.input_tokens + self.output_tokens, + "input_tokens_details": {"cached_tokens": self.cached_tokens}, + "output_tokens_details": {"reasoning_tokens": self.reasoning_tokens}, + }, + } + + def chat_body(self, mode: instructor.Mode) -> dict[str, object]: + message: dict[str, object] = {"role": "assistant", "content": self.arguments} + if mode == instructor.Mode.TOOLS: + message = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_test", + "type": "function", + "function": {"name": self.schema_name, "arguments": self.arguments}, + } + ], + } + return { + "id": "chatcmpl_test", + "object": "chat.completion", + "created": 1, + "model": "response-model", + "choices": [{"index": 0, "message": message, "finish_reason": "stop"}], + "usage": self.chat_usage().model_dump(), + } + + +def instructor_client(mode: instructor.Mode, responses: list[ScriptedResponse]) -> instructor.Instructor: + scripted = iter(responses) + + def respond(request: httpx.Request) -> httpx.Response: + response = next(scripted) + body = response.responses_body() if mode == instructor.Mode.RESPONSES_TOOLS else response.chat_body(mode) + expected_path = "/v1/responses" if mode == instructor.Mode.RESPONSES_TOOLS else "/v1/chat/completions" + if request.url.path != expected_path: + raise AssertionError(f"Unexpected transport path: {request.url.path}") + return httpx.Response(200, json=body) + + return instructor.from_openai( + openai.OpenAI(api_key="test", http_client=httpx.Client(transport=httpx.MockTransport(respond))), mode=mode + ) diff --git a/atomic-agents/tests/observability/test_llm_attempt_spans.py b/atomic-agents/tests/observability/test_llm_attempt_spans.py index 68ac6c97..8c21b822 100644 --- a/atomic-agents/tests/observability/test_llm_attempt_spans.py +++ b/atomic-agents/tests/observability/test_llm_attempt_spans.py @@ -11,8 +11,11 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter -from opentelemetry.trace import StatusCode +from opentelemetry.trace import SpanKind, StatusCode from pydantic import ValidationError +from instructor.core.exceptions import InstructorRetryException + +from instructor_transport import ScriptedResponse, instructor_client from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema from atomic_agents.observability import TracingConfig, using_conversation @@ -42,12 +45,7 @@ def exporter(): def usage(input_tokens=3, output_tokens=2): - return SimpleNamespace( - prompt_tokens=input_tokens, - completion_tokens=output_tokens, - prompt_tokens_details=SimpleNamespace(cached_tokens=1), - completion_tokens_details=SimpleNamespace(reasoning_tokens=2), - ) + return ScriptedResponse(input_tokens=input_tokens, output_tokens=output_tokens).chat_usage() def finished_spans(exporter, count): @@ -84,11 +82,9 @@ def agent(client): def retry_script(hooks, kwargs): hooks.emit_completion_arguments(**kwargs) hooks.emit_completion_response(response()) - hooks.emit_completion_usage(usage(), attempt_number=1) hooks.emit_parse_error(ValueError("invalid output")) hooks.emit_completion_arguments(**kwargs) hooks.emit_completion_response(response()) - hooks.emit_completion_usage(usage(6, 4), attempt_number=2) def run_agent(client): @@ -96,14 +92,20 @@ def run_agent(client): def test_retry_spans_usage_validation_and_parentage(exporter): - client = scripted_client(retry_script) + client = instructor_client( + instructor.Mode.TOOLS, [ScriptedResponse(arguments='{"answer": {"secret": "private output"}}'), ScriptedResponse()] + ) with using_conversation("conversation"): run_agent(client) first, second, parent = finished_spans(exporter, 3) assert [first.name, second.name] == ["chat test-model", "chat test-model"] assert [first.attributes["atomic_agents.llm.attempt"], second.attributes["atomic_agents.llm.attempt"]] == [1, 2] assert first.parent.span_id == second.parent.span_id == parent.context.span_id - assert first.status.status_code == second.status.status_code == StatusCode.OK + assert first.status.status_code == StatusCode.ERROR + assert second.status.status_code == StatusCode.OK + assert first.kind == second.kind == SpanKind.CLIENT + assert first.events[0].name == "validation_error" + assert first.attributes["error.type"] == "ValidationError" assert first.attributes["gen_ai.provider.name"] == "openai" assert first.attributes["gen_ai.conversation.id"] == "conversation" assert first.attributes["gen_ai.usage.input_tokens"] == second.attributes["gen_ai.usage.input_tokens"] == 3 @@ -113,7 +115,7 @@ def test_retry_spans_usage_validation_and_parentage(exporter): assert parent.attributes["gen_ai.usage.input_tokens"] == 6 assert parent.attributes["gen_ai.usage.output_tokens"] == 4 assert parent.attributes.get("atomic_agents.llm.attempts") == 2 - assert parent.attributes["atomic_agents.llm.last_validation_error"] == "invalid output" + assert "private output" in parent.attributes["atomic_agents.llm.last_validation_error"] assert parent.attributes["gen_ai.response.model"] == first.attributes["gen_ai.response.model"] == "response-model" event = parent.events[0] assert event.name == "validation_error" @@ -133,7 +135,7 @@ def test_completion_error_is_bounded_and_ends_attempt(exporter): installed = installation._current_installation() installation.install(tracer_provider=installed.tracer_provider, config=TracingConfig(max_attribute_length=32)) error = ValueError("failed " * 20) - expected = str(error)[:32] + "...[truncated]" + expected = str(error)[:18] + "...[truncated]" def script(hooks, kwargs): hooks.emit_completion_arguments(**kwargs) @@ -144,10 +146,10 @@ def script(hooks, kwargs): child, parent = finished_spans(exporter, 2) assert child.status.status_code == StatusCode.ERROR assert child.attributes["error.type"] == "ValueError" - assert child.events[0].attributes["exception.message"] == expected - assert child.events[0].attributes["exception.stacktrace"] == expected + assert child.events[-1].attributes["exception.message"] == expected + assert len(child.events[-1].attributes["exception.stacktrace"]) <= 32 assert child.status.description == expected - assert len(child.status.description) == 32 + len("...[truncated]") + assert len(child.status.description) == 32 assert parent.attributes["atomic_agents.llm.last_validation_error"] == expected assert parent.attributes["atomic_agents.llm.last_validation_error.original_length"] == len(str(error)) assert parent.status.status_code == StatusCode.OK @@ -159,10 +161,10 @@ def test_registration_once_and_uninstall_preserves_application_hooks(exporter): client.hooks.on("completion:response", application_hook) for _ in range(3): run_agent(client) - assert client.on.call_count == 5 + assert client.on.call_count == 4 assert application_hook.call_count == 6 installation.uninstall() - assert client.off.call_count == 5 + assert client.off.call_count == 4 client.hooks.emit_completion_response(response()) assert application_hook.call_count == 7 assert not llm_attempt_spans._clients @@ -176,8 +178,7 @@ async def invoke(number): async def complete(agent, *args, **kwargs): client.hooks.emit_completion_arguments(model="test-model") await asyncio.sleep(0) - client.hooks.emit_completion_response(response()) - client.hooks.emit_completion_usage(usage(number), attempt_number=1) + client.hooks.emit_completion_response(SimpleNamespace(model="response-model", usage=usage(number))) return Answer(answer="ok") if asynchronous: @@ -257,9 +258,9 @@ def script(hooks, kwargs): hooks.emit_completion_arguments(**kwargs) with pytest.raises(ValidationError) as invalid: Answer.model_validate({"answer": {"secret": "private output"}}) + hooks.emit_completion_response(SimpleNamespace(model="response-model", usage=usage(9, 4))) hooks.emit_parse_error(invalid.value) hooks.emit_completion_error(invalid.value, attempt_number=1, max_attempts=1, is_last_attempt=True) - hooks.emit_completion_usage(SimpleNamespace(input_tokens=9, output_tokens=4), attempt_number=1) run_agent(scripted_client(script)) child, parent = finished_spans(exporter, 2) @@ -267,8 +268,8 @@ def script(hooks, kwargs): assert parent.events[0].attributes["atomic_agents.llm.last_validation_error"] == "__REDACTED__" assert parent.attributes["gen_ai.usage.input_tokens"] == 9 assert child.status.description == "ValidationError: 1 errors (__REDACTED__)" - assert child.events[0].attributes["exception.message"] == child.status.description - assert child.events[0].attributes["exception.stacktrace"] == child.status.description + assert child.events[-1].attributes["exception.message"] == child.status.description + assert "ValidationError: 1 errors (__REDACTED__)" in child.events[-1].attributes["exception.stacktrace"] assert "private output" not in str(child.events) @@ -332,11 +333,10 @@ def script(hooks, kwargs): hooks.emit_completion_arguments(**kwargs) llm_attempt_spans._parse_error(ValueError("long error"), future_option=True) llm_attempt_spans._completion_response(response(), future_option=True) - llm_attempt_spans._completion_usage(usage(), attempt_number=1, future_option=True) run_agent(scripted_client(script)) child, parent = finished_spans(exporter, 2) - assert parent.attributes["atomic_agents.llm.last_validation_error"] == "long...[truncated]" + assert parent.attributes["atomic_agents.llm.last_validation_error"] == "...[" assert parent.attributes["atomic_agents.llm.last_validation_error.original_length"] == 10 @@ -373,7 +373,7 @@ def second_run(): release_registration.set() first.result(timeout=5) second.result(timeout=5) - assert len(client.hooks._handlers) == 5 + assert len(client.hooks._handlers) == 4 assert all(len(handlers) == 1 for handlers in client.hooks._handlers.values()) installation.uninstall() assert all(not handlers for handlers in client.hooks._handlers.values()) @@ -410,13 +410,11 @@ def script(hooks, kwargs): else: hooks.emit_completion_error(ValueError("suppressed"), attempt_number=1, max_attempts=2, is_last_attempt=False) hooks.emit_parse_error(ValueError("suppressed")) - hooks.emit_completion_usage(usage(99, 99), attempt_number=1) assert run.attempt_count == 0 assert run.open_attempt is None assert not exporter.get_finished_spans() hooks.emit_completion_arguments(**kwargs) hooks.emit_completion_response(response()) - hooks.emit_completion_usage(usage(), attempt_number=2) run_agent(scripted_client(script)) child, parent = finished_spans(exporter, 2) @@ -424,3 +422,118 @@ def script(hooks, kwargs): assert parent.attributes["atomic_agents.llm.attempts"] == 1 assert parent.attributes["gen_ai.usage.input_tokens"] == 3 assert not parent.events + + +@pytest.mark.parametrize( + "mode", [instructor.Mode.TOOLS, instructor.Mode.JSON, instructor.Mode.JSON_SCHEMA, instructor.Mode.RESPONSES_TOOLS] +) +def test_real_transport_usage_is_accumulated_per_response(exporter, mode): + client = instructor_client( + mode, + [ + ScriptedResponse(arguments='{"answer": {"private": "model text"}}'), + ScriptedResponse(input_tokens=7, output_tokens=5, cached_tokens=3, reasoning_tokens=4), + ], + ) + assert run_agent(client).answer == "42" + first, second, parent = finished_spans(exporter, 3) + assert first.status.status_code == StatusCode.ERROR + assert first.events[0].name == "validation_error" + assert second.status.status_code == StatusCode.OK + assert first.attributes["gen_ai.usage.input_tokens"] == 3 + assert second.attributes["gen_ai.usage.input_tokens"] == 7 + assert parent.attributes["gen_ai.usage.input_tokens"] == 10 + assert parent.attributes["gen_ai.usage.output_tokens"] == 7 + assert parent.attributes["gen_ai.usage.cache_read.input_tokens"] == 4 + assert parent.attributes["gen_ai.usage.reasoning.output_tokens"] == 6 + assert parent.attributes["atomic_agents.llm.attempts"] == 2 + assert "completion:usage" not in llm_attempt_spans._HANDLERS + + +@pytest.mark.parametrize("hidden", [False, True]) +def test_real_retry_exhaustion_redacts_or_bounds_every_exception(exporter, hidden): + installed = installation._current_installation() + installation.install( + tracer_provider=installed.tracer_provider, config=TracingConfig(hide_outputs=hidden, max_attribute_length=256) + ) + secret = "private model text " * 100 + client = instructor_client( + instructor.Mode.TOOLS, [ScriptedResponse(arguments='{"answer": {"secret": "' + secret + '"}}')] * 3 + ) + configured = agent(client) + configured.model_api_parameters["max_retries"] = 2 + with pytest.raises(InstructorRetryException): + configured.run(Question(question="test")) + first, second, third, parent = finished_spans(exporter, 4) + assert parent.status.status_code == StatusCode.ERROR + assert parent.attributes["atomic_agents.llm.attempts"] == 3 + assert parent.attributes["gen_ai.usage.input_tokens"] == 9 + message = parent.events[-1].attributes["exception.message"] + assert len(message) <= 256 + assert len(parent.status.description) <= 256 + assert len(parent.events[-1].attributes["exception.stacktrace"]) <= 256 + if hidden: + assert message == parent.status.description == "InstructorRetryException: 3 attempts (__REDACTED__)" + assert "private model text" not in str(parent.events) + assert "private model text" not in str(first.events) + str(second.events) + str(third.events) + assert ( + first.events[0].attributes["atomic_agents.llm.last_validation_error"] == "ValidationError: 1 errors (__REDACTED__)" + ) + else: + assert message.endswith("...[truncated]") + + +def test_direct_instructor_calls_inside_run_add_to_agent_totals(exporter): + client = instructor_client(instructor.Mode.TOOLS, [ScriptedResponse(), ScriptedResponse(input_tokens=11)]) + + def call_twice(configured): + for _ in range(2): + client.chat.completions.create( + model="test-model", response_model=Answer, messages=[{"role": "user", "content": "test"}] + ) + return Answer(answer="42") + + agent_spans._trace_agent_call(call_twice)(agent(client)) + first, second, parent = finished_spans(exporter, 3) + assert parent.attributes["gen_ai.usage.input_tokens"] == 14 + assert parent.attributes["atomic_agents.llm.attempts"] == 2 + assert first.parent.span_id == second.parent.span_id == parent.context.span_id + + +def test_missing_model_and_final_completion_error_still_count(exporter): + def script(hooks, kwargs): + hooks.emit_completion_arguments() + hooks.emit_completion_response(response()) + hooks.emit_completion_arguments() + hooks.emit_completion_error(RuntimeError("transport failed")) + + run_agent(scripted_client(script)) + first, second, parent = finished_spans(exporter, 3) + assert first.name == second.name == "chat" + assert first.kind == second.kind == SpanKind.CLIENT + assert second.status.status_code == StatusCode.ERROR + assert parent.attributes["atomic_agents.llm.attempts"] == 2 + assert parent.attributes["gen_ai.usage.input_tokens"] == 3 + + +@pytest.mark.parametrize("chain", ["__cause__", "__context__"]) +def test_wrapped_validation_error_and_tool_exception_are_redacted(exporter, chain): + from atomic_agents.observability import tool_spans + from atomic_agents.base.base_tool import BaseTool + + installed = installation._current_installation() + installation.install(tracer_provider=installed.tracer_provider, config=TracingConfig(hide_outputs=True)) + with pytest.raises(ValidationError) as invalid: + Answer.model_validate({"answer": {"secret": "private model text"}}) + wrapped = RuntimeError("private wrapper text") + setattr(wrapped, chain, invalid.value) + with pytest.raises(RuntimeError), tool_spans._tool_span(Mock(spec=BaseTool, tool_name="test"), (), {}): + raise wrapped + (span,) = finished_spans(exporter, 1) + assert span.status.description == "RuntimeError: 1 errors (__REDACTED__)" + assert span.events[0].attributes["exception.message"] == span.status.description + stacktrace = span.events[0].attributes["exception.stacktrace"] + assert "Traceback (most recent call last)" in stacktrace + assert stacktrace.rstrip().endswith(span.status.description) + assert "private model text" not in stacktrace + assert "private wrapper text" not in stacktrace diff --git a/atomic-agents/tests/observability/test_tracing.py b/atomic-agents/tests/observability/test_tracing.py index 66e6036a..74b4da6b 100644 --- a/atomic-agents/tests/observability/test_tracing.py +++ b/atomic-agents/tests/observability/test_tracing.py @@ -940,7 +940,7 @@ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema: result = LargeResultTool().run(CalculationInputSchema(expression="large")) (span,) = spans_of(exporter) serialized = result.model_dump_json() - expected = "__REDACTED__" if hide_outputs else serialized[:4096] + "...[truncated]" + expected = "__REDACTED__" if hide_outputs else serialized[: 4096 - len("...[truncated]")] + "...[truncated]" assert span.attributes["gen_ai.tool.call.result"] == expected assert span.attributes["output.value"] == expected if hide_outputs: @@ -1270,11 +1270,11 @@ def test_error_status_and_recorded_message_are_bounded(configured_tracing): with pytest.raises(ValueError) as raised, agent_spans._agent_span(agent, (), {}): raise error (span,) = spans_of(exporter) - expected = str(error)[:12] + "...[truncated]" + expected = "...[truncate" assert raised.value is error assert span.status.description == expected assert span.events[0].attributes["exception.message"] == expected - assert len(span.events[0].attributes["exception.stacktrace"]) <= 12 + len("...[truncated]") + assert len(span.events[0].attributes["exception.stacktrace"]) <= 12 def test_hidden_validation_errors_do_not_export_model_output(configured_tracing): @@ -1299,5 +1299,5 @@ def test_tool_parameters_use_the_payload_size_limit(configured_tracing): CalculatorTool().run(CalculationInputSchema(expression="x")) (span,) = spans_of(exporter) parameters = json.dumps(CalculationInputSchema.model_json_schema()) - assert span.attributes["tool.parameters"] == parameters[:12] + "...[truncated]" + assert span.attributes["tool.parameters"] == "...[truncate" assert span.attributes["tool.parameters.original_length"] == len(parameters) From fd7634a1d182092c6e04c99f2eb03547508c6f4a Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 13:01:40 +0200 Subject: [PATCH 19/25] Preserve installed tracing settings and tolerate invalid environment limits Co-Authored-By: Claude Code --- .../observability/installation.py | 10 ++-- .../observability/tracing_config.py | 27 ++++++++-- .../tests/observability/test_tracing.py | 50 ++++++++++++++++++- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/atomic-agents/atomic_agents/observability/installation.py b/atomic-agents/atomic_agents/observability/installation.py index f8f6d6e8..08e8cd73 100644 --- a/atomic-agents/atomic_agents/observability/installation.py +++ b/atomic-agents/atomic_agents/observability/installation.py @@ -59,12 +59,10 @@ def install( *, tracer_provider: Optional["TracerProvider"] = None, config: TracingConfig | PayloadVisibility | None = None ) -> None: global _installation - config = TracingConfig.coerce(config) if _installation is not None: - if config.openinference_attributes is None: - config = replace(config, openinference_attributes=_installation.config.openinference_attributes) _replace_installation(_installation, tracer_provider, config) return + config = TracingConfig.coerce(config) context_attributes = span_attributes._import_context_attributes() if config.openinference_attributes is None: config = replace(config, openinference_attributes=context_attributes is not None) @@ -90,8 +88,12 @@ def uninstall() -> None: def _replace_installation( - installed: _Installation, tracer_provider: Optional["TracerProvider"], config: TracingConfig + installed: _Installation, tracer_provider: Optional["TracerProvider"], config: TracingConfig | PayloadVisibility | None ) -> None: + config = installed.config if config is None else TracingConfig.coerce(config) + tracer_provider = installed.tracer_provider if tracer_provider is None else tracer_provider + if config.openinference_attributes is None: + config = replace(config, openinference_attributes=installed.config.openinference_attributes) if tracer_provider is installed.tracer_provider and config == installed.config: return installed.tracer = trace_api.get_tracer(TRACER_NAME, __version__, tracer_provider) diff --git a/atomic-agents/atomic_agents/observability/tracing_config.py b/atomic-agents/atomic_agents/observability/tracing_config.py index f0200874..62029157 100644 --- a/atomic-agents/atomic_agents/observability/tracing_config.py +++ b/atomic-agents/atomic_agents/observability/tracing_config.py @@ -1,10 +1,15 @@ """Tracing configuration and payload visibility.""" +import logging import os from dataclasses import dataclass from typing import Protocol +logger = logging.getLogger("atomic_agents.observability.tracing") +DEFAULT_MAX_ATTRIBUTE_LENGTH = 4096 + + class PayloadVisibility(Protocol): hide_inputs: bool hide_outputs: bool @@ -14,7 +19,7 @@ class PayloadVisibility(Protocol): class TracingConfig: hide_inputs: bool = False hide_outputs: bool = False - max_attribute_length: int = 4096 + max_attribute_length: int = DEFAULT_MAX_ATTRIBUTE_LENGTH openinference_attributes: bool | None = None llm_spans: bool = True @@ -23,7 +28,7 @@ def from_environment(cls) -> "TracingConfig": return cls( hide_inputs=_visibility_environment("INPUTS").lower() in ("true", "1"), hide_outputs=_visibility_environment("OUTPUTS").lower() in ("true", "1"), - max_attribute_length=int(os.environ.get("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", "4096")), + max_attribute_length=_attribute_length_environment(), llm_spans=os.environ.get("ATOMIC_AGENTS_TRACE_LLM_SPANS", "true").lower() not in ("false", "0"), ) @@ -33,8 +38,24 @@ def coerce(cls, config: "TracingConfig | PayloadVisibility | None") -> "TracingC return cls.from_environment() if isinstance(config, cls): return config - return cls(hide_inputs=config.hide_inputs, hide_outputs=config.hide_outputs) + return cls( + hide_inputs=config.hide_inputs, + hide_outputs=config.hide_outputs, + max_attribute_length=_attribute_length_environment(), + ) def _visibility_environment(direction: str) -> str: return os.environ.get(f"ATOMIC_AGENTS_TRACE_HIDE_{direction}", os.environ.get(f"OPENINFERENCE_HIDE_{direction}", "")) + + +def _attribute_length_environment() -> int: + value = os.environ.get("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", str(DEFAULT_MAX_ATTRIBUTE_LENGTH)) + try: + length = int(value) + if length >= 1: + return length + except ValueError: + pass + logger.warning("Invalid ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH value %r; using %d", value, DEFAULT_MAX_ATTRIBUTE_LENGTH) + return DEFAULT_MAX_ATTRIBUTE_LENGTH diff --git a/atomic-agents/tests/observability/test_tracing.py b/atomic-agents/tests/observability/test_tracing.py index 4b971838..e555cac0 100644 --- a/atomic-agents/tests/observability/test_tracing.py +++ b/atomic-agents/tests/observability/test_tracing.py @@ -992,7 +992,7 @@ def test_tracing_config_environment_and_coercion(monkeypatch): assert config == TracingConfig(hide_inputs=True, hide_outputs=True, max_attribute_length=128) assert TracingConfig.coerce(None) == config assert TracingConfig.coerce(config) is config - assert TracingConfig.coerce(TraceConfig(hide_inputs=True)) == TracingConfig(hide_inputs=True) + assert TracingConfig.coerce(TraceConfig(hide_inputs=True)) == TracingConfig(hide_inputs=True, max_attribute_length=128) @pytest.mark.parametrize( @@ -1320,3 +1320,51 @@ def test_tool_parameters_use_the_payload_size_limit(configured_tracing): parameters = json.dumps(CalculationInputSchema.model_json_schema()) assert span.attributes["tool.parameters"] == "...[truncate" assert span.attributes["tool.parameters.original_length"] == len(parameters) + + +@pytest.mark.parametrize("value, expected", [("", 4096), ("4k", 4096), ("0", 4096), ("-5", 4096), ("512", 512)]) +@pytest.mark.parametrize("visibility", [None, TraceConfig(hide_inputs=True)]) +def test_attribute_length_environment_is_tolerant(monkeypatch, caplog, value, expected, visibility): + monkeypatch.setenv("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", value) + with caplog.at_level(logging.WARNING): + config = tracing.TracingConfig.coerce(visibility) + assert config.max_attribute_length == expected + assert config.hide_inputs is (visibility is not None) + warnings = [record for record in caplog.records if record.levelno == logging.WARNING] + if expected == 4096: + assert len(warnings) == 1 + assert repr(value) in warnings[0].message + else: + assert warnings == [] + + +def test_reinstall_without_config_preserves_redaction(exporter, caplog): + config = tracing.TracingConfig(hide_inputs=True, openinference_attributes=False) + AtomicAgentsInstrumentor().instrument(tracer_provider=exporter.provider, config=config) + other_provider = TracerProvider() + other_exporter = InMemorySpanExporter() + other_provider.add_span_processor(SimpleSpanProcessor(other_exporter)) + caplog.clear() + try: + with caplog.at_level(logging.WARNING): + AtomicAgentsInstrumentor().instrument(tracer_provider=other_provider) + assert installation._current_installation().config is config + assert installation._current_installation().tracer_provider is other_provider + make_agent(sync_client()).run(QuestionSchema(question="secret")) + assert spans_of(other_exporter)[0].attributes["atomic_agents.input"] == "__REDACTED__" + assert len(caplog.records) == 1 + assert "replaced" in caplog.records[0].message + finally: + other_provider.shutdown() + + +def test_reinstall_without_provider_preserves_provider(exporter, caplog): + config = tracing.TracingConfig(hide_outputs=True) + with caplog.at_level(logging.WARNING): + AtomicAgentsInstrumentor().instrument(config=config) + assert installation._current_installation().tracer_provider is exporter.provider + caplog.clear() + AtomicAgentsInstrumentor().instrument() + assert caplog.records == [] + make_agent(sync_client()).run(QuestionSchema(question="?")) + assert spans_of(exporter)[0].attributes["atomic_agents.output"] == "__REDACTED__" From 33e98b52f8048607e185e44740af662b6910a2c3 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 13:03:12 +0200 Subject: [PATCH 20/25] Leave cancelled agent spans unset without exception events Co-Authored-By: Claude Code --- .../observability/agent_spans.py | 6 +++ .../tests/observability/test_tracing.py | 53 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/atomic-agents/atomic_agents/observability/agent_spans.py b/atomic-agents/atomic_agents/observability/agent_spans.py index cbfe21d1..ffba0839 100644 --- a/atomic-agents/atomic_agents/observability/agent_spans.py +++ b/atomic-agents/atomic_agents/observability/agent_spans.py @@ -61,6 +61,9 @@ def _agent_span(agent: AtomicAgent, args: tuple[object, ...], kwargs: dict[str, try: with llm_attempt_spans.active_run(run): yield span + except (asyncio.CancelledError, GeneratorExit): + span.set_attribute("atomic_agents.cancelled", True) + raise except BaseException as error: _record_failure(span, error) raise @@ -199,6 +202,9 @@ def traced(agent: AtomicAgent, *args: object, **kwargs: object) -> Generator[Out except GeneratorExit: cursor.close_early() raise + except asyncio.CancelledError: + _mark_closed_early(span, cursor.output) + raise except BaseException as error: _record_failure(span, error) raise diff --git a/atomic-agents/tests/observability/test_tracing.py b/atomic-agents/tests/observability/test_tracing.py index e555cac0..75be4d23 100644 --- a/atomic-agents/tests/observability/test_tracing.py +++ b/atomic-agents/tests/observability/test_tracing.py @@ -1368,3 +1368,56 @@ def test_reinstall_without_provider_preserves_provider(exporter, caplog): assert caplog.records == [] make_agent(sync_client()).run(QuestionSchema(question="?")) assert spans_of(exporter)[0].attributes["atomic_agents.output"] == "__REDACTED__" + + +@pytest.mark.asyncio +async def test_run_async_cancellation_leaves_span_unset(exporter): + started = asyncio.Event() + release = asyncio.Event() + + async def create(**kwargs): + started.set() + await release.wait() + return AnswerSchema(answer="never") + + client = async_client() + client.chat.completions.create = create + task = asyncio.create_task(make_agent(client).run_async(QuestionSchema(question="?"))) + await asyncio.wait_for(started.wait(), timeout=2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + (span,) = spans_of(exporter) + assert span.attributes.get("atomic_agents.cancelled") is True + assert span.status.status_code == StatusCode.UNSET + assert span.events == () + assert "error.type" not in span.attributes + + +@pytest.mark.parametrize("error_type", [GeneratorExit, asyncio.CancelledError]) +def test_run_cancellation_leaves_span_unset(exporter, error_type): + client = sync_client() + client.chat.completions.create.side_effect = error_type + with pytest.raises(error_type): + make_agent(client).run(QuestionSchema(question="?")) + (span,) = spans_of(exporter) + assert span.attributes.get("atomic_agents.cancelled") is True + assert span.status.status_code == StatusCode.UNSET + assert span.events == () + + +def test_run_stream_cancellation_marks_closed_early(exporter): + def create_partial(**kwargs): + yield AnswerSchema(answer="4") + raise asyncio.CancelledError + + client = sync_client() + client.chat.completions.create_partial = create_partial + stream = make_agent(client).run_stream(QuestionSchema(question="?")) + next(stream) + with pytest.raises(asyncio.CancelledError): + next(stream) + (span,) = spans_of(exporter) + assert span.attributes.get(tracing.ATTR_STREAM_CLOSED_EARLY) is True + assert span.status.status_code == StatusCode.UNSET + assert span.events == () From bc06074ff3263bd0a01505038ba56dd9b295f341 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 13:05:25 +0200 Subject: [PATCH 21/25] Keep shared-client hooks and tolerate uninitialized agent names Co-Authored-By: Claude Code --- .../atomic_agents/agents/atomic_agent.py | 15 ++++------- .../observability/span_attributes.py | 3 ++- .../tests/agents/test_agent_hooks.py | 26 +++++++++++++++++++ .../tests/observability/test_tracing.py | 21 +++++++++++++++ 4 files changed, 54 insertions(+), 11 deletions(-) create mode 100644 atomic-agents/tests/agents/test_agent_hooks.py diff --git a/atomic-agents/atomic_agents/agents/atomic_agent.py b/atomic-agents/atomic_agents/agents/atomic_agent.py index 5189986a..76d2423b 100644 --- a/atomic-agents/atomic_agents/agents/atomic_agent.py +++ b/atomic-agents/atomic_agents/agents/atomic_agent.py @@ -799,16 +799,11 @@ def clear_hooks(self, event: Optional[str] = None) -> None: Args: event (Optional[str]): The event name to clear, or None to clear all """ - if event: - if event in self._hook_handlers: - # Clear from instructor client first - if hasattr(self.client, "clear"): - self.client.clear(event) - self._hook_handlers[event].clear() - else: - # Clear all hooks - if hasattr(self.client, "clear"): - self.client.clear() + events = [event] if event else list(self._hook_handlers) + for registered_event in events: + for handler in list(self._hook_handlers.get(registered_event, [])): + self.unregister_hook(registered_event, handler) + if not event: self._hook_handlers.clear() def _dispatch_hook(self, event: str, *args, **kwargs) -> None: diff --git a/atomic-agents/atomic_agents/observability/span_attributes.py b/atomic-agents/atomic_agents/observability/span_attributes.py index 6a2e73b1..ccbf615b 100644 --- a/atomic-agents/atomic_agents/observability/span_attributes.py +++ b/atomic-agents/atomic_agents/observability/span_attributes.py @@ -98,7 +98,8 @@ def using_conversation(conversation_id: str) -> Iterator[None]: def _agent_display_name(agent: AtomicAgent) -> str: - return agent.name or type(agent).__name__ + attributes = _best_effort_attributes({ATTR_AGENT_NAME: lambda: getattr(agent, "name", None)}) + return str(attributes.get(ATTR_AGENT_NAME) or type(agent).__name__) def _agent_class_name(agent: AtomicAgent) -> str: diff --git a/atomic-agents/tests/agents/test_agent_hooks.py b/atomic-agents/tests/agents/test_agent_hooks.py new file mode 100644 index 00000000..0afedfbf --- /dev/null +++ b/atomic-agents/tests/agents/test_agent_hooks.py @@ -0,0 +1,26 @@ +import instructor +import pytest +from unittest.mock import Mock + +from atomic_agents import AgentConfig, AtomicAgent + + +@pytest.mark.parametrize("event", [None, "completion:kwargs"]) +def test_clear_hooks_preserves_foreign_handlers(event): + client = instructor.Instructor(client=Mock(), create=Mock()) + agent = AtomicAgent(AgentConfig(client=client)) + foreign = Mock() + owned = Mock() + other_owned = Mock() + client.on("completion:kwargs", foreign) + agent.register_hook("completion:kwargs", owned) + agent.register_hook("completion:response", other_owned) + + agent.clear_hooks(event) + client.hooks.emit_completion_arguments(model="test-model") + client.hooks.emit_completion_response(Mock()) + + foreign.assert_called_once_with(model="test-model") + owned.assert_not_called() + assert other_owned.call_count == (0 if event is None else 1) + assert agent._hook_handlers.get("completion:kwargs", []) == [] diff --git a/atomic-agents/tests/observability/test_tracing.py b/atomic-agents/tests/observability/test_tracing.py index 75be4d23..d63dbc87 100644 --- a/atomic-agents/tests/observability/test_tracing.py +++ b/atomic-agents/tests/observability/test_tracing.py @@ -1421,3 +1421,24 @@ def create_partial(**kwargs): assert span.attributes.get(tracing.ATTR_STREAM_CLOSED_EARLY) is True assert span.status.status_code == StatusCode.UNSET assert span.events == () + + +@pytest.mark.parametrize("broken_name", [False, True]) +def test_agent_name_falls_back_without_initialized_base(exporter, broken_name): + class UninitializedAgent(AtomicAgent[QuestionSchema, AnswerSchema]): + def __init__(self): + self.client = sync_client() + + if broken_name: + UninitializedAgent.name = property(lambda self: 1 / 0) + agent = UninitializedAgent() + try: + display_name = span_attributes._agent_display_name(agent) + except (AttributeError, ZeroDivisionError): + display_name = None + assert display_name == "UninitializedAgent" + with agent_spans._agent_span(agent, (), {}): + pass + (span,) = spans_of(exporter) + assert span.name == "invoke_agent UninitializedAgent" + assert span.attributes["gen_ai.agent.name"] == "UninitializedAgent" From 5f88f91228d5412ad3ea213ae3b6f058c5e36044 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 13:07:40 +0200 Subject: [PATCH 22/25] Trim tracing facade and remove unused agent-span context Keep provider-default resolution at the installation boundary. Co-Authored-By: Claude Code --- .../atomic_agents/observability/__init__.py | 10 ++-- .../observability/agent_spans.py | 12 ----- .../observability/installation.py | 2 +- .../atomic_agents/observability/tracing.py | 35 ------------- .../tests/observability/test_tracing.py | 50 +++++++++++-------- 5 files changed, 32 insertions(+), 77 deletions(-) diff --git a/atomic-agents/atomic_agents/observability/__init__.py b/atomic-agents/atomic_agents/observability/__init__.py index d19c2e23..cd132aa8 100644 --- a/atomic-agents/atomic_agents/observability/__init__.py +++ b/atomic-agents/atomic_agents/observability/__init__.py @@ -1,11 +1,7 @@ -""" -Opt-in OpenTelemetry tracing for Atomic Agents. +"""Opt-in tracing for any OpenTelemetry backend using GenAI attributes. -Emits an AGENT span per ``AtomicAgent`` run and a TOOL span per ``BaseTool`` run, rendered by -Arize Phoenix and any other OpenTelemetry backend. Needs the ``observability`` extra: -``pip install "atomic-agents[observability]"``. Phoenix discovers :class:`AtomicAgentsInstrumentor` -through the ``openinference_instrumentor`` entry point, so ``phoenix.otel.register(auto_instrument=True)`` -activates it without further code. See ``docs/guides/observability.md``. +Install the observability extra for tracing. The optional phoenix extra adds +Arize Phoenix integration and OpenInference attributes. """ import importlib.util diff --git a/atomic-agents/atomic_agents/observability/agent_spans.py b/atomic-agents/atomic_agents/observability/agent_spans.py index ffba0839..d49000f4 100644 --- a/atomic-agents/atomic_agents/observability/agent_spans.py +++ b/atomic-agents/atomic_agents/observability/agent_spans.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -import contextvars import functools from contextlib import contextmanager from typing import AsyncGenerator, Awaitable, Callable, Coroutine, Generator, Generic, Iterator, TypeVar @@ -27,11 +26,6 @@ AGENT_ASYNC_STREAM_METHOD = "run_async_stream" -_active_agent_span: contextvars.ContextVar[Span | None] = contextvars.ContextVar( - "atomic_agents_active_agent_span", default=None -) - - def _user_input_from_call(args: tuple[object, ...], kwargs: dict[str, object]) -> object: return args[0] if args else kwargs.get("user_input") @@ -57,7 +51,6 @@ def _agent_span(agent: AtomicAgent, args: tuple[object, ...], kwargs: dict[str, ) as span: _set_input_safely(span, _user_input_from_call(args, kwargs)) run = llm_attempt_spans.new_run(span, agent) - token = _active_agent_span.set(span) try: with llm_attempt_spans.active_run(run): yield span @@ -69,7 +62,6 @@ def _agent_span(agent: AtomicAgent, args: tuple[object, ...], kwargs: dict[str, raise finally: llm_attempt_spans.finish_run(run) - _active_agent_span.reset(token) def _trace_agent_call(wrapped: Callable[..., OutputSchema]) -> Callable[..., OutputSchema]: @@ -114,24 +106,20 @@ def _start_agent_span(agent: AtomicAgent, args: tuple[object, ...], kwargs: dict def _call_within_span(run: llm_attempt_spans._AgentRun, function: Callable[..., CallResult], *args: object) -> CallResult: span = run.span token = context_api.attach(trace_api.set_span_in_context(span)) - agent_token = _active_agent_span.set(span) try: with llm_attempt_spans.active_run(run): return function(*args) finally: - _active_agent_span.reset(agent_token) context_api.detach(token) async def _await_within_span(run: llm_attempt_spans._AgentRun, awaitable: Awaitable[CallResult]) -> CallResult: span = run.span token = context_api.attach(trace_api.set_span_in_context(span)) - agent_token = _active_agent_span.set(span) try: with llm_attempt_spans.active_run(run): return await awaitable finally: - _active_agent_span.reset(agent_token) context_api.detach(token) diff --git a/atomic-agents/atomic_agents/observability/installation.py b/atomic-agents/atomic_agents/observability/installation.py index 08e8cd73..88052359 100644 --- a/atomic-agents/atomic_agents/observability/installation.py +++ b/atomic-agents/atomic_agents/observability/installation.py @@ -60,6 +60,7 @@ def install( ) -> None: global _installation if _installation is not None: + tracer_provider = _installation.tracer_provider if tracer_provider is None else tracer_provider _replace_installation(_installation, tracer_provider, config) return config = TracingConfig.coerce(config) @@ -91,7 +92,6 @@ def _replace_installation( installed: _Installation, tracer_provider: Optional["TracerProvider"], config: TracingConfig | PayloadVisibility | None ) -> None: config = installed.config if config is None else TracingConfig.coerce(config) - tracer_provider = installed.tracer_provider if tracer_provider is None else tracer_provider if config.openinference_attributes is None: config = replace(config, openinference_attributes=installed.config.openinference_attributes) if tracer_provider is installed.tracer_provider and config == installed.config: diff --git a/atomic-agents/atomic_agents/observability/tracing.py b/atomic-agents/atomic_agents/observability/tracing.py index f1c54e9a..4cb43e57 100644 --- a/atomic-agents/atomic_agents/observability/tracing.py +++ b/atomic-agents/atomic_agents/observability/tracing.py @@ -3,44 +3,9 @@ from . import installation, span_attributes, tracing_config TRACER_NAME = installation.TRACER_NAME -TRACED_MARKER = installation.TRACED_MARKER install = installation.install is_installed = installation.is_installed uninstall = installation.uninstall PayloadVisibility = tracing_config.PayloadVisibility TracingConfig = tracing_config.TracingConfig using_conversation = span_attributes.using_conversation - -ATTR_MODEL = span_attributes.ATTR_MODEL -ATTR_MODE = span_attributes.ATTR_MODE -ATTR_INPUT_SCHEMA = span_attributes.ATTR_INPUT_SCHEMA -ATTR_OUTPUT_SCHEMA = span_attributes.ATTR_OUTPUT_SCHEMA -ATTR_HISTORY_MESSAGE_COUNT = span_attributes.ATTR_HISTORY_MESSAGE_COUNT -ATTR_CONTEXT_PROVIDERS = span_attributes.ATTR_CONTEXT_PROVIDERS -ATTR_SYSTEM_PROMPT_SHA256 = span_attributes.ATTR_SYSTEM_PROMPT_SHA256 -ATTR_STREAM_CLOSED_EARLY = span_attributes.ATTR_STREAM_CLOSED_EARLY -ATTR_OPERATION = span_attributes.ATTR_OPERATION -ATTR_AGENT_NAME = span_attributes.ATTR_AGENT_NAME -ATTR_REQUEST_MODEL = span_attributes.ATTR_REQUEST_MODEL -ATTR_PROVIDER_NAME = span_attributes.ATTR_PROVIDER_NAME -ATTR_CONVERSATION_ID = span_attributes.ATTR_CONVERSATION_ID -ATTR_TOOL_NAME = span_attributes.ATTR_TOOL_NAME -ATTR_TOOL_DESCRIPTION = span_attributes.ATTR_TOOL_DESCRIPTION -ATTR_TOOL_TYPE = span_attributes.ATTR_TOOL_TYPE -ATTR_TOOL_ARGUMENTS = span_attributes.ATTR_TOOL_ARGUMENTS -ATTR_TOOL_RESULT = span_attributes.ATTR_TOOL_RESULT -ATTR_SPAN_KIND = span_attributes.ATTR_SPAN_KIND -ATTR_INPUT_VALUE = span_attributes.ATTR_INPUT_VALUE -ATTR_INPUT_MIME_TYPE = span_attributes.ATTR_INPUT_MIME_TYPE -ATTR_OUTPUT_VALUE = span_attributes.ATTR_OUTPUT_VALUE -ATTR_OUTPUT_MIME_TYPE = span_attributes.ATTR_OUTPUT_MIME_TYPE -ATTR_OI_AGENT_NAME = span_attributes.ATTR_OI_AGENT_NAME -ATTR_OI_TOOL_NAME = span_attributes.ATTR_OI_TOOL_NAME -ATTR_OI_TOOL_DESCRIPTION = span_attributes.ATTR_OI_TOOL_DESCRIPTION -ATTR_TOOL_PARAMETERS = span_attributes.ATTR_TOOL_PARAMETERS -ATTR_SESSION_ID = span_attributes.ATTR_SESSION_ID -ATTR_INPUT = span_attributes.ATTR_INPUT -ATTR_OUTPUT = span_attributes.ATTR_OUTPUT -ATTR_AGENT_CLASS = span_attributes.ATTR_AGENT_CLASS -ATTR_ERROR_TYPE = span_attributes.ATTR_ERROR_TYPE -ATTR_TOOL_CLASS = span_attributes.ATTR_TOOL_CLASS diff --git a/atomic-agents/tests/observability/test_tracing.py b/atomic-agents/tests/observability/test_tracing.py index d63dbc87..5713be29 100644 --- a/atomic-agents/tests/observability/test_tracing.py +++ b/atomic-agents/tests/observability/test_tracing.py @@ -190,18 +190,18 @@ def test_run_emits_agent_span_with_io_and_framework_attributes(exporter): assert span.name == "invoke_agent AtomicAgent" assert kind_of(span) == OpenInferenceSpanKindValues.AGENT.value assert span.attributes[SpanAttributes.AGENT_NAME] == "AtomicAgent" - assert span.attributes[tracing.ATTR_AGENT_CLASS] == "AtomicAgent[QuestionSchema, AnswerSchema]" + assert span.attributes[span_attributes.ATTR_AGENT_CLASS] == "AtomicAgent[QuestionSchema, AnswerSchema]" assert input_json(span) == {"question": "6*7?"} assert output_json(span) == {"answer": "42"} assert span.attributes[SpanAttributes.INPUT_MIME_TYPE] == "application/json" assert span.attributes[SpanAttributes.OUTPUT_MIME_TYPE] == "application/json" - assert span.attributes[tracing.ATTR_MODEL] == "test-model" - assert span.attributes[tracing.ATTR_MODE] == agent.mode.value - assert span.attributes[tracing.ATTR_INPUT_SCHEMA] == "QuestionSchema" - assert span.attributes[tracing.ATTR_OUTPUT_SCHEMA] == "AnswerSchema" - assert span.attributes[tracing.ATTR_HISTORY_MESSAGE_COUNT] == 0 - assert span.attributes[tracing.ATTR_CONTEXT_PROVIDERS] == () - assert len(span.attributes[tracing.ATTR_SYSTEM_PROMPT_SHA256]) == 64 + assert span.attributes[span_attributes.ATTR_MODEL] == "test-model" + assert span.attributes[span_attributes.ATTR_MODE] == agent.mode.value + assert span.attributes[span_attributes.ATTR_INPUT_SCHEMA] == "QuestionSchema" + assert span.attributes[span_attributes.ATTR_OUTPUT_SCHEMA] == "AnswerSchema" + assert span.attributes[span_attributes.ATTR_HISTORY_MESSAGE_COUNT] == 0 + assert span.attributes[span_attributes.ATTR_CONTEXT_PROVIDERS] == () + assert len(span.attributes[span_attributes.ATTR_SYSTEM_PROMPT_SHA256]) == 64 assert span.status.status_code == StatusCode.OK @@ -212,8 +212,8 @@ def test_second_turn_reports_history_size_entering_the_turn(exporter): agent.run(QuestionSchema(question="second")) first, second = spans_of(exporter) - assert first.attributes[tracing.ATTR_HISTORY_MESSAGE_COUNT] == 0 - assert second.attributes[tracing.ATTR_HISTORY_MESSAGE_COUNT] == 2 + assert first.attributes[span_attributes.ATTR_HISTORY_MESSAGE_COUNT] == 0 + assert second.attributes[span_attributes.ATTR_HISTORY_MESSAGE_COUNT] == 2 def test_agent_without_system_role_has_no_prompt_hash(exporter): @@ -222,7 +222,7 @@ def test_agent_without_system_role_has_no_prompt_hash(exporter): agent.run(QuestionSchema(question="?")) (span,) = spans_of(exporter) - assert tracing.ATTR_SYSTEM_PROMPT_SHA256 not in span.attributes + assert span_attributes.ATTR_SYSTEM_PROMPT_SHA256 not in span.attributes def test_abstract_intermediate_tool_base_is_not_wrapped(exporter): @@ -231,7 +231,7 @@ class AbstractCalculator(BaseTool[CalculationInputSchema, CalculationOutputSchem def run(self, params: CalculationInputSchema) -> CalculationOutputSchema: raise NotImplementedError - assert not hasattr(AbstractCalculator.__dict__["run"], tracing.TRACED_MARKER) + assert not hasattr(AbstractCalculator.__dict__["run"], installation.TRACED_MARKER) def test_run_without_input_sets_no_input_attribute(exporter): @@ -398,7 +398,7 @@ def create_partial(**kwargs): assert inner_closed == [True] (span,) = spans_of(exporter) - assert span.attributes[tracing.ATTR_STREAM_CLOSED_EARLY] is True + assert span.attributes[span_attributes.ATTR_STREAM_CLOSED_EARLY] is True assert output_json(span) == {"answer": "4"} assert span.status.status_code == StatusCode.OK @@ -462,7 +462,7 @@ async def test_run_async_stream_closed_early_ends_span(exporter): assert first.answer == "4" (span,) = spans_of(exporter) - assert span.attributes[tracing.ATTR_STREAM_CLOSED_EARLY] is True + assert span.attributes[span_attributes.ATTR_STREAM_CLOSED_EARLY] is True assert span.status.status_code == StatusCode.OK @@ -734,8 +734,11 @@ def test_prompt_hash_identifies_static_sections_despite_dynamic_providers(export agent.run(QuestionSchema(question="b")) first, second = spans_of(exporter) - assert first.attributes[tracing.ATTR_SYSTEM_PROMPT_SHA256] == second.attributes[tracing.ATTR_SYSTEM_PROMPT_SHA256] - assert first.attributes[tracing.ATTR_CONTEXT_PROVIDERS] == ("counter",) + assert ( + first.attributes[span_attributes.ATTR_SYSTEM_PROMPT_SHA256] + == second.attributes[span_attributes.ATTR_SYSTEM_PROMPT_SHA256] + ) + assert first.attributes[span_attributes.ATTR_CONTEXT_PROVIDERS] == ("counter",) def test_prompt_hash_changes_when_static_sections_change(exporter): @@ -746,7 +749,10 @@ def test_prompt_hash_changes_when_static_sections_change(exporter): agent.run(QuestionSchema(question="b")) first, second = spans_of(exporter) - assert first.attributes[tracing.ATTR_SYSTEM_PROMPT_SHA256] != second.attributes[tracing.ATTR_SYSTEM_PROMPT_SHA256] + assert ( + first.attributes[span_attributes.ATTR_SYSTEM_PROMPT_SHA256] + != second.attributes[span_attributes.ATTR_SYSTEM_PROMPT_SHA256] + ) def test_run_stream_dropped_without_close_still_ends_span(exporter): @@ -758,7 +764,7 @@ def test_run_stream_dropped_without_close_still_ends_span(exporter): gc.collect() (span,) = spans_of(exporter) - assert span.attributes[tracing.ATTR_STREAM_CLOSED_EARLY] is True + assert span.attributes[span_attributes.ATTR_STREAM_CLOSED_EARLY] is True @pytest.mark.asyncio @@ -785,7 +791,7 @@ async def consume(): await task (span,) = spans_of(exporter) - assert span.attributes[tracing.ATTR_STREAM_CLOSED_EARLY] is True + assert span.attributes[span_attributes.ATTR_STREAM_CLOSED_EARLY] is True assert output_json(span) == {"answer": "4"} assert span.status.status_code != StatusCode.ERROR @@ -837,8 +843,8 @@ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema: async def run_async(self, params: CalculationInputSchema): yield CalculationOutputSchema(result="chunk") - assert not hasattr(StreamingTool.__dict__["run_async"], tracing.TRACED_MARKER) - assert hasattr(StreamingTool.__dict__["run"], tracing.TRACED_MARKER) + assert not hasattr(StreamingTool.__dict__["run_async"], installation.TRACED_MARKER) + assert hasattr(StreamingTool.__dict__["run"], installation.TRACED_MARKER) def test_instrument_uninstrument_instrument_cycle_traces_tools_defined_between_cycles(): @@ -1418,7 +1424,7 @@ def create_partial(**kwargs): with pytest.raises(asyncio.CancelledError): next(stream) (span,) = spans_of(exporter) - assert span.attributes.get(tracing.ATTR_STREAM_CLOSED_EARLY) is True + assert span.attributes.get(span_attributes.ATTR_STREAM_CLOSED_EARLY) is True assert span.status.status_code == StatusCode.UNSET assert span.events == () From 82fc9a30e12eaad09fcee2a7f87d86d26436e125 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 13:07:41 +0200 Subject: [PATCH 23/25] Test Instructor modes and real-client fresh-install tracing Co-Authored-By: Claude Code --- .github/workflows/code-quality.yml | 71 ++++++--- .../observability/instructor_transport.py | 19 ++- .../observability/test_instructor_modes.py | 141 ++++++++++++++++++ 3 files changed, 205 insertions(+), 26 deletions(-) create mode 100644 atomic-agents/tests/observability/test_instructor_modes.py diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index fa41b319..9b47588a 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -96,9 +96,9 @@ jobs: - name: Trace a tool and agent with only OpenTelemetry run: | fresh-install-venv/bin/python - <<'PY' - from types import SimpleNamespace - - from instructor import Instructor + import httpx + import instructor + import openai from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter @@ -125,34 +125,43 @@ jobs: def run(self, params: SmokeInput) -> SmokeOutput: return SmokeOutput(text=params.text) - class StubCompletions: - def create(self, **options: object) -> SmokeOutput: - return SmokeOutput(text="offline response") - - class StubClient(Instructor): - chat = SimpleNamespace(completions=StubCompletions()) - provider = None - - def __init__(self) -> None: - pass - - def on(self, event: str, handler: object) -> None: - pass - - def off(self, event: str, handler: object) -> None: - pass + def respond(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/chat/completions" + return httpx.Response(200, json={ + "id": "chatcmpl_smoke", "object": "chat.completion", "created": 1, + "model": "response-model", + "choices": [{"index": 0, "finish_reason": "tool_calls", "message": { + "role": "assistant", "content": None, "tool_calls": [{ + "id": "call_smoke", "type": "function", "function": { + "name": "SmokeOutput", "arguments": '{"text": "offline response"}' + } + }] + }}], + "usage": { + "prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5, + "prompt_tokens_details": {"cached_tokens": 1}, + "completion_tokens_details": {"reasoning_tokens": 2}, + }, + }) + + client = instructor.from_openai(openai.OpenAI( + api_key="test", http_client=httpx.Client(transport=httpx.MockTransport(respond)) + )) class SmokeAgent(AtomicAgent[SmokeInput, SmokeOutput]): pass assert SmokeTool().run(SmokeInput(text="tool input")).text == "tool input" - agent = SmokeAgent(config=AgentConfig(client=StubClient(), model="offline-model")) + agent = SmokeAgent(config=AgentConfig(client=client, model="offline-model")) assert agent.run(SmokeInput(text="private agent input")).text == "offline response" spans = exporter.get_finished_spans() for span in spans: print(f"{span.name}: {sorted(span.attributes)}") spans_by_operation = {span.attributes["gen_ai.operation.name"]: span for span in spans} + assert len(spans) == 3 + assert len(spans_by_operation) == 3 + chat_span = spans_by_operation["chat"] tool_span = spans_by_operation["execute_tool"] agent_span = spans_by_operation["invoke_agent"] assert tool_span.name == f"execute_tool {SmokeTool().tool_name}" @@ -160,9 +169,27 @@ jobs: assert tool_span.attributes["gen_ai.tool.name"] == SmokeTool().tool_name assert tool_span.attributes["atomic_agents.tool.class"] == "SmokeTool" assert agent_span.attributes["atomic_agents.input"] == "__REDACTED__" - for span in (tool_span, agent_span): + assert agent_span.attributes["gen_ai.agent.name"] == "SmokeAgent" + assert agent_span.attributes["atomic_agents.llm.attempts"] == 1 + assert chat_span.name == "chat offline-model" + assert chat_span.parent.span_id == agent_span.context.span_id + for span in (agent_span, chat_span): + assert span.attributes["gen_ai.request.model"] == "offline-model" + assert span.attributes["gen_ai.response.model"] == "response-model" + assert span.attributes["gen_ai.provider.name"] == "openai" + assert span.attributes["gen_ai.usage.input_tokens"] == 3 + assert span.attributes["gen_ai.usage.output_tokens"] == 2 + assert span.attributes["gen_ai.usage.cache_read.input_tokens"] == 1 + assert span.attributes["gen_ai.usage.reasoning.output_tokens"] == 2 + forbidden_keys = { + "input.value", "output.value", "input.mime_type", "output.mime_type", + "agent.name", "tool.name", "tool.description", "tool.parameters", "session.id", + } + for span in spans: + assert forbidden_keys.isdisjoint(span.attributes) assert not any(key.startswith("openinference.") for key in span.attributes) - print("api-only tracing ok: tool and agent spans, GenAI attributes, redacted input, no OpenInference keys") + client.client.close() + print("api-only tracing ok: tool, agent and chat spans, real Instructor usage, redacted input, no OpenInference keys") PY - name: Import every subpackage the way a new user would diff --git a/atomic-agents/tests/observability/instructor_transport.py b/atomic-agents/tests/observability/instructor_transport.py index b9ef9cea..c1d2a9ad 100644 --- a/atomic-agents/tests/observability/instructor_transport.py +++ b/atomic-agents/tests/observability/instructor_transport.py @@ -77,17 +77,28 @@ def chat_body(self, mode: instructor.Mode) -> dict[str, object]: } -def instructor_client(mode: instructor.Mode, responses: list[ScriptedResponse]) -> instructor.Instructor: +def response_transport(mode: instructor.Mode, responses: list[ScriptedResponse]) -> httpx.MockTransport: scripted = iter(responses) def respond(request: httpx.Request) -> httpx.Response: response = next(scripted) - body = response.responses_body() if mode == instructor.Mode.RESPONSES_TOOLS else response.chat_body(mode) - expected_path = "/v1/responses" if mode == instructor.Mode.RESPONSES_TOOLS else "/v1/chat/completions" + body = response.responses_body() if mode.value == "responses_tools" else response.chat_body(mode) + expected_path = "/v1/responses" if mode.value == "responses_tools" else "/v1/chat/completions" if request.url.path != expected_path: raise AssertionError(f"Unexpected transport path: {request.url.path}") return httpx.Response(200, json=body) + return httpx.MockTransport(respond) + + +def instructor_client(mode: instructor.Mode, responses: list[ScriptedResponse]) -> instructor.Instructor: + return instructor.from_openai( + openai.OpenAI(api_key="test", http_client=httpx.Client(transport=response_transport(mode, responses))), mode=mode + ) + + +def async_instructor_client(mode: instructor.Mode, responses: list[ScriptedResponse]) -> instructor.AsyncInstructor: return instructor.from_openai( - openai.OpenAI(api_key="test", http_client=httpx.Client(transport=httpx.MockTransport(respond))), mode=mode + openai.AsyncOpenAI(api_key="test", http_client=httpx.AsyncClient(transport=response_transport(mode, responses))), + mode=mode, ) diff --git a/atomic-agents/tests/observability/test_instructor_modes.py b/atomic-agents/tests/observability/test_instructor_modes.py new file mode 100644 index 00000000..7c2e4dce --- /dev/null +++ b/atomic-agents/tests/observability/test_instructor_modes.py @@ -0,0 +1,141 @@ +"""Real Instructor transport, validation and tracing across Ada's request modes.""" + +import asyncio + +import instructor +import pytest +from instructor.core.exceptions import InstructorRetryException +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import SpanKind, StatusCode + +from instructor_transport import ScriptedResponse, async_instructor_client, instructor_client + +from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema +from atomic_agents.observability import TracingConfig +from atomic_agents.observability import installation + + +class Question(BaseIOSchema): + """A question sent through the real client.""" + + question: str + + +class Answer(BaseIOSchema): + """The validated model answer.""" + + answer: str + + +@pytest.fixture(params=["TOOLS", "JSON", "JSON_SCHEMA", "RESPONSES_TOOLS"]) +def mode(request): + selected = getattr(instructor.Mode, request.param, None) + if selected is None: + pytest.skip(f"Installed Instructor has no Mode.{request.param} for from_openai") + return selected + + +@pytest.fixture(params=[False, True], ids=["run", "run_async"]) +def asynchronous(request): + return request.param + + +@pytest.fixture +def exporter(): + exported = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exported)) + installation.install(tracer_provider=provider, config=TracingConfig(openinference_attributes=False)) + yield exported + installation.uninstall() + provider.shutdown() + + +def run_script(mode, asynchronous, responses): + construct_client = async_instructor_client if asynchronous else instructor_client + client = construct_client(mode, responses) + configured = AtomicAgent[Question, Answer](AgentConfig(client=client, model="test-model")) + configured.model_api_parameters["max_retries"] = 1 + question = Question(question="What is the answer?") + if asynchronous: + return asyncio.run(configured.run_async(question)) + return configured.run(question) + + +def assert_usage(span, input_tokens, output_tokens, cached_tokens, reasoning_tokens): + assert span.attributes["gen_ai.usage.input_tokens"] == input_tokens + assert span.attributes["gen_ai.usage.output_tokens"] == output_tokens + assert span.attributes["gen_ai.usage.cache_read.input_tokens"] == cached_tokens + assert span.attributes["gen_ai.usage.reasoning.output_tokens"] == reasoning_tokens + + +def assert_attempt_spans(exporter, responses): + spans = exporter.get_finished_spans() + parent = spans[-1] + attempts = spans[:-1] + assert parent.attributes["gen_ai.operation.name"] == "invoke_agent" + assert len(attempts) == len(responses) + assert len(spans) == len(responses) + 1 + assert parent.attributes["atomic_agents.llm.attempts"] == len(responses) + assert_models(parent) + for number, (attempt, response) in enumerate(zip(attempts, responses), start=1): + assert attempt.attributes["gen_ai.operation.name"] == "chat" + assert attempt.name == "chat test-model" + assert attempt.kind == SpanKind.CLIENT + assert attempt.parent.span_id == parent.context.span_id + assert attempt.attributes["atomic_agents.llm.attempt"] == number + assert_models(attempt) + assert_usage(attempt, response.input_tokens, response.output_tokens, response.cached_tokens, response.reasoning_tokens) + return attempts, parent + + +def assert_models(span): + assert span.attributes["gen_ai.request.model"] == "test-model" + assert span.attributes["gen_ai.response.model"] == "response-model" + assert span.attributes["gen_ai.provider.name"] == "openai" + + +def test_success_model_usage_and_attempt_count(mode, asynchronous, exporter): + responses = [ScriptedResponse()] + assert run_script(mode, asynchronous, responses).answer == "42" + attempts, parent = assert_attempt_spans(exporter, responses) + assert attempts[0].status.status_code == parent.status.status_code == StatusCode.OK + assert_usage(parent, 3, 2, 1, 2) + + +def test_validation_retry_preserves_each_attempt_and_total_usage(mode, asynchronous, exporter): + responses = [ + ScriptedResponse(arguments='{"answer": {"secret": "private output"}}'), + ScriptedResponse(input_tokens=7, output_tokens=5, cached_tokens=3, reasoning_tokens=4), + ] + assert run_script(mode, asynchronous, responses).answer == "42" + attempts, parent = assert_attempt_spans(exporter, responses) + assert [attempt.status.status_code for attempt in attempts] == [StatusCode.ERROR, StatusCode.OK] + assert parent.status.status_code == StatusCode.OK + assert any(event.name == "validation_error" for event in attempts[0].events) + assert any(event.name == "validation_error" for event in parent.events) + assert_usage(parent, 10, 7, 4, 6) + + +def test_hidden_retry_exhaustion_redacts_all_span_failures(mode, asynchronous, exporter): + installed = installation._current_installation() + installation.install( + tracer_provider=installed.tracer_provider, config=TracingConfig(hide_outputs=True, openinference_attributes=False) + ) + responses = [ScriptedResponse(arguments='{"answer": {"secret": "private model output"}}')] * 2 + with pytest.raises(InstructorRetryException): + run_script(mode, asynchronous, responses) + attempts, parent = assert_attempt_spans(exporter, responses) + assert [attempt.status.status_code for attempt in attempts] == [StatusCode.ERROR, StatusCode.ERROR] + assert parent.status.status_code == StatusCode.ERROR + assert "__REDACTED__" in parent.attributes["atomic_agents.llm.last_validation_error"] + assert_usage(parent, 6, 4, 2, 4) + for span in (*attempts, parent): + assert "__REDACTED__" in span.status.description + assert "private model output" not in str(span.attributes) + str(span.events) + span.status.description + for event in span.events: + if event.name == "exception": + assert "__REDACTED__" in event.attributes["exception.message"] + assert "__REDACTED__" in event.attributes["exception.stacktrace"] From b36db37bbdbeb81484b560de599acaaedd6793e9 Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 13:13:15 +0200 Subject: [PATCH 24/25] Refresh the codebase map for generic OpenTelemetry tracing The observability package is now plain OpenTelemetry with GenAI attributes primary and OpenInference optional, split into nine modules, with the observability/phoenix extras, TracingConfig, using_conversation, AgentConfig.name, LLM attempt spans and the ATOMIC_AGENTS_TRACE_* variables. Eight map documents and the state manifest updated; CLAUDE.md untouched. Co-Authored-By: Claude Fable 5.1 --- .claude/.codebase-info/.map-state.json | 20 ++++---- .claude/.codebase-info/INDEX.md | 4 +- .claude/.codebase-info/architecture.md | 4 +- .claude/.codebase-info/communication.md | 23 ++++++---- .claude/.codebase-info/dependencies.md | 18 ++++---- .claude/.codebase-info/directory-structure.md | 4 +- .claude/.codebase-info/entry-points.md | 11 +++-- .claude/.codebase-info/modules.md | 46 +++++++++++++------ .claude/.codebase-info/tech-landscape.md | 12 +++-- 9 files changed, 83 insertions(+), 59 deletions(-) diff --git a/.claude/.codebase-info/.map-state.json b/.claude/.codebase-info/.map-state.json index 0d5046f0..461aa09a 100644 --- a/.claude/.codebase-info/.map-state.json +++ b/.claude/.codebase-info/.map-state.json @@ -2,8 +2,8 @@ "schemaVersion": 1, "tool": "codebase-mapper", "version": "2.15.10", - "mappedAt": "2026-10-20", - "gitCommit": "609bacb464177649bb73b5d4eea56e631c3c91f0", + "mappedAt": "2026-10-21", + "gitCommit": "d1c1bc8a910d3a80a98d08d3a35e49d748611cea", "documents": [ "architecture.md", "coding-style.md", @@ -17,16 +17,16 @@ "tech-landscape.md" ], "hashes": { - "INDEX.md": "fa005d57802311a643c11e92ea8788e62f8d7eb5eb0001952d26e0e91deb9a49", - "architecture.md": "59d1265c89d3b42671b7c47165408625018537c016a72df421113cae3f80bb70", + "INDEX.md": "8fec73a52b4a1a7b5c9f2ab7de1e02f273aa49aeb73dc5e29a4d5e7049b2c3ec", + "architecture.md": "0ff15b04928495b48cdcf40845b5bf9e3bb548c602b734321c4b10f630d83150", "coding-style.md": "b63e8dff06d16a2d882f02624c62f490291317aec8b3032f6d79b3ee035e8567", - "communication.md": "4c49f01a99b25a2f56b2b3ab77464f05c7d70d057aa8e998deffe845afc3e7e6", - "dependencies.md": "22b8d208df66c973659a34cea10011a7ac315ddaa65b243c8aaa3adc4eed1a9e", - "directory-structure.md": "4efb931cdefecf2c733f0ffb905ad519aff3a81e245a978c0b62b2c5cf0ad49a", - "entry-points.md": "e03c248c63823276c21dbbe894b98488643ee401c2db136ad2467a00ba57bc9a", - "modules.md": "cd4f81f8ce74dfc4a7bab15ba91dc569245bc342ddf494845d658b51a0a1dc51", + "communication.md": "0a61b6090f5e67341ad605517111270bdd04caede037e70ec6565de8dcdf5f28", + "dependencies.md": "abadb58027b4092afaaa0efcaec92893eac7bb598318f7f54a542fd77d607dc1", + "directory-structure.md": "c8d654506843622fd394e0a998fddba769e4ec534449ccd6755d312192184bfd", + "entry-points.md": "65ebc321a35f1ae36f587adce8b7b04b71cc4006ec7d518f4c092836c2984705", + "modules.md": "e1185cdddfbcc7a05a69b80661a6c720aa2a1fc10bc445e319983fffb875524e", "onboarding.md": "3d28c671dc095a2aadb297bd7ab200c677779a99c7534e03e55a3d64c935f07e", "patterns.md": "09d04b9b8d079ac4ce9e1f5ddafabb6f3faf53b5ef48dca1c2d62d0875e149b0", - "tech-landscape.md": "919d254fae6fa5212a18072ee0d26a1165ce2aeef861faace6c34b8804c025b1" + "tech-landscape.md": "8a4459f7309dd13fa9e2425507fec35d21b8b3f691bc6a39b6626fffb8d1744d" } } diff --git a/.claude/.codebase-info/INDEX.md b/.claude/.codebase-info/INDEX.md index d18ba749..70a6c336 100644 --- a/.claude/.codebase-info/INDEX.md +++ b/.claude/.codebase-info/INDEX.md @@ -1,12 +1,12 @@ # Codebase Map — Atomic Agents -*Last Updated: 2026-10-20* +*Last Updated: 2026-10-21* Atomic Agents is a lightweight, modular Python framework for building agentic AI applications as composable, schema-driven building blocks (built on Instructor + Pydantic). This repository is a `uv`-workspace **monorepo**: the core framework, a TUI tool installer, a tool library, and examples. -**Stack:** Python ≥3.12 · Instructor · Pydantic v2 · LiteLLM · MCP · Textual · uv + Hatchling · Pyright · `VideoURL` multimodal content · optional OpenInference tracing (Phoenix) +**Stack:** Python ≥3.12 · Instructor · Pydantic v2 · LiteLLM · MCP · Textual · uv + Hatchling · Pyright · `VideoURL` multimodal content · optional OpenTelemetry tracing (GenAI attributes; Phoenix/OpenInference optional) **Shape:** Monorepo — `atomic-agents/` (core lib) · `atomic-assembler/` (CLI) · `atomic-forge/` (tools) · `atomic-examples/` (examples) **Package:** `atomic-agents` v2.14.0 · core import package is `atomic_agents` diff --git a/.claude/.codebase-info/architecture.md b/.claude/.codebase-info/architecture.md index 4db0b973..97a8b425 100644 --- a/.claude/.codebase-info/architecture.md +++ b/.claude/.codebase-info/architecture.md @@ -1,6 +1,6 @@ # Architecture -*Last Updated: 2026-10-15* +*Last Updated: 2026-10-21* ## Summary @@ -50,7 +50,7 @@ Tool install (atomic-assembler TUI): | Context | `atomic-agents/atomic_agents/context/` | `SystemPromptGenerator`, `BaseDynamicContextProvider`, `BaseChatHistory` (pluggable memory contract) + `ChatHistory` with Instructor media and `VideoURL` | | Connectors | `atomic-agents/atomic_agents/connectors/mcp/` | Model Context Protocol tools / resources / prompts | | Utils | `atomic-agents/atomic_agents/utils/` | Token counting (LiteLLM), tool-message formatting | -| Observability (opt-in) | `atomic-agents/atomic_agents/observability/` | OpenInference AGENT/TOOL spans for Phoenix / OpenTelemetry; `observability` extra + `openinference_instrumentor` entry point | +| Observability (opt-in) | `atomic-agents/atomic_agents/observability/` | OpenTelemetry AGENT / TOOL / LLM-attempt spans with GenAI attributes for any backend (OpenInference attributes optional via the `phoenix` extra); `observability` extra = `opentelemetry-api`; `openinference_instrumentor` + `opentelemetry_instrumentor` entry points | | Assembler (CLI) | `atomic-assembler/atomic_assembler/` | Textual TUI to fetch/install forge tools | | Forge (tools) | `atomic-forge/tools/` | 13 standalone tools, each `BaseTool`-based | diff --git a/.claude/.codebase-info/communication.md b/.claude/.codebase-info/communication.md index 7325f5e5..b4fbce37 100644 --- a/.claude/.codebase-info/communication.md +++ b/.claude/.codebase-info/communication.md @@ -1,6 +1,6 @@ # Communication & Integrations -*Last Updated: 2026-10-19* +*Last Updated: 2026-10-21* The framework exposes no HTTP API of its own; "communication" here means how it talks to LLM providers and external tools. @@ -37,15 +37,18 @@ providers and external tools. - Built on Instructor's hook system: `register_hook(event, handler)`, `unregister_hook`, `clear_hooks`. Events include `parse:error`, `completion:kwargs`, `completion:response`, `token:counted`. See the `hooks-example` example and `docs/guides/hooks.md`. -- OpenInference tracing (`atomic_agents.observability`, optional `observability` extra): an AGENT span per - `AtomicAgent` run and a TOOL span per `BaseTool` run, auto-activated by `phoenix.otel.register(auto_instrument=True)` - through `openinference_instrumentor`, by the standard `opentelemetry-instrument` launcher through - `opentelemetry_instrumentor`, or by `AtomicAgentsInstrumentor().instrument(tracer_provider=...)` - for an explicitly configured backend. Both discovery paths share the first installation's provider/config. - Missing optional tracing dependencies are DEBUG-skipped when the launcher passes `skip_dep_check=True`; - manual setup retains its import error and install hint. Choose one configuration path or share the - provider explicitly. LLM spans come from one provider-level OpenInference instrumentor and nest under - the agent span. See `docs/guides/observability.md` and the `phoenix-tracing` example. +- OpenTelemetry tracing (`atomic_agents.observability`, `observability` extra = `opentelemetry-api` only): an AGENT + span per `AtomicAgent` run, a TOOL span per `BaseTool` run and one CLIENT `chat` span per Instructor attempt + (driven by Instructor hooks registered once per client), all with OpenTelemetry GenAI attributes; OpenInference + attributes are added only when its package is importable (`phoenix` extra). Activated by + `phoenix.otel.register(auto_instrument=True)` through `openinference_instrumentor`, by the `opentelemetry-instrument` + launcher through `opentelemetry_instrumentor`, or by `AtomicAgentsInstrumentor().instrument(tracer_provider=..., + config=TracingConfig(...))`. A later install with different explicit values replaces the provider/config in place + with a WARNING; `None` keeps the installed value. Missing `opentelemetry-api` is DEBUG-skipped when the launcher + passes `skip_dep_check=True`; manual setup raises `ModuleNotFoundError` with the install hint. Application hooks on + the same client keep running; `AtomicAgent.clear_hooks()` removes only its own. Set `llm_spans=False` (or + `ATOMIC_AGENTS_TRACE_LLM_SPANS=false`) when a provider instrumentor also emits LLM spans. See + `docs/guides/observability.md` and the `phoenix-tracing` example. - Forge tools that run their async body through a `ThreadPoolExecutor` submit via `contextvars.copy_context().run` so the caller's OpenTelemetry context (and the tracing re-entrancy guard) survives in the pool thread. diff --git a/.claude/.codebase-info/dependencies.md b/.claude/.codebase-info/dependencies.md index ec1ab3c4..a2e76b38 100644 --- a/.claude/.codebase-info/dependencies.md +++ b/.claude/.codebase-info/dependencies.md @@ -1,6 +1,6 @@ # Dependencies -*Last Updated: 2026-10-19* +*Last Updated: 2026-10-21* Declared in `pyproject.toml` (Hatchling build, `uv` workspace; versions locked in `uv.lock`). @@ -12,15 +12,15 @@ Declared in `pyproject.toml` (Hatchling build, `uv` workspace; versions locked i | litellm (≥1.50,<2) | Provider-agnostic token counting & model metadata | | rich (≥13.7.1,<14) | Console formatting | -## Optional extra — `observability` -| Package | Purpose | -|---------|---------| -| openinference-instrumentation (≥0.1.38,<0.2) | `OITracer` / `TraceConfig` used by `atomic_agents.observability` | -| openinference-semantic-conventions (≥0.1.20,<0.2) | Span attribute names (`agent.name` needs ≥0.1.20) | -| opentelemetry-api (≥1.30,<2) | Tracer / context API | +## Optional extras — `observability` and `phoenix` +| Extra | Package | Purpose | +|-------|---------|---------| +| `observability` | opentelemetry-api (≥1.30,<2) | Tracer / context API; the only runtime dependency of `atomic_agents.observability` | +| `phoenix` | opentelemetry-api, openinference-instrumentation (≥0.1.38,<0.2), openinference-semantic-conventions (≥0.1.20,<0.2) | Adds the OpenInference attribute family and `using_session` / `using_user` / `using_metadata`; needed for Arize Phoenix views | -Also listed in the dev group (plus `opentelemetry-sdk`) so CI runs the observability tests. `arize-phoenix-otel` -already brings the two OpenInference packages, so the Phoenix auto path works without the extra. +Users bring their own SDK and exporter (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`, ...). The dev +group carries the OpenInference packages plus `opentelemetry-sdk` so the tests run; `arize-phoenix-otel` already +brings the OpenInference packages, so the Phoenix auto path works without the `phoenix` extra. ## Runtime — CLI / tooling | Package | Purpose | diff --git a/.claude/.codebase-info/directory-structure.md b/.claude/.codebase-info/directory-structure.md index b74bb241..1602c3cb 100644 --- a/.claude/.codebase-info/directory-structure.md +++ b/.claude/.codebase-info/directory-structure.md @@ -1,6 +1,6 @@ # Directory Structure -*Last Updated: 2026-10-15* +*Last Updated: 2026-10-21* ## Root Layout @@ -13,7 +13,7 @@ atomic-agents/ # repo root (uv workspace) │ ├── context/ # SystemPromptGenerator, BaseChatHistory/ChatHistory, context providers │ ├── connectors/mcp/ # Model Context Protocol integration │ ├── utils/ # token counter, tool-message formatting -│ └── observability/ # opt-in OpenInference instrumentor (Phoenix / OpenTelemetry) +│ └── observability/ # opt-in OpenTelemetry instrumentor (GenAI attributes; OpenInference optional) │ └── tests/ # pytest suite (agents/, base/, context/, connectors/, observability/, utils/) ├── atomic-assembler/ # Textual TUI (`atomic` command) to install forge tools │ └── atomic_assembler/ # main.py, app.py, screens/, widgets/, utils.py, constants.py diff --git a/.claude/.codebase-info/entry-points.md b/.claude/.codebase-info/entry-points.md index a1037d48..bb5feb31 100644 --- a/.claude/.codebase-info/entry-points.md +++ b/.claude/.codebase-info/entry-points.md @@ -1,6 +1,6 @@ # Entry Points -*Last Updated: 2026-10-19* +*Last Updated: 2026-10-21* ## 1. Library API (the primary entry point) @@ -49,7 +49,7 @@ MCP server tools/resources/prompts become agent components via `atomic_agents.co | `fastapi-memory` | multi-user / multi-session memory behind a FastAPI service | | `persistent-memory` | custom `BaseChatHistory` backend: cross-session recall via stdlib `sqlite3` (no extra deps) | | `hooks-example` | monitoring, error handling, retry via hooks | -| `phoenix-tracing` | Arize Phoenix traces: planner agent -> calculator tool -> answer agent under one CHAIN span, via `register(auto_instrument=True)` | +| `phoenix-tracing` | Arize Phoenix traces: planner agent -> calculator tool -> answer agent under one CHAIN span, via `register(auto_instrument=True)`; depends on `atomic-agents[phoenix]` and sets `ATOMIC_AGENTS_TRACE_LLM_SPANS=false` because the OpenAI instrumentor owns LLM spans there | | `basic-multimodal` / `nested-multimodal` | images / PDFs in (nested) schemas | | `basic-pdf-analysis` | PDF analysis with a multimodal model | | `youtube-summarizer` / `youtube-to-recipe` | transcript → summary / structured recipe | @@ -60,9 +60,10 @@ MCP server tools/resources/prompts become agent components via `atomic_agents.co `atomic_agents.observability:AtomicAgentsInstrumentor` is registered as `atomic_agents` in both `openinference_instrumentor` (Phoenix) and `opentelemetry_instrumentor` (the standard `opentelemetry-instrument` launcher). Both use the same installation state. The launcher needs -its own SDK/exporter setup and the optional OpenInference dependencies; it does not install them -through `opentelemetry-bootstrap`. Set `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=atomic_agents` to -disable launcher discovery. See `docs/guides/observability.md`. +its own SDK/exporter setup; only `opentelemetry-api` is required for the instrumentor, and +`opentelemetry-bootstrap` does not install Atomic Agents. Set `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=atomic_agents` +to disable launcher discovery; `ATOMIC_AGENTS_TRACE_*` environment variables configure hiding, size cap and +LLM attempt spans on both automatic paths. See `docs/guides/observability.md`. ## Representative Flow diff --git a/.claude/.codebase-info/modules.md b/.claude/.codebase-info/modules.md index ab7154f6..f70bbe5a 100644 --- a/.claude/.codebase-info/modules.md +++ b/.claude/.codebase-info/modules.md @@ -1,6 +1,6 @@ # Key Modules -*Last Updated: 2026-10-15* +*Last Updated: 2026-10-21* ## Core framework — `atomic-agents/atomic_agents/` @@ -46,19 +46,37 @@ ### observability (opt-in) - **Location:** `atomic-agents/atomic_agents/observability/` -- **Purpose:** OpenInference tracing for Arize Phoenix and any OpenTelemetry backend. `AtomicAgentsInstrumentor` - (`__init__.py`, stdlib-only imports) lazily loads `openinference_tracing.py`, which patches the four - `AtomicAgent.run*` methods into AGENT spans and every `BaseTool` subclass's `run`/`run_async`/`arun` into TOOL - spans (existing classes via a `__subclasses__` walk, future ones via a wrapped `BaseTool.__init_subclass__`). -- **Contract:** `instrument(tracer_provider=, config=TraceConfig)` / `uninstrument()` / - `is_instrumented_by_opentelemetry`; idempotent and fully reversible. Needs the `observability` extra - (`openinference-instrumentation>=0.1.38`); older versions are refused with a warning so Phoenix's - `register(auto_instrument=True)` (which loads the `openinference_instrumentor` entry point) never fails. -- **Span facts:** `agent.name` = `Class[InputSchema, OutputSchema]`; namespaced `atomic_agents.*` attributes - (model, mode, schema names, history size, context provider names, SHA-256 of the static prompt sections); - streams record the final/last partial and `atomic_agents.stream.closed_early`; a `ContextVar` re-entrancy guard - keeps `run -> arun` / `to_thread` delegation to one TOOL span. Nothing in `atomic_agents/__init__.py` imports it. -- **Tests:** `atomic-agents/tests/observability/` (in-memory span exporter). +- **Purpose:** plain OpenTelemetry tracing for any backend (OTLP collector, Grafana Tempo, Sentry, Datadog, + Honeycomb, PostHog, Arize Phoenix, Langfuse). Only `opentelemetry-api` is required; the OpenTelemetry GenAI + semantic-convention attributes are primary and the OpenInference family is an optional layer, auto-on when the + `openinference-instrumentation` package is importable (`phoenix` extra). +- **Modules:** `__init__.py` (loader, `AtomicAgentsInstrumentor`, stdlib-only imports), `tracing.py` (thin facade), + `tracing_config.py` (`TracingConfig`, `PayloadVisibility`, env parsing), `span_payloads.py` (serialize, hide before + serialize, truncate with `.original_length`, bounded/redacted error text), `span_attributes.py` (key constants, + Instructor-to-semconv provider map, None filtering, best-effort attribute collection, OpenInference context seam, + `using_conversation`), `agent_spans.py` (four `AtomicAgent.run*` wrappers, stream cursors, run state), + `tool_spans.py` (`BaseTool` wrappers via `__subclasses__` walk + wrapped `__init_subclass__`, re-entrancy guard), + `installation.py` (install / uninstall / replace-in-place), `llm_attempt_spans.py` (Instructor hook handlers). +- **Contract:** `instrument(tracer_provider=, config=TracingConfig | PayloadVisibility | None, **options)` / + `uninstrument()` / `is_instrumented_by_opentelemetry`. Same arguments: no-op; different provider or config: + replaced in place with a WARNING, `None` keeps the installed value; in-flight spans finish on their original + provider. Hooks are registered once per Instructor client under a lock and removed on uninstall; + `suppress_instrumentation` is honored; hooks never raise into user code. +- **Spans:** `invoke_agent {AgentConfig.name or class}` (`gen_ai.operation.name=invoke_agent`, `gen_ai.agent.name`, + `gen_ai.request.model`, `gen_ai.provider.name`, `gen_ai.conversation.id`, `atomic_agents.agent.class`, `.input`, + `.output`, model/mode/schema/history/provider/prompt-hash keys, usage totals and `atomic_agents.llm.attempts` at end); + `execute_tool {tool.tool_name}` (`gen_ai.tool.*`, `atomic_agents.tool.class`); one `chat {model}` CLIENT span per + Instructor attempt from `completion:kwargs`/`completion:response`/`completion:error` with per-attempt usage + (cached/reasoning tokens for both chat-completion and Responses shapes), `parse:error` as a `validation_error` + event; `error.type` on failures; cancellation = `atomic_agents.cancelled`, never ERROR. Spans always attach to the + current context. `llm_spans=False` keeps totals and drops chat spans. +- **Config:** `TracingConfig(hide_inputs, hide_outputs, max_attribute_length=4096, openinference_attributes=None, + llm_spans=True)`; env `ATOMIC_AGENTS_TRACE_HIDE_INPUTS/_OUTPUTS` (fallback `OPENINFERENCE_HIDE_*`), + `ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH` (tolerant, 4096 fallback), `ATOMIC_AGENTS_TRACE_LLM_SPANS`. Under + `hide_outputs` every exception becomes `: attempts (__REDACTED__)`. Nothing in `atomic_agents/__init__.py` + imports the package. +- **Tests:** `atomic-agents/tests/observability/` (in-memory exporter; `instructor_transport.py` builds real Instructor + clients over `httpx.MockTransport`; `test_instructor_modes.py` covers TOOLS/JSON/JSON_SCHEMA/RESPONSES_TOOLS). ## Subprojects diff --git a/.claude/.codebase-info/tech-landscape.md b/.claude/.codebase-info/tech-landscape.md index fe1776d6..9dcf0ee9 100644 --- a/.claude/.codebase-info/tech-landscape.md +++ b/.claude/.codebase-info/tech-landscape.md @@ -1,6 +1,6 @@ # Technology Landscape -*Last Updated: 2026-10-20* +*Last Updated: 2026-10-21* ## Source-of-Truth Files @@ -40,10 +40,12 @@ - `code-quality.yml` — on push/PR to `main`/`v2.0`, three jobs: `quality` (`uv sync`, `black --check` and `flake8` across all four project directories, a focused Pyright 1.1.414 constructor check, then `pytest --cov=atomic_agents` for the core); `fresh-install` (build the wheel, install - it with the observability extra into a clean venv without the lockfile, import every module under - `atomic_agents`, which is what a new user's `pip install` sees); `lowest-versions` (resolve runtime - dependencies with `uv pip compile --resolution lowest-direct`, install current pytest tooling plus - `opentelemetry-instrumentation==0.51b0` compatible with the OpenTelemetry API floor, then run the + it with the `observability` extra plus `opentelemetry-sdk` into a clean venv without the lockfile, prove + `openinference` is not importable, run a real `instructor.from_openai` client over `httpx.MockTransport` + through a tool and an agent and assert the GenAI span names, usage keys and the absence of every OpenInference + key, then import every module under `atomic_agents`); `lowest-versions` (resolve the runtime dependencies plus + the `observability` and `phoenix` extras with `uv pip compile --resolution lowest-direct`, install current pytest + tooling plus `opentelemetry-instrumentation==0.51b0` compatible with the OpenTelemetry API floor, then run the suite). Observability tests exercise real installed discovery metadata, the auto-loader and enabled/disabled launcher subprocesses with local exporters; no live model/backend calls are needed. - `docs.yml` — on push to `main`: build Sphinx docs, run From b56a45190d193f6136d544cfe00a55fd803bdeab Mon Sep 17 00:00:00 2001 From: Kenny Vaneetvelde Date: Wed, 21 Oct 2026 13:30:41 +0200 Subject: [PATCH 25/25] Fix tracing cancellation and environment configuration gaps Keep cancelled tools unset, inherit environment settings with visibility configs, and exercise conversation attributes in the API-only smoke. Document the installed Instructor retry logger's privacy limits. Co-Authored-By: Claude Code --- .github/workflows/code-quality.yml | 6 +- .../atomic_agents/observability/__init__.py | 2 +- .../atomic_agents/observability/tool_spans.py | 4 ++ .../observability/tracing_config.py | 8 +-- .../tests/observability/test_tracing.py | 56 +++++++++++++++++++ docs/guides/observability.md | 16 +++++- 6 files changed, 81 insertions(+), 11 deletions(-) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 9b47588a..ba02e24e 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -104,7 +104,7 @@ jobs: from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema, BaseTool - from atomic_agents.observability import AtomicAgentsInstrumentor, TracingConfig + from atomic_agents.observability import AtomicAgentsInstrumentor, TracingConfig, using_conversation exporter = InMemorySpanExporter() provider = TracerProvider() @@ -153,7 +153,8 @@ jobs: assert SmokeTool().run(SmokeInput(text="tool input")).text == "tool input" agent = SmokeAgent(config=AgentConfig(client=client, model="offline-model")) - assert agent.run(SmokeInput(text="private agent input")).text == "offline response" + with using_conversation("smoke-conversation"): + assert agent.run(SmokeInput(text="private agent input")).text == "offline response" spans = exporter.get_finished_spans() for span in spans: @@ -174,6 +175,7 @@ jobs: assert chat_span.name == "chat offline-model" assert chat_span.parent.span_id == agent_span.context.span_id for span in (agent_span, chat_span): + assert span.attributes["gen_ai.conversation.id"] == "smoke-conversation" assert span.attributes["gen_ai.request.model"] == "offline-model" assert span.attributes["gen_ai.response.model"] == "response-model" assert span.attributes["gen_ai.provider.name"] == "openai" diff --git a/atomic-agents/atomic_agents/observability/__init__.py b/atomic-agents/atomic_agents/observability/__init__.py index cd132aa8..254230a0 100644 --- a/atomic-agents/atomic_agents/observability/__init__.py +++ b/atomic-agents/atomic_agents/observability/__init__.py @@ -60,7 +60,7 @@ def _warn_unknown_options(options: dict[str, object]) -> None: class AtomicAgentsInstrumentor: """ - Installs AGENT and TOOL spans around ``AtomicAgent`` and ``BaseTool`` calls. + Installs AGENT, TOOL and LLM-attempt spans around ``AtomicAgent`` and ``BaseTool`` calls. The public surface mirrors OpenTelemetry's ``BaseInstrumentor`` (``instrument`` / ``uninstrument``) without depending on it, because Arize Phoenix only guarantees ``opentelemetry-api`` and diff --git a/atomic-agents/atomic_agents/observability/tool_spans.py b/atomic-agents/atomic_agents/observability/tool_spans.py index 59b3f16d..7a140aae 100644 --- a/atomic-agents/atomic_agents/observability/tool_spans.py +++ b/atomic-agents/atomic_agents/observability/tool_spans.py @@ -1,5 +1,6 @@ """Tool spans and tool class instrumentation.""" +import asyncio import contextvars import functools import inspect @@ -140,6 +141,9 @@ def _tool_span(tool: BaseTool, args: tuple[object, ...], kwargs: dict[str, objec _set_input_safely(span, _tool_input_from_call(args, kwargs), ATTR_TOOL_ARGUMENTS) try: yield span + except (asyncio.CancelledError, GeneratorExit): + span.set_attribute("atomic_agents.cancelled", True) + raise except BaseException as error: _record_failure(span, error) raise diff --git a/atomic-agents/atomic_agents/observability/tracing_config.py b/atomic-agents/atomic_agents/observability/tracing_config.py index 62029157..be16d881 100644 --- a/atomic-agents/atomic_agents/observability/tracing_config.py +++ b/atomic-agents/atomic_agents/observability/tracing_config.py @@ -2,7 +2,7 @@ import logging import os -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Protocol @@ -38,11 +38,7 @@ def coerce(cls, config: "TracingConfig | PayloadVisibility | None") -> "TracingC return cls.from_environment() if isinstance(config, cls): return config - return cls( - hide_inputs=config.hide_inputs, - hide_outputs=config.hide_outputs, - max_attribute_length=_attribute_length_environment(), - ) + return replace(cls.from_environment(), hide_inputs=config.hide_inputs, hide_outputs=config.hide_outputs) def _visibility_environment(direction: str) -> str: diff --git a/atomic-agents/tests/observability/test_tracing.py b/atomic-agents/tests/observability/test_tracing.py index 5713be29..96d0c8e3 100644 --- a/atomic-agents/tests/observability/test_tracing.py +++ b/atomic-agents/tests/observability/test_tracing.py @@ -562,6 +562,51 @@ def test_tool_failure_propagates_same_exception_and_marks_error(exporter): (span,) = spans_of(exporter) assert span.status.status_code == StatusCode.ERROR assert raised.value.args == ("boom",) + assert span.attributes["error.type"] == "ValueError" + assert [event.name for event in span.events] == ["exception"] + + +@pytest.mark.asyncio +async def test_tool_run_async_cancellation_leaves_span_unset(exporter): + started = asyncio.Event() + release = asyncio.Event() + + class WaitingTool(CalculatorTool): + async def run_async(self, params: CalculationInputSchema) -> CalculationOutputSchema: + started.set() + await release.wait() + return CalculationOutputSchema(result="never") + + task = asyncio.create_task(WaitingTool().run_async(CalculationInputSchema(expression="x"))) + await asyncio.wait_for(started.wait(), timeout=2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + (span,) = spans_of(exporter) + assert span.attributes.get("atomic_agents.cancelled") is True + assert span.status.status_code == StatusCode.UNSET + assert span.events == () + assert "error.type" not in span.attributes + + +@pytest.mark.parametrize("error_type", [GeneratorExit, asyncio.CancelledError]) +@pytest.mark.parametrize("method_name", ["run", "run_async"]) +@pytest.mark.asyncio +async def test_tool_cancellation_propagates_without_error(exporter, monkeypatch, error_type, method_name): + def cancelled_result(*args, **kwargs): + raise error_type + + monkeypatch.setattr(CalculationOutputSchema, "__init__", cancelled_result) + tool = ThreadedCalculatorTool() + with pytest.raises(error_type): + result = getattr(tool, method_name)(CalculationInputSchema(expression="x")) + if method_name == "run_async": + await result + (span,) = spans_of(exporter) + assert span.attributes.get("atomic_agents.cancelled") is True + assert span.status.status_code == StatusCode.UNSET + assert span.events == () + assert "error.type" not in span.attributes def test_unserializable_input_still_returns_result(exporter, caplog): @@ -1012,6 +1057,17 @@ def test_tracing_config_llm_spans_environment(monkeypatch, value, expected): assert TracingConfig.coerce(None).llm_spans is expected +def test_payload_visibility_preserves_environment_settings(monkeypatch): + monkeypatch.setenv("ATOMIC_AGENTS_TRACE_LLM_SPANS", "false") + monkeypatch.setenv("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", "128") + monkeypatch.setenv("ATOMIC_AGENTS_TRACE_HIDE_OUTPUTS", "true") + config = tracing.TracingConfig.coerce(TraceConfig(hide_inputs=True, hide_outputs=False)) + assert config.llm_spans is False + assert config.max_attribute_length == 128 + assert config.hide_inputs is True + assert config.hide_outputs is False + + def test_tracing_config_llm_spans_defaults_to_enabled(monkeypatch): from atomic_agents.observability import TracingConfig diff --git a/docs/guides/observability.md b/docs/guides/observability.md index a0062cc7..fe479272 100644 --- a/docs/guides/observability.md +++ b/docs/guides/observability.md @@ -175,7 +175,7 @@ share one installation, so activating several does not duplicate spans. OpenTele provider, so calling Phoenix registration after launcher startup does not switch where spans are exported. See "Re-instrumentation" for what a second `instrument()` call does. -The launcher adds AGENT and TOOL spans and no parent span. The example above disables Atomic Agents' LLM attempt +The launcher adds AGENT, TOOL and LLM-attempt spans and no parent span. The example above disables Atomic Agents' LLM attempt spans for applications that also use a provider instrumentor. Leave `ATOMIC_AGENTS_TRACE_LLM_SPANS` unset to keep attempt spans enabled. Use the grouping example below when you want several agent and tool calls in one trace. @@ -362,7 +362,8 @@ as a fallback (`true` or `1`). `ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH` sets t `ATOMIC_AGENTS_TRACE_LLM_SPANS` disables attempt spans for `false` or `0` (case-insensitive); any other value or unset keeps them enabled. If you pass your own `TracingConfig(...)`, the environment is not read; use `TracingConfig.from_environment()` to start from it. An -OpenInference `TraceConfig` is also accepted for `config`, for its `hide_inputs` and `hide_outputs` settings. +OpenInference `TraceConfig` is also accepted for `config`; its `hide_inputs` and `hide_outputs` override those +environment settings, while the other settings still come from the environment. A truncated value gets a companion attribute with the original size, for example `atomic_agents.input.original_length`. Hiding wins over truncation, and a hidden payload is never serialized in the first place. @@ -373,6 +374,17 @@ is truncated but never hidden. Status descriptions and recorded exception messag `hide_outputs` a Pydantic validation error message is replaced by `__REDACTED__` plus the error count, because the raw model output is quoted inside it. +Instructor's own logging is separate from Atomic Agents' tracing: in the installed Instructor 1.17.0 source, +`instructor.v2.retry` (re-exported through `instructor.core.retry`) logs per-attempt validation failures at DEBUG +and retry exhaustion's raw last error, including model output, at ERROR regardless of `hide_outputs`. +Setting its level to ERROR does not silence exhaustion. To keep model output out of logs, use CRITICAL below +or a filter/handler that drops those records. The logger name and levels may change in later Instructor versions. + +```python +import logging +logging.getLogger("instructor.v2.retry").setLevel(logging.CRITICAL) +``` + Not covered: tool names and descriptions, the system prompt hash, and whatever you put in `using_metadata`. A provider instrumentor and your own tracer apply their own settings, so configure them the same way if you want matching masking. Multimodal inputs (images, audio, PDFs) are serialized into the input payload in full, so `max_attribute_length` is the