diff --git a/.coverage b/.coverage deleted file mode 100644 index f4e6857..0000000 Binary files a/.coverage and /dev/null differ diff --git a/.gitignore b/.gitignore index 2878557..23e68ac 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,7 @@ test.py .idea/ venv/ .github/instructions/*.md -.copilot-tracking \ No newline at end of file +.copilot-tracking +.env* +requirements* +.serena diff --git a/basalt/observability/api.py b/basalt/observability/api.py index ed8fb6a..5d59bc9 100644 --- a/basalt/observability/api.py +++ b/basalt/observability/api.py @@ -109,6 +109,8 @@ def __enter__(self) -> StartSpanHandle: evaluators=self.evaluators, feature_slug=self.feature_slug, metadata=self._metadata, + evaluate_config=self.evaluate_config, + experiment=self.experiment, ) span = self._ctx_manager.__enter__() # Type assertion: we know this is StartSpanHandle since we passed it as handle_cls @@ -150,6 +152,8 @@ def wrapper(*args, **kwargs): evaluators=pre_evaluators, feature_slug=self.feature_slug, metadata=self._metadata, + evaluate_config=self.evaluate_config, + experiment=self.experiment, ) as handle: # Type assertion: we know this is StartSpanHandle since we passed it as handle_cls assert isinstance(handle, StartSpanHandle) @@ -188,6 +192,8 @@ async def async_wrapper(*args, **kwargs): evaluators=pre_evaluators, feature_slug=self.feature_slug, metadata=self._metadata, + evaluate_config=self.evaluate_config, + experiment=self.experiment, ) as handle: # Type assertion: we know this is StartSpanHandle since we passed it as handle_cls assert isinstance(handle, StartSpanHandle) @@ -879,6 +885,8 @@ async def __aenter__(self) -> StartSpanHandle: evaluators=self.evaluators, feature_slug=self.feature_slug, metadata=self._metadata, + evaluate_config=self.evaluate_config, + experiment=self.experiment, ) span = await self._ctx_manager.__aenter__() # Type assertion: we know this is StartSpanHandle since we passed it as handle_cls diff --git a/basalt/observability/config.py b/basalt/observability/config.py index 100ed54..8bc36c0 100644 --- a/basalt/observability/config.py +++ b/basalt/observability/config.py @@ -131,6 +131,17 @@ class TelemetryConfig: extra_resource_attributes: dict[str, Any] = field(default_factory=dict) + sample_rate: float = 0.0 + """ + Global default sampling rate for trace-level evaluation (0.0-1.0, default 0.0). + Controls whether evaluators run for a trace via should_evaluate attribute. + Can be overridden per-trace via EvaluationConfig(sample_rate=...) in start_observe(). + """ + + def __post_init__(self) -> None: + if not 0.0 <= self.sample_rate <= 1.0: + raise ValueError("sample_rate must be within [0.0, 1.0].") + def clone(self) -> TelemetryConfig: """Return a defensive copy of the telemetry configuration.""" cloned = replace(self) @@ -175,6 +186,15 @@ def with_env_overrides(self) -> TelemetryConfig: if disabled_instruments: cfg.disabled_providers = [p.strip() for p in disabled_instruments.split(",") if p.strip()] + sample_rate_env = os.getenv("BASALT_SAMPLE_RATE") + if sample_rate_env: + try: + rate = float(sample_rate_env) + if 0.0 <= rate <= 1.0: + cfg.sample_rate = rate + except ValueError: + pass # Ignore invalid values + if not cfg.service_version: # basalt_sdk_config is a mapping defined in `basalt.config` module cfg.service_version = basalt_sdk_config.get("sdk_version", "unknown") diff --git a/basalt/observability/context_managers.py b/basalt/observability/context_managers.py index 53998d9..fc7626b 100644 --- a/basalt/observability/context_managers.py +++ b/basalt/observability/context_managers.py @@ -5,6 +5,7 @@ import json import logging import os +import random from collections.abc import AsyncGenerator, Generator, Mapping, Sequence from contextlib import asynccontextmanager, contextmanager from dataclasses import dataclass, field @@ -22,6 +23,7 @@ from . import semconv from .trace_context import ( ORGANIZATION_CONTEXT_KEY, + SHOULD_EVALUATE_CONTEXT_KEY, USER_CONTEXT_KEY, TraceIdentity, _current_trace_defaults, @@ -44,14 +46,16 @@ class EvaluationConfig: """ Type-safe configuration for evaluators attached to a span. - This configuration is span-scoped and shared by all evaluators in the span. - It's not handled client-side but attached to the span for server-side processing. + This configuration is span-scoped and controls trace-level sampling for evaluators. + The sample_rate determines whether evaluators run for the entire trace. Attributes: - sample_rate: Sampling rate for evaluators (0.0-1.0). Default is 1.0 (100%). + sample_rate: Sampling rate for trace-level evaluation (0.0-1.0). Default is 0.0 (no sampling). + When set, one sampling decision is made at root span creation and propagated + to all spans in the trace via basalt.span.should_evaluate attribute. """ - sample_rate: float = 1.0 + sample_rate: float = 0.0 def __post_init__(self) -> None: if not 0.0 <= self.sample_rate <= 1.0: @@ -678,6 +682,8 @@ def _with_span_handle( organization: TraceIdentity | Mapping[str, Any] | None = None, feature_slug: str | None = None, metadata: Mapping[str, Any] | None = None, + evaluate_config: EvaluationConfig | None = None, + experiment: Any = None, ) -> Generator[SpanHandle, None, None]: tracer = get_tracer(tracer_name) defaults = _current_trace_defaults() @@ -714,6 +720,37 @@ def _with_span_handle( # Check if we're inside a basalt trace in_basalt_trace = otel_context.get_value(ROOT_SPAN_CONTEXT_KEY) is not None + # Make trace-level sampling decision + should_evaluate_token = None + if is_root: + # Root span: make new sampling decision + # If experiment is attached, ALWAYS evaluate (should_evaluate=True) + if experiment is not None: + should_evaluate = True + else: + # Get sample_rate from evaluate_config if provided, otherwise use global default + if evaluate_config is not None: + effective_sample_rate = evaluate_config.sample_rate + else: + effective_sample_rate = defaults.sample_rate + should_evaluate = random.random() < effective_sample_rate + should_evaluate_token = attach(set_value(SHOULD_EVALUATE_CONTEXT_KEY, should_evaluate)) + else: + # Check if should_evaluate already exists in context + existing_should_evaluate = otel_context.get_value(SHOULD_EVALUATE_CONTEXT_KEY) + if existing_should_evaluate is None: + # Orphan span without root - make its own decision + # If experiment is attached, ALWAYS evaluate + if experiment is not None: + should_evaluate = True + else: + if evaluate_config is not None: + effective_sample_rate = evaluate_config.sample_rate + else: + effective_sample_rate = defaults.sample_rate + should_evaluate = random.random() < effective_sample_rate + should_evaluate_token = attach(set_value(SHOULD_EVALUATE_CONTEXT_KEY, should_evaluate)) + try: with tracer.start_as_current_span(name) as span: # Store root span in context for retrieval from nested spans @@ -780,6 +817,10 @@ def _with_span_handle( handle.set_output(output_payload) finally: + # Detach should_evaluate token if it was set + if should_evaluate_token is not None: + detach(should_evaluate_token) + # Detach root span token if it was set if root_span_token is not None: detach(root_span_token) @@ -805,6 +846,8 @@ async def _async_with_span_handle( organization: TraceIdentity | Mapping[str, Any] | None = None, feature_slug: str | None = None, metadata: Mapping[str, Any] | None = None, + evaluate_config: EvaluationConfig | None = None, + experiment: Any = None, ) -> AsyncGenerator[SpanHandle, None]: """Async version of _with_span_handle. @@ -847,6 +890,37 @@ async def _async_with_span_handle( # Check if we're inside a basalt trace in_basalt_trace = otel_context.get_value(ROOT_SPAN_CONTEXT_KEY) is not None + # Make trace-level sampling decision + should_evaluate_token = None + if is_root: + # Root span: make new sampling decision + # If experiment is attached, ALWAYS evaluate (should_evaluate=True) + if experiment is not None: + should_evaluate = True + else: + # Get sample_rate from evaluate_config if provided, otherwise use global default + if evaluate_config is not None: + effective_sample_rate = evaluate_config.sample_rate + else: + effective_sample_rate = defaults.sample_rate + should_evaluate = random.random() < effective_sample_rate + should_evaluate_token = attach(set_value(SHOULD_EVALUATE_CONTEXT_KEY, should_evaluate)) + else: + # Check if should_evaluate already exists in context + existing_should_evaluate = otel_context.get_value(SHOULD_EVALUATE_CONTEXT_KEY) + if existing_should_evaluate is None: + # Orphan span without root - make its own decision + # If experiment is attached, ALWAYS evaluate + if experiment is not None: + should_evaluate = True + else: + if evaluate_config is not None: + effective_sample_rate = evaluate_config.sample_rate + else: + effective_sample_rate = defaults.sample_rate + should_evaluate = random.random() < effective_sample_rate + should_evaluate_token = attach(set_value(SHOULD_EVALUATE_CONTEXT_KEY, should_evaluate)) + try: with tracer.start_as_current_span(name) as span: # Store root span in context for retrieval from nested spans @@ -913,6 +987,10 @@ async def _async_with_span_handle( handle.set_output(output_payload) finally: + # Detach should_evaluate token if it was set + if should_evaluate_token is not None: + detach(should_evaluate_token) + # Detach root span token if it was set if root_span_token is not None: detach(root_span_token) diff --git a/basalt/observability/instrumentation.py b/basalt/observability/instrumentation.py index a81b152..1955f2c 100644 --- a/basalt/observability/instrumentation.py +++ b/basalt/observability/instrumentation.py @@ -31,6 +31,7 @@ BasaltAutoInstrumentationProcessor, BasaltCallEvaluatorProcessor, BasaltContextProcessor, + BasaltShouldEvaluateProcessor, ) from .resilient_exporters import ResilientSpanExporter @@ -431,6 +432,10 @@ def _initialize_instrumentation(self, config: TelemetryConfig) -> None: Args: config: Telemetry configuration specifying trace content and provider settings. """ + # Set global sample rate from config + from .trace_context import set_global_sample_rate + set_global_sample_rate(config.sample_rate) + # Set environment variables for third-party OpenTelemetry instrumentors # These variables are READ by the instrumentation libraries (openai, anthropic, etc.) # and control whether they capture prompts/completions in traces. @@ -459,6 +464,7 @@ def _install_basalt_processors(self, provider: TracerProvider) -> None: processors: list[OTelSpanProcessor] = [ BasaltContextProcessor(), BasaltCallEvaluatorProcessor(), + BasaltShouldEvaluateProcessor(), BasaltAutoInstrumentationProcessor(), ] diff --git a/basalt/observability/processors.py b/basalt/observability/processors.py index 1fd55da..a866b61 100644 --- a/basalt/observability/processors.py +++ b/basalt/observability/processors.py @@ -212,6 +212,39 @@ def force_flush(self, timeout_millis: int = 30000) -> bool: # type: ignore[over return True +class BasaltShouldEvaluateProcessor(SpanProcessor): + """ + Span processor that applies the trace-level should_evaluate attribute. + + Reads the should_evaluate decision from OpenTelemetry context and applies + it as a span attribute. This ensures all spans in a trace have the same + should_evaluate value, enabling trace-level sampling for evaluators. + """ + + def on_start(self, span: Span, parent_context: Any | None = None) -> None: # type: ignore[override] + if not span.is_recording(): + return + + from .trace_context import SHOULD_EVALUATE_CONTEXT_KEY + + # Read should_evaluate from context + # Use parent_context if provided, otherwise use current context + ctx = parent_context if parent_context is not None else otel_context.get_current() + should_evaluate = otel_context.get_value(SHOULD_EVALUATE_CONTEXT_KEY, ctx) + + if should_evaluate is not None: + span.set_attribute(semconv.BasaltSpan.SHOULD_EVALUATE, bool(should_evaluate)) + + def on_end(self, span: ReadableSpan) -> None: # type: ignore[override] + return + + def shutdown(self) -> None: # type: ignore[override] + return + + def force_flush(self, timeout_millis: int = 30000) -> bool: # type: ignore[override] + return True + + # Known auto-instrumentation scope names KNOWN_AUTO_INSTRUMENTATION_SCOPES: Final[frozenset[str]] = frozenset({ "opentelemetry.instrumentation.openai", diff --git a/basalt/observability/semconv.py b/basalt/observability/semconv.py index 5cfe3f3..0f25ada 100644 --- a/basalt/observability/semconv.py +++ b/basalt/observability/semconv.py @@ -316,16 +316,15 @@ class BasaltSpan: """ Optional, span-scoped configuration applied to evaluators as a whole. Type: JSON object (string-serialized) or key/value attributes under this prefix - Examples: '{"sample_rate": 0.25, "mode": "async"}' + Examples: '{"sample_rate": 0.25}' """ - # Optional prefix for span-scoped evaluator metadata (not per evaluator) - # Example usage: set attributes like "basalt.span.evaluator.sample_rate" = 0.5 - EVALUATOR_PREFIX: Final[str] = "basalt.span.evaluator" + SHOULD_EVALUATE: Final[str] = "basalt.span.should_evaluate" """ - Prefix for evaluator-related, span-scoped attributes. - Type: various (string, number, boolean) - Examples: "basalt.span.evaluator.sample_rate" = 0.5 + Boolean indicating whether evaluators should run for this span's trace. + Determined once at root span creation via trace-level sampling, propagated to all child spans. + Type: boolean + Value: true (run evaluators) or false (skip evaluators) """ FEATURE_SLUG: Final[str] = "basalt.span.feature_slug" diff --git a/basalt/observability/trace_context.py b/basalt/observability/trace_context.py index c55afbc..4050d24 100644 --- a/basalt/observability/trace_context.py +++ b/basalt/observability/trace_context.py @@ -16,6 +16,7 @@ USER_CONTEXT_KEY: Final[str] = "basalt.context.user" ORGANIZATION_CONTEXT_KEY: Final[str] = "basalt.context.organization" FEATURE_SLUG_CONTEXT_KEY: Final[str] = "basalt.context.feature_slug" +SHOULD_EVALUATE_CONTEXT_KEY: Final[str] = "basalt.context.should_evaluate" @dataclass(frozen=True, slots=True) @@ -44,16 +45,20 @@ class _TraceContextConfig: experiment: TraceExperiment | str | None = None observe_metadata: dict[str, Any] | None = None + sample_rate: float = 0.0 def __post_init__(self) -> None: self.experiment = _coerce_experiment(self.experiment) self.observe_metadata = dict(self.observe_metadata) if self.observe_metadata else {} + if not 0.0 <= self.sample_rate <= 1.0: + raise ValueError("sample_rate must be within [0.0, 1.0].") def clone(self) -> _TraceContextConfig: """Return a defensive copy of the configuration.""" return _TraceContextConfig( experiment=self.experiment, observe_metadata=dict(self.observe_metadata) if self.observe_metadata is not None else {}, + sample_rate=self.sample_rate, ) @@ -106,6 +111,28 @@ def _current_trace_defaults() -> _TraceContextConfig: return _DEFAULT_CONTEXT.clone() +def set_global_sample_rate(sample_rate: float) -> None: + """ + Set the global default sample rate for trace-level evaluation. + + Args: + sample_rate: Sampling rate (0.0-1.0) where 1.0 means 100% sampling. + """ + if not 0.0 <= sample_rate <= 1.0: + raise ValueError("sample_rate must be within [0.0, 1.0].") + + # Take a snapshot of the current defaults under the lock, then + # construct a new config that preserves existing fields while + # updating the sample_rate, and install it via _set_trace_defaults. + with _LOCK: + current = _DEFAULT_CONTEXT.clone() + + new_config = _TraceContextConfig( + experiment=current.experiment, + observe_metadata=current.observe_metadata, + sample_rate=sample_rate, + ) + _set_trace_defaults(new_config) def configure_global_metadata(metadata: dict[str, Any] | None) -> None: """ Configure global observability metadata applied to all traces. diff --git a/examples/gemini_random_data_example.py b/examples/gemini_random_data_example.py index a412f10..4d4abf9 100644 --- a/examples/gemini_random_data_example.py +++ b/examples/gemini_random_data_example.py @@ -1,11 +1,21 @@ +import asyncio import logging import os import httpx from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +try: + from google import genai +except ImportError: + genai = None + from basalt import Basalt, TelemetryConfig -from basalt.observability import ObserveKind, evaluate, observe, start_observe +from basalt.observability import EvaluationConfig, ObserveKind, evaluate, observe, start_observe + +# --- Constants --- +# specific model version to ensure consistency across execution and telemetry +GEMINI_MODEL_NAME = "gemini-2.5-flash-lite" # --- 1. Build Basalt client with custom OTLP exporter --- @@ -17,25 +27,26 @@ def build_custom_exporter_client() -> Basalt: headers if your collector requires authentication. The SDK only adds headers automatically when building the default exporter from environment variables. """ - # Get API key for authentication - api_key = os.getenv("BASALT_API_KEY", "valid-token") + # Get API key for authentication (Fail fast if not set, or use placeholder) + api_key = os.getenv("BASALT_API_KEY") + if not api_key: + logging.warning("BASALT_API_KEY not found. Using placeholder.") + api_key = "place-holder-key" # Use environment variable for OTLP endpoint or default to localhost - otlp_endpoint = os.getenv("BASALT_OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317") + otlp_endpoint = os.getenv("BASALT_OTEL_EXPORTER_OTLP_ENDPOINT", "localhost:4317") # Create a custom exporter with authentication headers - # For local development without authentication, you can omit the headers parameter + # Note: insecure=True is used here for local/demo purposes. + # Use secure credentials for production. exporter = OTLPSpanExporter( - endpoint=otlp_endpoint, - headers={"authorization": f"Bearer {api_key}"}, - insecure=True, - timeout=10 + endpoint=otlp_endpoint, headers={"authorization": f"Bearer {api_key}"}, insecure=True, timeout=10 ) telemetry = TelemetryConfig( service_name="gemini-demo", - exporter=exporter, - llm_enabled_providers=["google_generativeai"], # Only instrument Gemini calls + enabled_providers=["google-genai", "google_generativeai"], + # exporter=[exporter], # Uncommented to make the example functional ) # Initialize Basalt client first (this sets up the TracerProvider) @@ -43,126 +54,112 @@ def build_custom_exporter_client() -> Basalt: return client + basalt_client = build_custom_exporter_client() + # --- 2. Gather random data from a public API --- @observe(name="http.get_random_joke", kind=ObserveKind.RETRIEVAL) -def get_random_joke() -> str: +async def get_random_joke() -> str: """Fetch a random joke from the Official Joke API using httpx (instrumented).""" - with httpx.Client() as client: - resp = client.get("https://official-joke-api.appspot.com/random_joke", timeout=10) + async with httpx.AsyncClient() as client: + resp = await client.get("https://official-joke-api.appspot.com/random_joke", timeout=10) resp.raise_for_status() data = resp.json() logging.debug(f"Joke API response: {data}") return f"{data['setup']} {data['punchline']}" -# --- 3. Fetch a prompt from Basalt API (demonstrates internal instrumentation) --- -def get_prompt_from_basalt(slug: str, joke_text: str, explanation_audience: str) -> str: - """ - Fetch a prompt from Basalt's Prompt API. - - This call is automatically traced by the Basalt SDK, demonstrating - internal HTTP call instrumentation. - """ - try: - - prompt = basalt_client.prompts.get_sync( - slug, - variables={ - "jokeText": joke_text, - "explanationAudience": explanation_audience, - }, - ) - logging.info(f"Fetched prompt: {prompt.slug} (version: {prompt.version})") - return prompt.text - except Exception as exc: - logging.warning(f"Failed to fetch prompt '{slug}': {exc}") - raise - - -# --- 4. Query Gemini (Google AI Studio) with the random data --- -try: - from google import genai -except ImportError: - genai = None - +# --- 3. Query Gemini (Google AI Studio) with the random data --- @evaluate(slugs=["hallucinations", "clarity"]) -def summarize_joke_with_gemini(joke: str) -> str | None: +async def summarize_joke_with_gemini(joke: str) -> str | None: """ Send the joke to Gemini and get a summary or explanation. - - The @evaluator decorator will: - - Attach evaluator slugs to the auto-instrumented Gemini span - - Set config with sample_rate to the span - - Resolve and attach metadata (joke_length) to the span - - All of this happens automatically via the BasaltCallEvaluatorProcessor - when the Gemini instrumentation creates its span! """ if genai is None: raise RuntimeError("google-genai is not installed") - model_name = "gemini-2.5-flash-lite" - client = genai.Client(api_key=os.getenv("GEMINI_API_KEY", "fake-key")) - response = client.models.generate_content(model=model_name, contents=joke) + # Use a dummy key if env var is missing to allow code to 'run' (and fail gracefully) + # rather than crash on import or setup. + gemini_key = os.getenv("GEMINI_API_KEY", "dummy-key") + + client = genai.Client(api_key=gemini_key) + async with client.aio as aclient: + response = await aclient.models.generate_content(model=GEMINI_MODEL_NAME, contents=joke) logging.debug(f"Gemini response: {getattr(response, 'text', response)}") return response.text +# --- 4. Main Workflow --- @start_observe( - name="workflow.gemini_random_data", - identity={ - "organization": {"id": "123", "name": "ACME"}, - "user": {"id": "456", "name": "John Doe"} - }, - metadata={ - "workflow.type": "gemini-joke-demo", - "service": "gemini-random-demo" - } + name="workflow.random_data_pipeline", + feature_slug="support-ticket", + evaluate_config=EvaluationConfig(sample_rate=0.5), ) -def main(): - logging.basicConfig(level=logging.DEBUG) - +async def start_workflow() -> None: # 1. Fetch a random joke using httpx (external HTTP call - instrumented) - joke = get_random_joke() + joke = await get_random_joke() - observe.metadata({"joke.length": len(joke)}) - observe.input({"joke": joke}) + observe.set_metadata({"joke.length": len(joke)}) + observe.set_input({"joke": joke}) logging.info(f"Random joke: {joke}") # 2. Fetch a prompt from Basalt API (internal SDK call - instrumented) prompt_slug = "joke-analyzer" - prompt_text = get_prompt_from_basalt(prompt_slug, joke, "a curious geek adult") - #logging.info(f"Basalt prompt preview: {prompt_text[:100]}...") # 3. Query Gemini with the joke (LLM call - instrumented) try: - gemini_result = summarize_joke_with_gemini(prompt_text) - logging.info(f"Gemini summary: {gemini_result}") - observe.metadata({ - "gemini.status": "success", - "gemini.response_length": len(gemini_result) if gemini_result else 0 - }) - observe.output({ - "summary": gemini_result, - "status": "success", - }) + prompt_context_manager = await basalt_client.prompts.get( + prompt_slug, + variables={ + "jokeText": joke, + "explanationAudience": "a curious geek adult", + }, + ) + async with prompt_context_manager as prompt_context: + gemini_result = await summarize_joke_with_gemini(prompt_context.text) + + logging.info(f"Gemini summary: {gemini_result}") + + observe.set_metadata( + {"gemini.status": "success", "gemini.response_length": len(gemini_result) if gemini_result else 0} + ) + + # Use the constant to ensure attributes match the actual model used + observe.set_attributes({"gemini.model": GEMINI_MODEL_NAME}) + + observe.set_output( + { + "summary": gemini_result, + "status": "success", + } + ) except Exception as exc: logging.error(f"Gemini error: {exc}") - observe.fail(exc) - observe.metadata({"gemini.status": "error"}) - observe.output({ - "status": "error", - "error": str(exc), - }) - - # Shutdown and flush telemetry - logging.info("Shutting down and flushing telemetry...") - basalt_client.shutdown() + observe.set_metadata({"gemini.status": "error"}) + observe.set_output( + { + "status": "error", + "error": str(exc), + } + ) + + +def main(): + logging.basicConfig(level=logging.DEBUG) + + try: + asyncio.run(start_workflow()) + except KeyboardInterrupt: + pass + finally: + # Shutdown and flush telemetry + logging.info("Shutting down and flushing telemetry...") + basalt_client.shutdown() + if __name__ == "__main__": main() diff --git a/examples/mistral_example.py b/examples/mistral_example.py deleted file mode 100644 index 8b2df59..0000000 --- a/examples/mistral_example.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Mistral + Basalt Observability Example (Updated Client). - -Demonstrates manual instrumentation of a Mistral chat completion using the new -`mistralai` client API (post-migration from deprecated `MistralClient`). The -flow wraps a workflow span and an LLM generation span with explicit input/output -metadata for Basalt observability. - -Environment Variables: - BASALT_API_KEY - Basalt API key (falls back to test-key). - MISTRAL_API_KEY - Mistral API key (required for real responses). - MISTRAL_MODEL - Optional model alias/name (e.g. 'small', 'tiny'). -""" - -from __future__ import annotations - -import logging -import os -from collections.abc import Sequence - -from mistralai import Mistral, UserMessage -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter - -from basalt import Basalt -from basalt.observability import ObserveKind, observe, start_observe -from basalt.observability.config import TelemetryConfig - -# --------------------------------------------------------------------------- -# Environment bootstrap -# --------------------------------------------------------------------------- -if "BASALT_API_KEY" not in os.environ: - os.environ["BASALT_API_KEY"] = "test-key" - -API_KEY = os.environ.get("MISTRAL_API_KEY", "") # Empty -> likely to error for real call - -# Use environment variable for OTLP endpoint or default to localhost -OTLP_ENDPOINT = os.environ.get("BASALT_OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317") - -exporter = OTLPSpanExporter( - endpoint=OTLP_ENDPOINT, - headers={"authorization": f"Bearer {os.environ['BASALT_API_KEY']}"}, - insecure=True, - timeout=10 -) - -telemetry = TelemetryConfig( - service_name="gemini-demo", - exporter=exporter, - enable_llm_instrumentation=True, # Enable automatic Gemini instrumentation - llm_trace_content=True, - llm_enabled_providers=["google_generativeai", "openai"], # Only instrument Gemini calls -) - -# --------------------------------------------------------------------------- -# Basalt client initialization -# --------------------------------------------------------------------------- -client = Basalt( - api_key=os.environ["BASALT_API_KEY"], - observability_metadata={ - "env": "staging", - "provider": "mistral", - "example": "manual-instrumentation" - }, - telemetry_config=telemetry -) - -# --------------------------------------------------------------------------- -# Mistral client (new API) -# --------------------------------------------------------------------------- -mistral_client = Mistral(api_key=API_KEY) - - -def resolve_model(raw: str) -> str: - """Resolve a user-provided model alias to a full model name.""" - aliases = { - "small": "mistral-small-latest", - "tiny": "mistral-tiny", - "open-7b": "open-mistral-7b", - "7b": "open-mistral-7b", - "large": "mistral-large-latest", - } - return aliases.get(raw, raw) if raw else "mistral-small-latest" - - -def run_mistral_flow(topic: str) -> str: - """Execute a simple observed workflow that queries Mistral. - - Parameters: - topic: Subject to explain. - - Returns: - Assistant response string (or fallback message if unavailable). - """ - with start_observe( - name="mistral_workflow", - identity={ - "organization": {"id": "123", "name": "ACME"}, - "user": {"id": "456", "name": "John Doe"} - }, - metadata={"provider": "mistral", "model_type": "chat"}, - ): - observe.input({"topic": topic}) - - with observe(name="mistral_chat", kind=ObserveKind.GENERATION) as llm_span: - model = resolve_model(os.environ.get("MISTRAL_MODEL", "")) - # Prefer typed message helper for compatibility with the SDK - messages: Sequence[UserMessage] = [ - UserMessage(content=f"Explain {topic} in one concise sentence.") - ] - - llm_span.set_input({"messages": messages}) - llm_span.set_attribute("llm.model", model) - llm_span.set_attribute("llm.provider", "mistral") - - try: - response = mistral_client.chat.complete(model=model, messages=list(messages)) - # Some responses may return complex content objects; convert to string safely - raw_content = response.choices[0].message.content - content = str(raw_content) if raw_content is not None else "" - llm_span.set_output(content) - if getattr(response, "usage", None): - llm_span.set_attribute("llm.usage.total_tokens", response.usage.total_tokens) - return content - except Exception as exc: # noqa: BLE001 - fallback = f"Mistral call failed: {exc}"[:250] - llm_span.set_output(fallback) - llm_span.set_attribute("error", str(exc)) - return fallback - - -def main() -> None: - logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s") - result = run_mistral_flow("Quantum Physics") - logging.info("Result: %s", result) - - -if __name__ == "__main__": - main() diff --git a/examples/observability_playbook.ipynb b/examples/observability_playbook.ipynb deleted file mode 100644 index 2fc0213..0000000 --- a/examples/observability_playbook.ipynb +++ /dev/null @@ -1,286 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7fb27b941602401d91542211134fc71a", - "metadata": {}, - "source": [ - "# Basalt Observability End-to-End Playbook\n", - "\n", - "This notebook demonstrates how to combine Basalt's observability helpers with common LLM flows, evaluators,\n", - "experiment tagging, and Google AI Studio (Gemini) interactions. Each scenario demonstrates modern OpenTelemetry-based\n", - "observability patterns with the Basalt SDK." - ] - }, - { - "cell_type": "markdown", - "id": "acae54e37e7d407bbb7b55eff062a284", - "metadata": {}, - "source": [ - "## Prerequisites\n", - "\n", - "- Install the SDK in editable mode with dev extras: `uv pip install -e \".[dev]\"`\n", - "- (Optional) Install the Google Generative AI SDK: `pip install google-generativeai`\n", - "- Set the following environment variables before running the notebook:\n", - " - `BASALT_API_KEY` – your Basalt API key\n", - " - `GOOGLE_API_KEY` – Google AI Studio key (Gemini)\n", - " - `TRACELOOP_TRACE_CONTENT=1` if you want prompts/completions captured\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a63283cbaf04dbcab1f6479b197f3a8", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "from basalt import Basalt\n", - "from basalt.observability import evaluate, observe, start_observe\n", - "\n", - "try:\n", - " from google import genai\n", - "except ImportError: # pragma: no cover - optional dependency\n", - " genai = None" - ] - }, - { - "cell_type": "markdown", - "id": "8dd0d8092fe74a7c96281538738b07e2", - "metadata": {}, - "source": [ - "## 1. Configure the Basalt client\n", - "\n", - "Initialize the Basalt client with telemetry configuration. Use `start_observe` to create root spans with identity and experiment tracking." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72eea5119410473aa328ad9291626812", - "metadata": {}, - "outputs": [], - "source": [ - "from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter\n", - "\n", - "from basalt import TelemetryConfig\n", - "\n", - "# Create telemetry configuration\n", - "exporter = OTLPSpanExporter(endpoint=\"http://127.0.0.1:4317\", insecure=True)\n", - "telemetry = TelemetryConfig(service_name=\"notebook\", exporter=exporter)\n", - "\n", - "# Initialize Basalt client\n", - "basalt_client = Basalt(\n", - " api_key=os.getenv(\"BASALT_API_KEY\", \"not-set\"),\n", - " telemetry_config=telemetry,\n", - ")\n", - "\n", - "basalt_client" - ] - }, - { - "cell_type": "markdown", - "id": "8edb47106e1a46a883d545849b8ab81b", - "metadata": {}, - "source": [ - "## 2. Decorator-driven LLM spans with identity and evaluators\n", - "\n", - "Use `@start_observe` as the root span decorator with identity tracking." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "10185d26023b46108eb7d9f57d49d2b3", - "metadata": {}, - "outputs": [], - "source": [ - "@evaluate(slugs=[\"accuracy\", \"toxicity\"])\n", - "def summarize_with_gemini(prompt: str, *, model: str = \"gemini-2.5-flash-lite\") -> str | None:\n", - " \"\"\"\n", - " Summarize text using Gemini with automatic tracing.\n", - "\n", - " The @evaluate decorator attaches evaluators to the span.\n", - " \"\"\"\n", - " if genai is None:\n", - " raise RuntimeError(\"google-genai is not installed\")\n", - "\n", - " client = genai.Client(api_key=os.getenv(\"GOOGLE_API_KEY\", \"fake-key\"))\n", - " response = client.models.generate_content(model=model, contents=prompt)\n", - " return response.text\n", - "\n", - "# Wrap the call in a root span with identity\n", - "@start_observe(\n", - " name=\"notebook.workflow\",\n", - " identity={\n", - " \"organization\": {\"id\": \"123\", \"name\": \"ACME\"},\n", - " \"user\": {\"id\": \"456\", \"name\": \"John Doe\"}\n", - " },\n", - " experiment={\"id\": \"exp-observability\", \"name\": \"demo-agent\"},\n", - " metadata={\"environment\": \"notebook\", \"workspace\": \"demo\"}\n", - ")\n", - "def run_gemini_demo():\n", - " try:\n", - " return summarize_with_gemini(\"Summarize the benefits of synthetic monitoring.\")\n", - " except Exception as exc:\n", - " observe.fail(exc)\n", - " return {\"error\": str(exc)}\n", - "\n", - "gemini_result = run_gemini_demo()" - ] - }, - { - "cell_type": "markdown", - "id": "8763a12b2bbd4a93a75aff182afb95dc", - "metadata": {}, - "source": [ - "## 3. Manual spans for orchestrating workflow stages\n", - "\n", - "Use `start_observe` for the root workflow span, then nest `observe(kind=...)` for retrieval, tool, generation, and event spans." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7623eae2785240b9bd12b16a66d81610", - "metadata": {}, - "outputs": [], - "source": [ - "with start_observe(\n", - " name=\"workflow.rag\",\n", - " identity={\n", - " \"organization\": {\"id\": \"123\", \"name\": \"ACME\"},\n", - " \"user\": {\"id\": \"456\", \"name\": \"John Doe\"}\n", - " },\n", - " # Only attach the experiment id; descriptive fields moved to metadata\n", - " experiment={\"id\": \"exp-rag-001\"},\n", - " metadata={\n", - " \"feature\": \"support-bot\",\n", - " \"experiment.name\": \"support-bot\"\n", - " }\n", - "):\n", - " # Set evaluators on root span\n", - " observe.evaluate(\"latency-budget\")\n", - " observe.input({\"query\": \"error connecting to database\"})\n", - "\n", - " # Step 1: Retrieval span for vector database search\n", - " with observe(kind=\"retrieval\", name=\"workflow.rag.retrieve\") as ret_span:\n", - " ret_span.set_attribute(\"query\", \"database_error\")\n", - " ret_span.set_attribute(\"results_count\", 3)\n", - " ret_span.set_attribute(\"top_k\", 5)\n", - "\n", - "\n", - " # Step 2: Tool span for external tool call (e.g., web search)\n", - " with observe(kind=\"tool\", name=\"workflow.rag.tool\") as tool_span:\n", - " tool_span.set_attribute(\"tool_name\", \"web-search\")\n", - " tool_span.set_input({\"query\": \"database connection refused troubleshooting\"})\n", - " tool_span.set_output({\"summary\": \"Check credentials and firewall rules.\"})\n", - "\n", - " # Step 3: LLM generation span for final answer\n", - " with observe(kind=\"generation\", name=\"workflow.rag.answer\") as llm_span:\n", - " llm_span.set_attribute(\"model\", \"gemini-1.5-flash\")\n", - " llm_span.set_attribute(\"prompt\", \"Provide mitigation steps\")\n", - " llm_span.set_attribute(\"completion\", \"1. Verify credentials...\")\n", - " llm_span.set_attribute(\"tokens_input\", 200)\n", - " llm_span.set_attribute(\"tokens_output\", 180)\n", - "\n", - " # Step 4: Event span for workflow state tracking\n", - " with observe(kind=\"event\", name=\"workflow.rag.event\") as event_span:\n", - " event_span.set_attribute(\"event_type\", \"handoff\")\n", - " event_span.set_attribute(\"payload\", {\"team\": \"support\", \"status\": \"ready\"})\n", - "\n", - " observe.output({\"status\": \"completed\", \"steps\": 4})" - ] - }, - { - "cell_type": "markdown", - "id": "7cdc8c89c7104fffa095e18ddfef8986", - "metadata": {}, - "source": [ - "## 4. Dynamic experiment and metadata updates\n", - "\n", - "Use `observe.experiment()` to attach or override experiment metadata, and `observe.metadata()` to add custom attributes to the current span." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b118ea5561624da68c537baed56e602f", - "metadata": {}, - "outputs": [], - "source": [ - "with start_observe(\n", - " name=\"workflow.ab-test\",\n", - " identity={\n", - " \"organization\": {\"id\": \"123\", \"name\": \"ACME\"},\n", - " \"user\": {\"id\": \"456\", \"name\": \"John Doe\"}\n", - " },\n", - " # Keep only the experiment id; move name & feature slug to metadata\n", - " experiment={\"id\": \"exp-baseline\"},\n", - " metadata={\n", - " \"experiment.name\": \"baseline-playbook\",\n", - " \"feature.slug\": \"demo-agent\"\n", - " }\n", - "):\n", - " # Override experiment dynamically (variant stored as attribute on override span)\n", - " observe.experiment(\"exp-variant\", variant=\"demo-agent-b\")\n", - "\n", - " # Add evaluators and metadata\n", - " observe.evaluate(\"judge-hallucination\")\n", - "\n", - " observe.metadata({\"latency_ms\": 245, \"variant_override\": True})\n", - " observe.output({\"status\": \"ab_test_complete\"})" - ] - }, - { - "cell_type": "markdown", - "id": "938c804e27f84196a10c8828c723f798", - "metadata": {}, - "source": [ - "## 5. Shutdown and cleanup\n", - "\n", - "Flush telemetry buffers when the workflow completes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "504fb2a444614c0babb325280ed9130a", - "metadata": {}, - "outputs": [], - "source": [ - "basalt_client.shutdown()" - ] - }, - { - "cell_type": "markdown", - "id": "59bbdb311c014d738909a11f9e486628", - "metadata": {}, - "source": [ - "\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "basalt-sdk", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.8" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/observe_decorator_example.py b/examples/observe_decorator_example.py deleted file mode 100644 index cd81253..0000000 --- a/examples/observe_decorator_example.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Example demonstrating the new observe decorator API.""" - -from basalt.observability import ( - ObserveKind, - observe, - start_observe, -) - - -@observe(kind=ObserveKind.SPAN, name="process.data") -def process_data(text: str) -> str: - """Process some data.""" - return text.upper() - - -@observe(kind=ObserveKind.GENERATION, name="llm.generate") -def generate_text(prompt: str, model: str = "gpt-4") -> str: - """Generate text with an LLM.""" - # Simulated LLM call - return f"Generated response for: {prompt}" - - -@observe( name="vector.search", kind=ObserveKind.RETRIEVAL) -def search_documents(query: str) -> list[dict]: - """Search documents in vector database.""" - return [ - {"id": 1, "content": "Document 1", "score": 0.95}, - {"id": 2, "content": "Document 2", "score": 0.87}, - ] - - -@observe(kind=ObserveKind.SPAN, name="workflow.execute") -def execute_workflow(steps: list[str]) -> dict: - """Execute a workflow with multiple steps.""" - return {"status": "completed", "steps_executed": len(steps)} - - -@observe( - name="llm.chat", - kind=ObserveKind.GENERATION, - evaluators=["quality", "relevance"], -) -def chat_with_llm(message: str) -> str: - """Chat with an LLM with quality evaluation.""" - return f"Response to: {message}" - - - - -@start_observe( - name="main_workflow", - identity={ - "organization": {"id": "123", "name": "Demo Corp"}, - "user": {"id": "456", "name": "Alice"} - }, - experiment={"id": "exp_123"}, - metadata={"environment": "demo"}, -) -def main(): - """Main workflow demonstrating nested observe calls.""" - # Test the new decorators - observe.input({"workflow": "demo_suite"}) - - result1 = process_data("hello world") - - result2 = generate_text("Write a poem") - - result3 = search_documents("machine learning") - - result4 = execute_workflow(["step1", "step2", "step3"]) - - result5 = chat_with_llm("How are you?") - - observe.output({"results_count": 5}) - return {"completed": True} - - -if __name__ == "__main__": - main() - - - diff --git a/examples/openai_example.py b/examples/openai_example.py index 6109ddd..4c00519 100644 --- a/examples/openai_example.py +++ b/examples/openai_example.py @@ -2,104 +2,130 @@ import logging import os -from openai import AzureOpenAI, OpenAI +from openai import OpenAI from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from basalt import Basalt -from basalt.observability import ObserveKind, evaluate, observe, start_observe +from basalt.observability import Identity, ObserveKind, evaluate, observe, start_observe from basalt.observability.config import TelemetryConfig -logging.basicConfig(level=logging.INFO) - -# Ensure API keys are set -if "BASALT_API_KEY" not in os.environ: - os.environ["BASALT_API_KEY"] = "test-key" -if "OPENAI_API_KEY" not in os.environ: - pass - # We don't exit to allow syntax checking, but real run needs key - -# Use environment variable for OTLP endpoint or default to localhost -otlp_endpoint = os.getenv("BASALT_OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317") - -exporter = OTLPSpanExporter( - endpoint="http://localhost:4317", - headers={"authorization": f"Bearer {os.environ['BASALT_API_KEY']}"}, - insecure=True, - timeout=10 -) +# --- Constants --- +# Use a standard model name that users are likely to have access to +OPENAI_MODEL_NAME = "gpt-4o-mini" + + +def build_basalt_client(): + """ + Initialize the Basalt client with specific telemetry configurations. + """ + # 1. Setup API Keys & Endpoints + basalt_key = os.getenv("BASALT_API_KEY") + if not basalt_key: + logging.warning("BASALT_API_KEY not found. Using placeholder.") + basalt_key = "test-key" + + otlp_endpoint = os.getenv("BASALT_OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317") + + # 2. Configure Exporter + # Note: insecure=True is for local/demo only. Use secure credentials for production. + exporter = OTLPSpanExporter( + endpoint=otlp_endpoint, + headers={"authorization": f"Bearer {basalt_key}"}, + insecure=True, + timeout=10, + ) + # 3. Configure Telemetry + telemetry = TelemetryConfig( + service_name="openai-demo", + exporter=exporter, + enabled_providers=["openai"], # Only instrument OpenAI calls + ) -telemetry = TelemetryConfig( - service_name="openai-demo", - exporter=exporter, - enabled_providers=["openai"], # Only instrument OpenAI calls + # 4. Return Client + return Basalt( + api_key=basalt_key, + observability_metadata={"env": "development", "provider": "openai", "example": "auto-instrumentation"}, + telemetry_config=telemetry, ) -# Initialize Basalt -# Auto-instrumentation for OpenAI is enabled by default when the library is installed. -client = Basalt( - api_key=os.environ["BASALT_API_KEY"], - observability_metadata={ - "env": "development", - "provider": "openai", - "example": "auto-instrumentation" - }, - telemetry_config=telemetry -) +# Initialize global client (logging should be configured before this in main, but for module level: careful) +# In a real app, do this inside startup logic. +basalt_client = build_basalt_client() +# Initialize OpenAI Client securely +openai_api_key = os.getenv("OPENAI_API_KEY") +if openai_api_key: + openai_client = OpenAI(api_key=openai_api_key) +else: + logging.warning("OPENAI_API_KEY not set. OpenAI calls will be mocked.") + openai_client = None -openai_api_version = os.environ.get("OPENAI_API_VERSION", "2025-03-01-preview") -azure_endpoint = os.environ.get("AZURE_OPENAI_ENDPOINT") -openai_api_key = os.environ.get("OPENAI_API_KEY") -# openai_client = AzureOpenAI(api_key=openai_api_key, api_version=openai_api_version, azure_endpoint=azure_endpoint) -openai_client = OpenAI(api_key=openai_api_key) @observe(kind=ObserveKind.TOOL, name="get_weather") def get_weather(location: str): """Mock weather tool.""" - observe.set_metadata({"tool_used": "get_weather"}) - observe.set_metadata({"location": location}) + # Observe metadata helps track tool usage in traces + observe.set_metadata({"tool_used": "get_weather", "location": location}) return json.dumps({"location": location, "temperature": "22C", "condition": "Sunny"}) -@start_observe( - name="weather_assistant", - feature_slug="weather_api", - identity={ - "organization": {"id": "123", "name": "ACME"}, - "user": {"id": "456", "name": "John Doe"} - }, - metadata={"service_metadata_Start": "weather_api_start"}, -) + @evaluate("helpfulness") def run_weather_assistant(user_query: str): - observe.set_input({"query": user_query}) - - # 1. Mock Tool Call (simulating a decision to call a tool) - weather_data = get_weather("San Francisco, CA") - - # 2. Real LLM Call (Auto-instrumented) - # Basalt automatically captures the span, input (messages), and output (content). - if openai_client is None: - # If the samples are executed locally without an API key, provide a mock response - content = "This is a mock response because OPENAI_API_KEY is not set." - else: - response = openai_client.chat.completions.create( - model="gpt-5-mini", - messages=[ - {"role": "system", "content": "You are a helpful weather assistant."}, - {"role": "user", "content": f"Context: {weather_data}\n\nQuery: {user_query}"} - ] - ) - - content = response.choices[0].message.content - # Ensure we always pass a string slice to the output - content_str = (content or "")[:100] - observe.set_output({"response": content_str}) - - return content - -try: - result = run_weather_assistant("What's the weather like in SF?") -except Exception: - pass + # 'start_observe' creates a span for this workflow + with start_observe( + name="weather_assistant", + feature_slug="weather-assistant", + identity=Identity(organization={"id": "123", "name": "Demo Corp"}, user={"id": "456", "name": "Alice"}), + ) as span: + span.set_input({"query": user_query}) + + # 1. Mock Tool Call + weather_data = get_weather("San Francisco, CA") + + # 2. LLM Call + if openai_client is None: + content = "Mock response: OPENAI_API_KEY is missing." + else: + # We wrap the call in a prompt context. Even if we don't explicitly inject the text, + # Basalt tracks that this specific prompt version was active during the LLM call. + # We changed the slug to 'weather-system-prompt' to make semantic sense. + with basalt_client.prompts.get_sync( + "weather-system-prompt", + variables={"location": "San Francisco"}, + ): + response = openai_client.chat.completions.create( + model=OPENAI_MODEL_NAME, + messages=[ + {"role": "system", "content": "You are a helpful weather assistant."}, + {"role": "user", "content": f"Context: {weather_data}\n\nQuery: {user_query}"}, + ], + ) + content = response.choices[0].message.content + + # 3. Finalize + # Ensure we always pass a string to the output for consistent logging + content_str = (content or "")[:100] + observe.set_output({"response": content_str}) + + return content + + +def main(): + logging.basicConfig(level=logging.INFO) + + try: + # Run the workflow + result = run_weather_assistant("What's the weather like in SF?") + logging.info(f"Final Result: {result}") + except Exception as e: + logging.error(f"Workflow failed: {e}") + finally: + # CRITICAL: Shutdown ensures traces are flushed to the exporter before exit + logging.info("Flushing telemetry...") + basalt_client.shutdown() + + +if __name__ == "__main__": + main() diff --git a/examples/prompts_context_manager_example.py b/examples/prompts_context_manager_example.py deleted file mode 100644 index f1f38b2..0000000 --- a/examples/prompts_context_manager_example.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Example demonstrating the context manager pattern for prompts. - -This example shows how to use prompts.get_sync() as a context manager -to automatically scope auto-instrumented GenAI calls. -""" -import os - -from basalt import Basalt - -# Initialize Basalt client -client = Basalt( - api_key=os.environ.get("BASALT_API_KEY", "test-key"), -) - -# Example 1: Context Manager Pattern (Recommended for Observability) -# This creates a span that scopes any LLM calls within the context - -with client.prompts.get_sync("qa-prompt", variables={"context": "Paris is the capital of France"}) as prompt: - - # Any auto-instrumented LLM calls here would automatically nest - # under the prompt span in the trace - # Example: - # response = openai.chat.completions.create( - # model=prompt.model.model, - # messages=[{"role": "user", "content": prompt.text}] - # ) - - pass - - -# Example 2: Imperative Pattern (Backward Compatible, Always Instrumented) -# This creates a span that immediately ends - -prompt = client.prompts.get_sync("qa-prompt", variables={"context": "London is the capital of UK"}) - -# LLM calls here create their own separate spans -# Example: -# response = openai.chat.completions.create( -# model=prompt.model.model, -# messages=[{"role": "user", "content": prompt.text}] -# ) - - - -# Example 3: Async Context Manager Pattern - - -async def async_example(): - async with await client.prompts.get("qa-prompt", variables={"context": "Berlin is the capital of Germany"}): - - # Async LLM calls would nest under this span - # Example: - # async with httpx.AsyncClient() as http_client: - # response = await openai_async.chat.completions.create(...) - - pass - -# Uncomment to run async example: -# asyncio.run(async_example()) - diff --git a/pyproject.toml b/pyproject.toml index a9bc9fa..be173c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,9 +48,10 @@ google-generativeai = [ "opentelemetry-instrumentation-google-generativeai~=0.48.0", ] # Note: google-genai instrumentation not yet available in openllmetry -# google-genai = [ -# "opentelemetry-instrumentation-google-genai~=0.48.0", -# ] + google-genai = [ + "opentelemetry-instrumentation-google-genai~=0.5b0", + ] + bedrock = [ "opentelemetry-instrumentation-bedrock~=0.48.0", ] @@ -123,6 +124,7 @@ dev = [ "google-cloud-aiplatform>=1.127.0", "opentelemetry-instrumentation-anthropic>=0.48.1", "opentelemetry-instrumentation-google-generativeai>=0.48.1", + "opentelemetry-instrumentation-google-genai", "opentelemetry-instrumentation-openai>=0.48.1", "opentelemetry-instrumentation-vertexai>=0.48.1", ] diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 887e3c5..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,399 +0,0 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile pyproject.toml --extra dev -c requirements.txt -o requirements-dev.txt -annotated-types==0.7.0 - # via pydantic -anthropic==0.73.0 - # via basalt-sdk (pyproject.toml) -anyio==4.11.0 - # via - # -c requirements.txt - # anthropic - # google-genai - # httpx - # openai -cachetools==6.2.1 - # via google-auth -certifi==2025.10.5 - # via - # -c requirements.txt - # httpcore - # httpx - # requests -cffi==2.0.0 - # via cryptography -charset-normalizer==3.4.4 - # via - # -c requirements.txt - # requests -coverage==7.11.3 - # via - # basalt-sdk (pyproject.toml) - # pytest-cov -cryptography==46.0.3 - # via secretstorage -distro==1.9.0 - # via - # anthropic - # openai -docstring-parser==0.17.0 - # via - # anthropic - # google-cloud-aiplatform -docutils==0.22.2 - # via readme-renderer -google-api-core==2.28.1 - # via - # google-cloud-aiplatform - # google-cloud-bigquery - # google-cloud-core - # google-cloud-resource-manager - # google-cloud-storage -google-auth==2.41.1 - # via - # google-api-core - # google-cloud-aiplatform - # google-cloud-bigquery - # google-cloud-core - # google-cloud-resource-manager - # google-cloud-storage - # google-genai -google-cloud-aiplatform==1.127.0 - # via basalt-sdk (pyproject.toml) -google-cloud-bigquery==3.38.0 - # via google-cloud-aiplatform -google-cloud-core==2.5.0 - # via - # google-cloud-bigquery - # google-cloud-storage -google-cloud-resource-manager==1.15.0 - # via google-cloud-aiplatform -google-cloud-storage==3.6.0 - # via google-cloud-aiplatform -google-crc32c==1.7.1 - # via - # google-cloud-storage - # google-resumable-media -google-genai==1.50.1 - # via - # basalt-sdk (pyproject.toml) - # google-cloud-aiplatform -google-resumable-media==2.8.0 - # via - # google-cloud-bigquery - # google-cloud-storage -googleapis-common-protos==1.71.0 - # via - # -c requirements.txt - # google-api-core - # grpc-google-iam-v1 - # grpcio-status - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -grpc-google-iam-v1==0.14.3 - # via google-cloud-resource-manager -grpcio==1.76.0 - # via - # -c requirements.txt - # google-api-core - # google-cloud-resource-manager - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-status - # opentelemetry-exporter-otlp-proto-grpc -grpcio-status==1.76.0 - # via google-api-core -h11==0.16.0 - # via - # -c requirements.txt - # httpcore -httpcore==1.0.9 - # via - # -c requirements.txt - # httpx -httpx==0.28.1 - # via - # -c requirements.txt - # basalt-sdk (pyproject.toml) - # anthropic - # google-genai - # openai -id==1.5.0 - # via twine -idna==3.11 - # via - # -c requirements.txt - # anyio - # httpx - # requests -importlib-metadata==8.7.0 - # via - # -c requirements.txt - # opentelemetry-api -iniconfig==2.3.0 - # via pytest -jaraco-classes==3.4.0 - # via keyring -jaraco-context==6.0.1 - # via keyring -jaraco-functools==4.3.0 - # via keyring -jeepney==0.9.0 - # via - # keyring - # secretstorage -jinja2==3.1.6 - # via - # -c requirements.txt - # basalt-sdk (pyproject.toml) -jiter==0.12.0 - # via - # anthropic - # openai -keyring==25.6.0 - # via twine -markdown-it-py==4.0.0 - # via rich -markupsafe==3.0.3 - # via - # -c requirements.txt - # jinja2 -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.8.0 - # via - # jaraco-classes - # jaraco-functools -nh3==0.3.1 - # via readme-renderer -numpy==2.3.5 - # via shapely -openai==2.8.0 - # via basalt-sdk (pyproject.toml) -opentelemetry-api==1.38.0 - # via - # -c requirements.txt - # basalt-sdk (pyproject.toml) - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-instrumentation - # opentelemetry-instrumentation-anthropic - # opentelemetry-instrumentation-google-generativeai - # opentelemetry-instrumentation-httpx - # opentelemetry-instrumentation-openai - # opentelemetry-instrumentation-vertexai - # opentelemetry-sdk - # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.38.0 - # via - # -c requirements.txt - # basalt-sdk (pyproject.toml) -opentelemetry-exporter-otlp-proto-common==1.38.0 - # via - # -c requirements.txt - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.38.0 - # via - # -c requirements.txt - # opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.38.0 - # via - # -c requirements.txt - # opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.59b0 - # via - # -c requirements.txt - # basalt-sdk (pyproject.toml) - # opentelemetry-instrumentation-anthropic - # opentelemetry-instrumentation-google-generativeai - # opentelemetry-instrumentation-httpx - # opentelemetry-instrumentation-openai - # opentelemetry-instrumentation-vertexai -opentelemetry-instrumentation-anthropic==0.48.1 - # via basalt-sdk (pyproject.toml) -opentelemetry-instrumentation-google-generativeai==0.48.1 - # via basalt-sdk (pyproject.toml) -opentelemetry-instrumentation-httpx==0.59b0 - # via - # -c requirements.txt - # basalt-sdk (pyproject.toml) -opentelemetry-instrumentation-openai==0.48.1 - # via basalt-sdk (pyproject.toml) -opentelemetry-instrumentation-vertexai==0.48.1 - # via basalt-sdk (pyproject.toml) -opentelemetry-proto==1.38.0 - # via - # -c requirements.txt - # opentelemetry-exporter-otlp-proto-common - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.38.0 - # via - # -c requirements.txt - # basalt-sdk (pyproject.toml) - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.59b0 - # via - # -c requirements.txt - # basalt-sdk (pyproject.toml) - # opentelemetry-instrumentation - # opentelemetry-instrumentation-anthropic - # opentelemetry-instrumentation-google-generativeai - # opentelemetry-instrumentation-httpx - # opentelemetry-instrumentation-openai - # opentelemetry-instrumentation-vertexai - # opentelemetry-sdk -opentelemetry-semantic-conventions-ai==0.4.13 - # via - # opentelemetry-instrumentation-anthropic - # opentelemetry-instrumentation-google-generativeai - # opentelemetry-instrumentation-openai - # opentelemetry-instrumentation-vertexai -opentelemetry-util-http==0.59b0 - # via - # -c requirements.txt - # opentelemetry-instrumentation-httpx -packaging==25.0 - # via - # -c requirements.txt - # google-cloud-aiplatform - # google-cloud-bigquery - # opentelemetry-instrumentation - # pytest - # twine -parameterized==0.9.0 - # via basalt-sdk (pyproject.toml) -pluggy==1.6.0 - # via - # pytest - # pytest-cov -proto-plus==1.26.1 - # via - # google-api-core - # google-cloud-aiplatform - # google-cloud-resource-manager -protobuf==6.33.0 - # via - # -c requirements.txt - # google-api-core - # google-cloud-aiplatform - # google-cloud-resource-manager - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-status - # opentelemetry-proto - # proto-plus -pyasn1==0.6.1 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.2 - # via google-auth -pycparser==2.23 - # via cffi -pydantic==2.12.3 - # via - # anthropic - # google-cloud-aiplatform - # google-genai - # openai -pydantic-core==2.41.4 - # via pydantic -pygments==2.19.2 - # via - # pytest - # readme-renderer - # rich -pytest==9.0.1 - # via - # basalt-sdk (pyproject.toml) - # pytest-asyncio - # pytest-cov -pytest-asyncio==1.3.0 - # via basalt-sdk (pyproject.toml) -pytest-cov==7.0.0 - # via basalt-sdk (pyproject.toml) -python-dateutil==2.9.0.post0 - # via google-cloud-bigquery -readme-renderer==44.0 - # via twine -requests==2.32.5 - # via - # -c requirements.txt - # basalt-sdk (pyproject.toml) - # google-api-core - # google-cloud-bigquery - # google-cloud-storage - # google-genai - # id - # opentelemetry-exporter-otlp-proto-http - # requests-toolbelt - # twine -requests-toolbelt==1.0.0 - # via twine -rfc3986==2.0.0 - # via twine -rich==14.2.0 - # via twine -rsa==4.9.1 - # via google-auth -ruff==0.14.5 - # via basalt-sdk (pyproject.toml) -secretstorage==3.4.0 - # via keyring -setuptools==80.9.0 - # via basalt-sdk (pyproject.toml) -shapely==2.1.2 - # via google-cloud-aiplatform -six==1.17.0 - # via python-dateutil -sniffio==1.3.1 - # via - # -c requirements.txt - # anthropic - # anyio - # openai -tenacity==9.1.2 - # via google-genai -tqdm==4.67.1 - # via openai -twine==6.2.0 - # via basalt-sdk (pyproject.toml) -typing-extensions==4.15.0 - # via - # -c requirements.txt - # anthropic - # google-cloud-aiplatform - # google-genai - # grpcio - # openai - # opentelemetry-api - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-sdk - # opentelemetry-semantic-conventions - # pydantic - # pydantic-core - # typing-inspection -typing-inspection==0.4.2 - # via pydantic -urllib3==2.5.0 - # via - # -c requirements.txt - # requests - # twine -websockets==15.0.1 - # via google-genai -wheel==0.45.1 - # via basalt-sdk (pyproject.toml) -wrapt==1.17.3 - # via - # -c requirements.txt - # basalt-sdk (pyproject.toml) - # opentelemetry-instrumentation - # opentelemetry-instrumentation-httpx -zipp==3.23.0 - # via - # -c requirements.txt - # importlib-metadata diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index cc32e40..0000000 --- a/requirements.txt +++ /dev/null @@ -1,104 +0,0 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile pyproject.toml -o requirements.txt -anyio==4.11.0 - # via httpx -certifi==2025.10.5 - # via - # httpcore - # httpx - # requests -charset-normalizer==3.4.4 - # via requests -googleapis-common-protos==1.71.0 - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -grpcio==1.76.0 - # via opentelemetry-exporter-otlp-proto-grpc -h11==0.16.0 - # via httpcore -httpcore==1.0.9 - # via httpx -httpx==0.28.1 - # via basalt-sdk (pyproject.toml) -idna==3.11 - # via - # anyio - # httpx - # requests -importlib-metadata==8.7.0 - # via opentelemetry-api -jinja2==3.1.6 - # via basalt-sdk (pyproject.toml) -markupsafe==3.0.3 - # via jinja2 -opentelemetry-api==1.38.0 - # via - # basalt-sdk (pyproject.toml) - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-instrumentation - # opentelemetry-instrumentation-httpx - # opentelemetry-sdk - # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.38.0 - # via basalt-sdk (pyproject.toml) -opentelemetry-exporter-otlp-proto-common==1.38.0 - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.38.0 - # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.38.0 - # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.59b0 - # via - # basalt-sdk (pyproject.toml) - # opentelemetry-instrumentation-httpx -opentelemetry-instrumentation-httpx==0.59b0 - # via basalt-sdk (pyproject.toml) -opentelemetry-proto==1.38.0 - # via - # opentelemetry-exporter-otlp-proto-common - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.38.0 - # via - # basalt-sdk (pyproject.toml) - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.59b0 - # via - # basalt-sdk (pyproject.toml) - # opentelemetry-instrumentation - # opentelemetry-instrumentation-httpx - # opentelemetry-sdk -opentelemetry-util-http==0.59b0 - # via opentelemetry-instrumentation-httpx -packaging==25.0 - # via opentelemetry-instrumentation -protobuf==6.33.0 - # via - # googleapis-common-protos - # opentelemetry-proto -requests==2.32.5 - # via opentelemetry-exporter-otlp-proto-http -sniffio==1.3.1 - # via anyio -typing-extensions==4.15.0 - # via - # grpcio - # opentelemetry-api - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-sdk - # opentelemetry-semantic-conventions -urllib3==2.5.0 - # via requests -wrapt==1.17.3 - # via - # basalt-sdk (pyproject.toml) - # opentelemetry-instrumentation - # opentelemetry-instrumentation-httpx -zipp==3.23.0 - # via importlib-metadata diff --git a/tests/observability/test_instrumentation.py b/tests/observability/test_instrumentation.py index a600d5d..554725e 100644 --- a/tests/observability/test_instrumentation.py +++ b/tests/observability/test_instrumentation.py @@ -222,7 +222,9 @@ def test_install_processors_on_existing_provider(self, mock_trace): self.assertTrue(external_provider._basalt_processors_installed) # Verify that the manager has references to the processors - self.assertEqual(len(manager._span_processors), 3) + # 4 processors: BasaltContextProcessor, BasaltCallEvaluatorProcessor, + # BasaltShouldEvaluateProcessor, BasaltAutoInstrumentationProcessor + self.assertEqual(len(manager._span_processors), 4) # Verify that the manager stored the external provider self.assertIs(manager._tracer_provider, external_provider) diff --git a/tests/observability/test_multi_exporters.py b/tests/observability/test_multi_exporters.py index f1dc429..349aaf1 100644 --- a/tests/observability/test_multi_exporters.py +++ b/tests/observability/test_multi_exporters.py @@ -130,8 +130,9 @@ def test_user_exporters_plus_env_exporter(self, mock_otlp_exporter): # Verify both exporters were used provider = manager._tracer_provider self.assertIsInstance(provider, TracerProvider) - # Should have 2 exporters + 3 Basalt processors = 5 total processors - self.assertEqual(len(provider._active_span_processor._span_processors), 5) + # Should have 2 exporters + 4 Basalt processors = 6 total processors + # Basalt processors: Context, CallEvaluator, ShouldEvaluate, AutoInstrumentation + self.assertEqual(len(provider._active_span_processor._span_processors), 6) def test_mixed_console_and_otlp_exporters(self): """Test mix of ConsoleSpanExporter and regular exporters.""" diff --git a/tests/observability/test_should_evaluate_propagation.py b/tests/observability/test_should_evaluate_propagation.py new file mode 100644 index 0000000..2e3f6a8 --- /dev/null +++ b/tests/observability/test_should_evaluate_propagation.py @@ -0,0 +1,454 @@ +"""Tests for should_evaluate propagation across all spans in a trace.""" + +from collections.abc import Sequence + +import pytest +from opentelemetry import trace +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExporter, SpanExportResult + +from basalt.observability import observe, start_observe +from basalt.observability.context_managers import EvaluationConfig +from basalt.observability.decorators import ObserveKind +from basalt.observability.processors import BasaltShouldEvaluateProcessor +from basalt.observability.semconv import BasaltSpan + + +class InMemorySpanExporter(SpanExporter): + """Simple in-memory span exporter for testing.""" + + def __init__(self): + self.spans = [] + + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + self.spans.extend(spans) + return SpanExportResult.SUCCESS + + def shutdown(self): + pass + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return True + + def get_finished_spans(self): + return self.spans + + def clear(self): + self.spans = [] + + +@pytest.fixture(scope="function") +def setup_tracer(): + """Setup tracer with in-memory exporter for testing.""" + exporter = InMemorySpanExporter() + provider = trace.get_tracer_provider() + + # If provider is a ProxyTracerProvider, create a real one + if type(provider).__name__ == 'ProxyTracerProvider': + provider = TracerProvider() + provider.add_span_processor(BasaltShouldEvaluateProcessor()) + trace.set_tracer_provider(provider) + + # Ensure BasaltShouldEvaluateProcessor is installed + if not hasattr(provider, '_basalt_should_evaluate_installed'): + processor = BasaltShouldEvaluateProcessor() + provider.add_span_processor(processor) + provider._basalt_should_evaluate_installed = True # type: ignore[attr-defined] + + # Add the exporter processor + span_processor = SimpleSpanProcessor(exporter) + provider.add_span_processor(span_processor) + + yield exporter + + exporter.clear() + + +class TestShouldEvaluatePropagation: + """Test suite for should_evaluate propagation.""" + + def test_sample_rate_1_propagates_to_children(self, setup_tracer): + """Test that should_evaluate=True propagates to all child spans.""" + exporter = setup_tracer + + with start_observe( + name="parent", + feature_slug="test", + evaluate_config=EvaluationConfig(sample_rate=1.0) + ): + with observe(name="child1", kind=ObserveKind.FUNCTION): + with observe(name="grandchild", kind=ObserveKind.FUNCTION): + pass + + with observe(name="child2", kind=ObserveKind.GENERATION): + pass + + spans = exporter.get_finished_spans() + assert len(spans) == 4, f"Expected 4 spans, got {len(spans)}" + + # All spans should have should_evaluate=True + for span in spans: + assert BasaltSpan.SHOULD_EVALUATE in span.attributes, \ + f"Span {span.name} missing should_evaluate" + assert span.attributes[BasaltSpan.SHOULD_EVALUATE] is True, \ + f"Span {span.name} has should_evaluate={span.attributes[BasaltSpan.SHOULD_EVALUATE]}, expected True" + + def test_sample_rate_0_propagates_to_children(self, setup_tracer): + """Test that should_evaluate=False propagates to all child spans.""" + exporter = setup_tracer + + with start_observe( + name="parent", + feature_slug="test", + evaluate_config=EvaluationConfig(sample_rate=0.0) + ): + with observe(name="child1", kind=ObserveKind.FUNCTION): + with observe(name="grandchild", kind=ObserveKind.FUNCTION): + pass + + with observe(name="child2", kind=ObserveKind.GENERATION): + pass + + spans = exporter.get_finished_spans() + assert len(spans) == 4, f"Expected 4 spans, got {len(spans)}" + + # All spans should have should_evaluate=False + for span in spans: + assert BasaltSpan.SHOULD_EVALUATE in span.attributes, \ + f"Span {span.name} missing should_evaluate" + assert span.attributes[BasaltSpan.SHOULD_EVALUATE] is False, \ + f"Span {span.name} has should_evaluate={span.attributes[BasaltSpan.SHOULD_EVALUATE]}, expected False" + + def test_experiment_forces_true_for_all_spans(self, setup_tracer): + """Test that experiment forces should_evaluate=True for all spans.""" + exporter = setup_tracer + + with start_observe( + name="parent", + feature_slug="test", + experiment="exp_123", + evaluate_config=EvaluationConfig(sample_rate=0.0) # Would normally be False + ): + with observe(name="child1", kind=ObserveKind.FUNCTION): + with observe(name="grandchild", kind=ObserveKind.FUNCTION): + pass + + with observe(name="child2", kind=ObserveKind.GENERATION): + pass + + spans = exporter.get_finished_spans() + assert len(spans) == 4, f"Expected 4 spans, got {len(spans)}" + + # All spans should have should_evaluate=True due to experiment + for span in spans: + assert BasaltSpan.SHOULD_EVALUATE in span.attributes, \ + f"Span {span.name} missing should_evaluate" + assert span.attributes[BasaltSpan.SHOULD_EVALUATE] is True, \ + f"Span {span.name} has should_evaluate={span.attributes[BasaltSpan.SHOULD_EVALUATE]}, expected True due to experiment" + + def test_experiment_overrides_sample_rate_for_all_spans(self, setup_tracer): + """Test that experiment overrides sample_rate=0.0 for entire trace.""" + exporter = setup_tracer + + with start_observe( + name="experiment_trace", + feature_slug="test", + experiment="exp_456" + # No evaluate_config, global default is 0.0 + ): + with observe(name="processing", kind=ObserveKind.FUNCTION): + pass + + with observe(name="llm_call", kind=ObserveKind.GENERATION): + with observe(name="retrieval", kind=ObserveKind.RETRIEVAL): + pass + + spans = exporter.get_finished_spans() + assert len(spans) == 4, f"Expected 4 spans, got {len(spans)}" + + # Verify all have should_evaluate=True + for span in spans: + assert span.attributes.get(BasaltSpan.SHOULD_EVALUATE) is True, \ + f"Span {span.name} should have should_evaluate=True with experiment" + + def test_deeply_nested_spans_propagate(self, setup_tracer): + """Test propagation through deeply nested span hierarchy.""" + exporter = setup_tracer + + with start_observe( + name="root", + feature_slug="test", + evaluate_config=EvaluationConfig(sample_rate=1.0) + ): + with observe(name="level1", kind=ObserveKind.FUNCTION): + with observe(name="level2", kind=ObserveKind.FUNCTION): + with observe(name="level3", kind=ObserveKind.FUNCTION): + with observe(name="level4", kind=ObserveKind.FUNCTION): + with observe(name="level5", kind=ObserveKind.FUNCTION): + pass + + spans = exporter.get_finished_spans() + assert len(spans) == 6, f"Expected 6 spans, got {len(spans)}" + + # All spans should have same should_evaluate value + should_evaluate_values = [ + span.attributes.get(BasaltSpan.SHOULD_EVALUATE) + for span in spans + ] + assert all(v is True for v in should_evaluate_values), \ + f"All spans should have should_evaluate=True, got: {should_evaluate_values}" + + def test_multiple_child_branches_propagate(self, setup_tracer): + """Test propagation across multiple child branches.""" + exporter = setup_tracer + + with start_observe( + name="root", + feature_slug="test", + evaluate_config=EvaluationConfig(sample_rate=0.0) + ): + # Branch 1 + with observe(name="branch1", kind=ObserveKind.FUNCTION): + with observe(name="branch1_child", kind=ObserveKind.FUNCTION): + pass + + # Branch 2 + with observe(name="branch2", kind=ObserveKind.GENERATION): + with observe(name="branch2_child", kind=ObserveKind.RETRIEVAL): + pass + + # Branch 3 + with observe(name="branch3", kind=ObserveKind.TOOL): + pass + + spans = exporter.get_finished_spans() + assert len(spans) == 6, f"Expected 6 spans, got {len(spans)}" + + # All should be False + for span in spans: + assert span.attributes.get(BasaltSpan.SHOULD_EVALUATE) is False, \ + f"Span {span.name} should have should_evaluate=False" + + def test_decorator_style_propagation(self, setup_tracer): + """Test propagation works with decorator-style usage.""" + exporter = setup_tracer + + @start_observe( + name="decorated_root", + feature_slug="test", + evaluate_config=EvaluationConfig(sample_rate=1.0) + ) + def root_function(): + with observe(name="child_in_decorator", kind=ObserveKind.FUNCTION): + nested_function() + + @observe(name="nested_decorated", kind=ObserveKind.FUNCTION) + def nested_function(): + pass + + root_function() + + spans = exporter.get_finished_spans() + assert len(spans) == 3, f"Expected 3 spans, got {len(spans)}" + + # All should have should_evaluate=True + for span in spans: + assert span.attributes.get(BasaltSpan.SHOULD_EVALUATE) is True, \ + f"Span {span.name} should have should_evaluate=True" + + def test_experiment_with_decorator_propagation(self, setup_tracer): + """Test experiment forces evaluation with decorator pattern.""" + exporter = setup_tracer + + @start_observe( + name="experiment_decorated", + feature_slug="test", + experiment="exp_789", + evaluate_config=EvaluationConfig(sample_rate=0.0) + ) + def experiment_function(): + with observe(name="child", kind=ObserveKind.FUNCTION): + pass + + experiment_function() + + spans = exporter.get_finished_spans() + assert len(spans) == 2, f"Expected 2 spans, got {len(spans)}" + + # Both should be True due to experiment + for span in spans: + assert span.attributes.get(BasaltSpan.SHOULD_EVALUATE) is True, \ + f"Span {span.name} should have should_evaluate=True with experiment" + + def test_mixed_span_kinds_propagation(self, setup_tracer): + """Test propagation across different span kinds.""" + exporter = setup_tracer + + with start_observe( + name="mixed_trace", + feature_slug="test", + evaluate_config=EvaluationConfig(sample_rate=1.0) + ): + with observe(name="generation", kind=ObserveKind.GENERATION): + pass + + with observe(name="retrieval", kind=ObserveKind.RETRIEVAL): + pass + + with observe(name="tool", kind=ObserveKind.TOOL): + pass + + with observe(name="function", kind=ObserveKind.FUNCTION): + pass + + with observe(name="event", kind=ObserveKind.EVENT): + pass + + spans = exporter.get_finished_spans() + assert len(spans) == 6, f"Expected 6 spans, got {len(spans)}" + + # All different kinds should have same should_evaluate + for span in spans: + assert span.attributes.get(BasaltSpan.SHOULD_EVALUATE) is True, \ + f"Span {span.name} (kind={span.attributes.get('basalt.span.kind')}) should have should_evaluate=True" + + def test_no_evaluate_config_uses_global_default(self, setup_tracer): + """Test that without evaluate_config, global default (0.0) is used for all spans.""" + exporter = setup_tracer + + with start_observe( + name="default_trace", + feature_slug="test" + # No evaluate_config, should use global default 0.0 + ): + with observe(name="child1", kind=ObserveKind.FUNCTION): + pass + + with observe(name="child2", kind=ObserveKind.GENERATION): + pass + + spans = exporter.get_finished_spans() + assert len(spans) == 3, f"Expected 3 spans, got {len(spans)}" + + # All should be False (global default is 0.0) + for span in spans: + assert span.attributes.get(BasaltSpan.SHOULD_EVALUATE) is False, \ + f"Span {span.name} should have should_evaluate=False with global default" + + def test_trace_consistency(self, setup_tracer): + """Test that all spans in a trace have the SAME should_evaluate value.""" + exporter = setup_tracer + + # Test with sample_rate=1.0 + with start_observe( + name="consistent_trace", + feature_slug="test", + evaluate_config=EvaluationConfig(sample_rate=1.0) + ): + with observe(name="child1", kind=ObserveKind.FUNCTION): + with observe(name="grandchild", kind=ObserveKind.GENERATION): + pass + + spans = exporter.get_finished_spans() + should_evaluate_values = [ + span.attributes.get(BasaltSpan.SHOULD_EVALUATE) + for span in spans + ] + + # All values should be identical + assert len(set(should_evaluate_values)) == 1, \ + f"All spans should have same should_evaluate value, got: {should_evaluate_values}" + assert should_evaluate_values[0] is True + + +class TestExperimentShouldEvaluate: + """Tests specific to experiment forcing evaluation.""" + + def test_experiment_string_forces_evaluation(self, setup_tracer): + """Test that string experiment ID forces evaluation.""" + exporter = setup_tracer + + with start_observe( + name="trace", + feature_slug="test", + experiment="exp_string", + evaluate_config=EvaluationConfig(sample_rate=0.0) + ): + with observe(name="child", kind=ObserveKind.FUNCTION): + pass + + spans = exporter.get_finished_spans() + + for span in spans: + assert span.attributes.get(BasaltSpan.SHOULD_EVALUATE) is True + # Verify experiment ID is attached to root span + if span.name == "trace": + assert "basalt.experiment.id" in span.attributes + assert span.attributes["basalt.experiment.id"] == "exp_string" + + def test_experiment_object_forces_evaluation(self, setup_tracer): + """Test that experiment object forces evaluation.""" + exporter = setup_tracer + + class MockExperiment: + def __init__(self, id, name=None): + self.id = id + self.name = name + + exp = MockExperiment(id="exp_obj", name="Test Experiment") + + with start_observe( + name="trace", + feature_slug="test", + experiment=exp, + evaluate_config=EvaluationConfig(sample_rate=0.0) + ): + with observe(name="child", kind=ObserveKind.FUNCTION): + pass + + spans = exporter.get_finished_spans() + + for span in spans: + assert span.attributes.get(BasaltSpan.SHOULD_EVALUATE) is True + + def test_experiment_without_evaluate_config(self, setup_tracer): + """Test experiment works without explicit evaluate_config.""" + exporter = setup_tracer + + with start_observe( + name="trace", + feature_slug="test", + experiment="exp_no_config" + # No evaluate_config at all + ): + with observe(name="child", kind=ObserveKind.FUNCTION): + pass + + spans = exporter.get_finished_spans() + + # Should all be True due to experiment + for span in spans: + assert span.attributes.get(BasaltSpan.SHOULD_EVALUATE) is True + + def test_no_experiment_respects_sample_rate_zero(self, setup_tracer): + """Test that without experiment, sample_rate=0.0 is respected.""" + exporter = setup_tracer + + with start_observe( + name="trace", + feature_slug="test", + evaluate_config=EvaluationConfig(sample_rate=0.0) + # No experiment + ): + with observe(name="child", kind=ObserveKind.FUNCTION): + pass + + spans = exporter.get_finished_spans() + + # Should all be False without experiment + for span in spans: + assert span.attributes.get(BasaltSpan.SHOULD_EVALUATE) is False + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/uv.lock b/uv.lock index f11dd46..4cb9019 100644 --- a/uv.lock +++ b/uv.lock @@ -71,7 +71,7 @@ wheels = [ [[package]] name = "basalt-sdk" -version = "1.0.0rc1" +version = "1.0.1" source = { editable = "." } dependencies = [ { name = "httpx" }, @@ -116,6 +116,7 @@ dev = [ { name = "mistralai" }, { name = "openai" }, { name = "opentelemetry-instrumentation-anthropic" }, + { name = "opentelemetry-instrumentation-google-genai" }, { name = "opentelemetry-instrumentation-google-generativeai" }, { name = "opentelemetry-instrumentation-openai" }, { name = "opentelemetry-instrumentation-vertexai" }, @@ -133,6 +134,9 @@ framework-all = [ { name = "opentelemetry-instrumentation-langchain" }, { name = "opentelemetry-instrumentation-llamaindex" }, ] +google-genai = [ + { name = "opentelemetry-instrumentation-google-genai" }, +] google-generativeai = [ { name = "opentelemetry-instrumentation-google-generativeai" }, ] @@ -192,6 +196,8 @@ requires-dist = [ { name = "opentelemetry-instrumentation-anthropic", marker = "extra == 'dev'", specifier = ">=0.48.1" }, { name = "opentelemetry-instrumentation-bedrock", marker = "extra == 'bedrock'", specifier = "~=0.48.0" }, { name = "opentelemetry-instrumentation-chromadb", marker = "extra == 'chromadb'", specifier = "~=0.48.0" }, + { name = "opentelemetry-instrumentation-google-genai", marker = "extra == 'dev'" }, + { name = "opentelemetry-instrumentation-google-genai", marker = "extra == 'google-genai'", specifier = "~=0.5b0" }, { name = "opentelemetry-instrumentation-google-generativeai", marker = "extra == 'dev'", specifier = ">=0.48.1" }, { name = "opentelemetry-instrumentation-google-generativeai", marker = "extra == 'google-generativeai'", specifier = "~=0.48.0" }, { name = "opentelemetry-instrumentation-httpx", specifier = "~=0.59b0" }, @@ -217,7 +223,7 @@ requires-dist = [ { name = "wheel", marker = "extra == 'dev'" }, { name = "wrapt", specifier = "~=1.17.3" }, ] -provides-extras = ["openai", "anthropic", "google-generativeai", "bedrock", "vertex-ai", "mistralai", "chromadb", "pinecone", "qdrant", "langchain", "llamaindex", "llm-all", "vector-all", "framework-all", "all", "dev"] +provides-extras = ["openai", "anthropic", "google-generativeai", "google-genai", "bedrock", "vertex-ai", "mistralai", "chromadb", "pinecone", "qdrant", "langchain", "llamaindex", "llm-all", "vector-all", "framework-all", "all", "dev"] [[package]] name = "cachetools" @@ -248,43 +254,31 @@ sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8 wheels = [ { url = "https://files.pythonhosted.org/packages/50/bd/b1a6362b80628111e6653c961f987faa55262b4002fcec42308cad1db680/cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", size = 208811, upload-time = "2025-09-08T23:22:12.267Z" }, { url = "https://files.pythonhosted.org/packages/4f/27/6933a8b2562d7bd1fb595074cf99cc81fc3789f6a6c05cdabb46284a3188/cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", size = 216402, upload-time = "2025-09-08T23:22:13.455Z" }, - { url = "https://files.pythonhosted.org/packages/05/eb/b86f2a2645b62adcfff53b0dd97e8dfafb5c8aa864bd0d9a2c2049a0d551/cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", size = 203217, upload-time = "2025-09-08T23:22:14.596Z" }, - { url = "https://files.pythonhosted.org/packages/9f/e0/6cbe77a53acf5acc7c08cc186c9928864bd7c005f9efd0d126884858a5fe/cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", size = 203079, upload-time = "2025-09-08T23:22:15.769Z" }, { url = "https://files.pythonhosted.org/packages/98/29/9b366e70e243eb3d14a5cb488dfd3a0b6b2f1fb001a203f653b93ccfac88/cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", size = 216475, upload-time = "2025-09-08T23:22:17.427Z" }, { url = "https://files.pythonhosted.org/packages/21/7a/13b24e70d2f90a322f2900c5d8e1f14fa7e2a6b3332b7309ba7b2ba51a5a/cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", size = 218829, upload-time = "2025-09-08T23:22:19.069Z" }, { url = "https://files.pythonhosted.org/packages/60/99/c9dc110974c59cc981b1f5b66e1d8af8af764e00f0293266824d9c4254bc/cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", size = 211211, upload-time = "2025-09-08T23:22:20.588Z" }, { url = "https://files.pythonhosted.org/packages/49/72/ff2d12dbf21aca1b32a40ed792ee6b40f6dc3a9cf1644bd7ef6e95e0ac5e/cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", size = 218036, upload-time = "2025-09-08T23:22:22.143Z" }, { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" }, - { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" }, - { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" }, { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" }, { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" }, { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" }, { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" }, { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, - { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, - { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, - { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, - { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, - { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, - { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, - { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, - { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, @@ -518,10 +512,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/49/498c86566a1d80e978b42f0d702795f69887005548c041636df6ae1ca64c/cryptography-46.0.3-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:01ca9ff2885f3acc98c29f1860552e37f6d7c7d013d7334ff2a9de43a449315d", size = 4450807, upload-time = "2025-10-15T23:16:56.414Z" }, { url = "https://files.pythonhosted.org/packages/4b/0a/863a3604112174c8624a2ac3c038662d9e59970c7f926acdcfaed8d61142/cryptography-46.0.3-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6eae65d4c3d33da080cff9c4ab1f711b15c1d9760809dad6ea763f3812d254cb", size = 4299615, upload-time = "2025-10-15T23:16:58.442Z" }, { url = "https://files.pythonhosted.org/packages/64/02/b73a533f6b64a69f3cd3872acb6ebc12aef924d8d103133bb3ea750dc703/cryptography-46.0.3-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5bf0ed4490068a2e72ac03d786693adeb909981cc596425d09032d372bcc849", size = 4016800, upload-time = "2025-10-15T23:17:00.378Z" }, - { url = "https://files.pythonhosted.org/packages/25/d5/16e41afbfa450cde85a3b7ec599bebefaef16b5c6ba4ec49a3532336ed72/cryptography-46.0.3-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5ecfccd2329e37e9b7112a888e76d9feca2347f12f37918facbb893d7bb88ee8", size = 4984707, upload-time = "2025-10-15T23:17:01.98Z" }, { url = "https://files.pythonhosted.org/packages/c9/56/e7e69b427c3878352c2fb9b450bd0e19ed552753491d39d7d0a2f5226d41/cryptography-46.0.3-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a2c0cd47381a3229c403062f764160d57d4d175e022c1df84e168c6251a22eec", size = 4482541, upload-time = "2025-10-15T23:17:04.078Z" }, { url = "https://files.pythonhosted.org/packages/78/f6/50736d40d97e8483172f1bb6e698895b92a223dba513b0ca6f06b2365339/cryptography-46.0.3-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:549e234ff32571b1f4076ac269fcce7a808d3bf98b76c8dd560e42dbc66d7d91", size = 4299464, upload-time = "2025-10-15T23:17:05.483Z" }, - { url = "https://files.pythonhosted.org/packages/00/de/d8e26b1a855f19d9994a19c702fa2e93b0456beccbcfe437eda00e0701f2/cryptography-46.0.3-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:c0a7bb1a68a5d3471880e264621346c48665b3bf1c3759d682fc0864c540bd9e", size = 4950838, upload-time = "2025-10-15T23:17:07.425Z" }, { url = "https://files.pythonhosted.org/packages/8f/29/798fc4ec461a1c9e9f735f2fc58741b0daae30688f41b2497dcbc9ed1355/cryptography-46.0.3-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:10b01676fc208c3e6feeb25a8b83d81767e8059e1fe86e1dc62d10a3018fa926", size = 4481596, upload-time = "2025-10-15T23:17:09.343Z" }, { url = "https://files.pythonhosted.org/packages/15/8d/03cd48b20a573adfff7652b76271078e3045b9f49387920e7f1f631d125e/cryptography-46.0.3-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0abf1ffd6e57c67e92af68330d05760b7b7efb243aab8377e583284dbab72c71", size = 4426782, upload-time = "2025-10-15T23:17:11.22Z" }, { url = "https://files.pythonhosted.org/packages/fa/b1/ebacbfe53317d55cf33165bda24c86523497a6881f339f9aae5c2e13e57b/cryptography-46.0.3-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a04bee9ab6a4da801eb9b51f1b708a1b5b5c9eb48c03f74198464c66f0d344ac", size = 4698381, upload-time = "2025-10-15T23:17:12.829Z" }, @@ -529,10 +521,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c5/fd/bc1daf8230eaa075184cbbf5f8cd00ba9db4fd32d63fb83da4671b72ed8a/cryptography-46.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:39b6755623145ad5eff1dab323f4eae2a32a77a7abef2c5089a04a3d04366715", size = 4435078, upload-time = "2025-10-15T23:17:23.042Z" }, { url = "https://files.pythonhosted.org/packages/82/98/d3bd5407ce4c60017f8ff9e63ffee4200ab3e23fe05b765cab805a7db008/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:db391fa7c66df6762ee3f00c95a89e6d428f4d60e7abc8328f4fe155b5ac6e54", size = 4293460, upload-time = "2025-10-15T23:17:24.885Z" }, { url = "https://files.pythonhosted.org/packages/26/e9/e23e7900983c2b8af7a08098db406cf989d7f09caea7897e347598d4cd5b/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:78a97cf6a8839a48c49271cdcbd5cf37ca2c1d6b7fdd86cc864f302b5e9bf459", size = 3995237, upload-time = "2025-10-15T23:17:26.449Z" }, - { url = "https://files.pythonhosted.org/packages/91/15/af68c509d4a138cfe299d0d7ddb14afba15233223ebd933b4bbdbc7155d3/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:dfb781ff7eaa91a6f7fd41776ec37c5853c795d3b358d4896fdbb5df168af422", size = 4967344, upload-time = "2025-10-15T23:17:28.06Z" }, { url = "https://files.pythonhosted.org/packages/ca/e3/8643d077c53868b681af077edf6b3cb58288b5423610f21c62aadcbe99f4/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6f61efb26e76c45c4a227835ddeae96d83624fb0d29eb5df5b96e14ed1a0afb7", size = 4466564, upload-time = "2025-10-15T23:17:29.665Z" }, { url = "https://files.pythonhosted.org/packages/0e/43/c1e8726fa59c236ff477ff2b5dc071e54b21e5a1e51aa2cee1676f1c986f/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:23b1a8f26e43f47ceb6d6a43115f33a5a37d57df4ea0ca295b780ae8546e8044", size = 4292415, upload-time = "2025-10-15T23:17:31.686Z" }, - { url = "https://files.pythonhosted.org/packages/42/f9/2f8fefdb1aee8a8e3256a0568cffc4e6d517b256a2fe97a029b3f1b9fe7e/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b419ae593c86b87014b9be7396b385491ad7f320bde96826d0dd174459e54665", size = 4931457, upload-time = "2025-10-15T23:17:33.478Z" }, { url = "https://files.pythonhosted.org/packages/79/30/9b54127a9a778ccd6d27c3da7563e9f2d341826075ceab89ae3b41bf5be2/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:50fc3343ac490c6b08c0cf0d704e881d0d660be923fd3076db3e932007e726e3", size = 4466074, upload-time = "2025-10-15T23:17:35.158Z" }, { url = "https://files.pythonhosted.org/packages/ac/68/b4f4a10928e26c941b1b6a179143af9f4d27d88fe84a6a3c53592d2e76bf/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:22d7e97932f511d6b0b04f2bfd818d73dcd5928db509460aaf48384778eb6d20", size = 4420569, upload-time = "2025-10-15T23:17:37.188Z" }, { url = "https://files.pythonhosted.org/packages/a3/49/3746dab4c0d1979888f125226357d3262a6dd40e114ac29e3d2abdf1ec55/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d55f3dffadd674514ad19451161118fd010988540cee43d8bc20675e775925de", size = 4681941, upload-time = "2025-10-15T23:17:39.236Z" }, @@ -540,10 +530,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/26/42/fa8389d4478368743e24e61eea78846a0006caffaf72ea24a15159215a14/cryptography-46.0.3-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:15ab9b093e8f09daab0f2159bb7e47532596075139dd74365da52ecc9cb46c5d", size = 4440029, upload-time = "2025-10-15T23:17:49.837Z" }, { url = "https://files.pythonhosted.org/packages/5f/eb/f483db0ec5ac040824f269e93dd2bd8a21ecd1027e77ad7bdf6914f2fd80/cryptography-46.0.3-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:46acf53b40ea38f9c6c229599a4a13f0d46a6c3fa9ef19fc1a124d62e338dfa0", size = 4297222, upload-time = "2025-10-15T23:17:51.357Z" }, { url = "https://files.pythonhosted.org/packages/fd/cf/da9502c4e1912cb1da3807ea3618a6829bee8207456fbbeebc361ec38ba3/cryptography-46.0.3-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10ca84c4668d066a9878890047f03546f3ae0a6b8b39b697457b7757aaf18dbc", size = 4012280, upload-time = "2025-10-15T23:17:52.964Z" }, - { url = "https://files.pythonhosted.org/packages/6b/8f/9adb86b93330e0df8b3dcf03eae67c33ba89958fc2e03862ef1ac2b42465/cryptography-46.0.3-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:36e627112085bb3b81b19fed209c05ce2a52ee8b15d161b7c643a7d5a88491f3", size = 4978958, upload-time = "2025-10-15T23:17:54.965Z" }, { url = "https://files.pythonhosted.org/packages/d1/a0/5fa77988289c34bdb9f913f5606ecc9ada1adb5ae870bd0d1054a7021cc4/cryptography-46.0.3-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1000713389b75c449a6e979ffc7dcc8ac90b437048766cef052d4d30b8220971", size = 4473714, upload-time = "2025-10-15T23:17:56.754Z" }, { url = "https://files.pythonhosted.org/packages/14/e5/fc82d72a58d41c393697aa18c9abe5ae1214ff6f2a5c18ac470f92777895/cryptography-46.0.3-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:b02cf04496f6576afffef5ddd04a0cb7d49cf6be16a9059d793a30b035f6b6ac", size = 4296970, upload-time = "2025-10-15T23:17:58.588Z" }, - { url = "https://files.pythonhosted.org/packages/78/06/5663ed35438d0b09056973994f1aec467492b33bd31da36e468b01ec1097/cryptography-46.0.3-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:71e842ec9bc7abf543b47cf86b9a743baa95f4677d22baa4c7d5c69e49e9bc04", size = 4940236, upload-time = "2025-10-15T23:18:00.897Z" }, { url = "https://files.pythonhosted.org/packages/fc/59/873633f3f2dcd8a053b8dd1d38f783043b5fce589c0f6988bf55ef57e43e/cryptography-46.0.3-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:402b58fc32614f00980b66d6e56a5b4118e6cb362ae8f3fda141ba4689bd4506", size = 4472642, upload-time = "2025-10-15T23:18:02.749Z" }, { url = "https://files.pythonhosted.org/packages/3d/39/8e71f3930e40f6877737d6f69248cf74d4e34b886a3967d32f919cc50d3b/cryptography-46.0.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef639cb3372f69ec44915fafcd6698b6cc78fbe0c2ea41be867f6ed612811963", size = 4423126, upload-time = "2025-10-15T23:18:04.85Z" }, { url = "https://files.pythonhosted.org/packages/cd/c7/f65027c2810e14c3e7268353b1681932b87e5a48e65505d8cc17c99e36ae/cryptography-46.0.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b51b8ca4f1c6453d8829e1eb7299499ca7f313900dd4d89a24b8b87c0a780d4", size = 4686573, upload-time = "2025-10-15T23:18:06.908Z" }, @@ -1707,6 +1695,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/fc/dc5de37c42441ca617b7d95dd52502caaabcbf6bb246244395b2fb2d5ed8/opentelemetry_instrumentation_chromadb-0.48.0-py3-none-any.whl", hash = "sha256:c93a4d918d23bf33808c76880db8ba90ff88a36f63fdc368fc847a3e7f03aa2e", size = 6302, upload-time = "2025-11-11T09:43:48.252Z" }, ] +[[package]] +name = "opentelemetry-instrumentation-google-genai" +version = "0.5b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-genai" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/62/b2506d74f50d4d0f150293d171dc1196c9334ba02c51d6ed19d64d0f76c4/opentelemetry_instrumentation_google_genai-0.5b0.tar.gz", hash = "sha256:1986cd1a69dafdcccee15ae9f114e45ff04954951af0fef8b5482e2930fc0b17", size = 47840, upload-time = "2025-12-11T14:50:48.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/9f/a55591e2b41f6c29c4cf4b459617b03318e6d1e9c06f3b3ce7f22b7da8fc/opentelemetry_instrumentation_google_genai-0.5b0-py3-none-any.whl", hash = "sha256:20467a96d7407affc975e63d1175c21a4d33dd83f5ec162dddde6cea9e8f3995", size = 29531, upload-time = "2025-12-11T14:50:47.323Z" }, +] + [[package]] name = "opentelemetry-instrumentation-google-generativeai" version = "0.48.1" @@ -1892,6 +1895,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/b5/cf25da2218910f0d6cdf7f876a06bed118c4969eacaf60a887cbaef44f44/opentelemetry_semantic_conventions_ai-0.4.13-py3-none-any.whl", hash = "sha256:883a30a6bb5deaec0d646912b5f9f6dcbb9f6f72557b73d0f2560bf25d13e2d5", size = 6080, upload-time = "2025-08-22T10:14:16.477Z" }, ] +[[package]] +name = "opentelemetry-util-genai" +version = "0.2b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/06/869c18f22fa32d6db9ebbc5bd82f18614c379b726aa9770d1b2d20f9178c/opentelemetry_util_genai-0.2b0.tar.gz", hash = "sha256:803d5d5e720f3e057c64d935dfd46dc013784820715d996980cb5b79bb5774a3", size = 20542, upload-time = "2025-10-15T20:07:31.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/0d/1dc4705e44540183ab603aa4e1b0e3f888e63156678192e8c64f42c782ed/opentelemetry_util_genai-0.2b0-py3-none-any.whl", hash = "sha256:06dc8664713f9cec216d7093585a1a5fbb5f6fb7a387f19430774a5028aaa30b", size = 22237, upload-time = "2025-10-15T20:07:30.991Z" }, +] + [[package]] name = "opentelemetry-util-http" version = "0.59b0"