diff --git a/.gitignore b/.gitignore index ce9aa74..378acfc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ basalt*.egg-info/ build/ dist/ __pycache__/ -test.py \ No newline at end of file +test.py +.DS_Store \ No newline at end of file diff --git a/basalt/_version.py b/basalt/_version.py index 485f44a..d3ec452 100644 --- a/basalt/_version.py +++ b/basalt/_version.py @@ -1 +1 @@ -__version__ = "0.1.1" +__version__ = "0.2.0" diff --git a/basalt/basalt_facade.py b/basalt/basalt_facade.py index 4964b2c..9c0e93f 100644 --- a/basalt/basalt_facade.py +++ b/basalt/basalt_facade.py @@ -1,5 +1,5 @@ from .utils.api import Api -from .utils.protocols import IPromptSDK, IBasaltSDK, IMonitorSDK +from .utils.protocols import IPromptSDK, IBasaltSDK, LogLevel from .sdk.promptsdk import PromptSDK from .sdk.monitorsdk import MonitorSDK from .basaltsdk import BasaltSDK @@ -8,6 +8,8 @@ from .config import config from .utils.logger import Logger +from .ressources.monitor.monitorsdk_types import IMonitorSDK + global_fallback_cache = MemoryCache() class BasaltFacade(IBasaltSDK): @@ -15,7 +17,7 @@ class BasaltFacade(IBasaltSDK): The Basalt client. """ - def __init__(self, api_key: str, log_level: str = 'all'): + def __init__(self, api_key: str, log_level: LogLevel = 'all'): """ Initializes the Basalt client with the given API key and log level. @@ -25,8 +27,8 @@ def __init__(self, api_key: str, log_level: str = 'all'): """ cache = MemoryCache() logger = Logger(log_level=log_level) - networker = Networker(logger=logger) - + networker = Networker() + api = Api( networker=networker, root_url=config["api_url"], diff --git a/basalt/basaltsdk.py b/basalt/basaltsdk.py index dedabf4..b3ef744 100644 --- a/basalt/basaltsdk.py +++ b/basalt/basaltsdk.py @@ -1,4 +1,5 @@ -from .utils.protocols import IPromptSDK, IBasaltSDK, IMonitorSDK +from .utils.protocols import IPromptSDK, IBasaltSDK +from .ressources.monitor.monitorsdk_types import IMonitorSDK class BasaltSDK(IBasaltSDK): """ @@ -9,7 +10,7 @@ class BasaltSDK(IBasaltSDK): def __init__(self, prompt_sdk: IPromptSDK, monitor_sdk: IMonitorSDK): self._prompt = prompt_sdk self._monitor = monitor_sdk - + @property def prompt(self) -> IPromptSDK: """Read-only access to the PromptSDK instance""" diff --git a/basalt/endpoints/list_prompts.py b/basalt/endpoints/list_prompts.py index 78fff06..0523456 100644 --- a/basalt/endpoints/list_prompts.py +++ b/basalt/endpoints/list_prompts.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Any, Dict, Optional, Tuple, List -from ..utils.dtos import PromptListResponse +from ..utils.dtos import PromptListResponse, PromptListDTO @dataclass class ListPromptsEndpointResponse: @@ -29,7 +29,7 @@ class ListPromptsEndpoint: Endpoint class for fetching a prompt. """ @staticmethod - def prepare_request() -> Dict[str, Any]: + def prepare_request(dto: PromptListDTO) -> Dict[str, Any]: """ Prepare the request dictionary for the ListPrompts endpoint. @@ -38,7 +38,10 @@ def prepare_request() -> Dict[str, Any]: """ return { "path": "/prompts", - "method": "GET" + "method": "GET", + "query": { + "featureSlug": dto.featureSlug + } } @staticmethod diff --git a/basalt/endpoints/monitor/create_experiment.py b/basalt/endpoints/monitor/create_experiment.py new file mode 100644 index 0000000..0f6048f --- /dev/null +++ b/basalt/endpoints/monitor/create_experiment.py @@ -0,0 +1,55 @@ +from dataclasses import dataclass +from typing import Any, Dict, Optional, Tuple +from datetime import datetime + +# Minimal Experiment model (expand as needed) +@dataclass +class Experiment: + feature_slug: str + name: str + id: str + created_at: datetime + # Add more fields as needed + + @classmethod + def from_dict(cls, data: Dict[str, Any]): + return cls( + feature_slug=data.get("featureSlug") or data.get("feature_slug"), + name=data.get("name"), + id=data.get("id"), + created_at=data.get("createdAt"), + ) + +@dataclass +class CreateExperimentDTO: + feature_slug: str + name: str + +@dataclass +class Output: + experiment: Experiment + +class CreateExperimentEndpoint: + """ + Endpoint for creating an experiment + """ + @staticmethod + def prepare_request(dto: CreateExperimentDTO) -> Dict[str, Any]: + body = { + "featureSlug": dto.feature_slug, + "name": dto.name, + } + + return { + "method": "post", + "path": "/monitor/experiments", + "body": body, + } + + @staticmethod + def decode_response(body: Any) -> Tuple[Optional[Exception], Optional[Output]]: + if not isinstance(body, dict): + return Exception("Failed to decode response (invalid body format)"), None + + experiment = Experiment.from_dict(body) + return None, Output(experiment=experiment) diff --git a/basalt/endpoints/monitor/send_trace.py b/basalt/endpoints/monitor/send_trace.py index 528b960..456831d 100644 --- a/basalt/endpoints/monitor/send_trace.py +++ b/basalt/endpoints/monitor/send_trace.py @@ -15,10 +15,10 @@ class SendTraceEndpoint: def prepare_request(self, dto: Optional[Input] = None) -> Dict[str, Any]: """ Prepares the request for sending a trace. - + Args: dto (Optional[Dict[str, Any]]): The data transfer object containing the trace. - + Returns: Dict[str, Any]: The request information. """ @@ -28,57 +28,40 @@ def prepare_request(self, dto: Optional[Input] = None) -> Dict[str, Any]: "path": "/monitor/trace", "body": {} } - + trace = dto["trace"] - + # Check if trace is already a dictionary or an object if isinstance(trace, dict): trace_data = trace - logs = trace_data.get("logs", []) + logs = trace.get("logs", []) else: trace_data = trace.to_dict() # Convert logs to a format suitable for the API logs = [] for log in trace_data["logs"]: - log_data = log.to_dict() - - # Convert dates to ISO format - log_data["startTime"] = log_data["start_time"].isoformat() if isinstance(log_data["start_time"], datetime) else log_data["start_time"] - log_data["endTime"] = log_data["end_time"].isoformat() if isinstance(log_data["end_time"], datetime) and log_data["end_time"] else None - - # Remove old format keys - del log_data["start_time"] - del log_data["end_time"] + dict_log = log.to_dict() + + # Convert dates and handle parent ID + log_data = { + "startTime": dict_log["start_time"].isoformat() if isinstance(dict_log["start_time"], datetime) else dict_log["start_time"], + "endTime": dict_log["end_time"].isoformat() if isinstance(dict_log["end_time"], datetime) and dict_log["end_time"] else None, + "parentId": dict_log["parent"]["id"] if dict_log["parent"] else None, + "inputTokens": dict_log["input_tokens"] if "input_tokens" in dict_log else None, + "outputTokens": dict_log["output_tokens"] if "output_tokens" in dict_log else None, + "cost": dict_log["cost"] if "cost" in dict_log else None, + "variables": [{"label": k, "value": v} for k, v in dict_log["variables"].items()] if "variables" in dict_log else None, + "input": dict_log["input"] if "input" in dict_log else None, + "output": dict_log["output"] if "output" in dict_log else None, + "prompt": dict_log["prompt"] if "prompt" in dict_log else None, + "evaluators": dict_log["evaluators"] if "evaluators" in dict_log else None + } - # Add input and output if they exist - if hasattr(log, "input"): - log_data["input"] = log.input - if hasattr(log, "output"): - log_data["output"] = log.output - - # Add prompt and variables if it's a generation - if hasattr(log, "prompt"): - log_data["prompt"] = log.prompt - if hasattr(log, "variables") and log.variables: - log_data["variables"] = [{"label": key, "value": value} for key, value in log.variables.items()] - if hasattr(log, "inputTokens"): - log_data["inputTokens"] = log.inputTokens - if hasattr(log, "outputTokens"): - log_data["outputTokens"] = log.outputTokens - if hasattr(log, "cost"): - log_data["cost"] = log.cost - - # Extract parent ID - if log_data["parent"]: - log_data["parentId"] = log_data["parent"]["id"] - del log_data["parent"] - else: - log_data["parentId"] = None - logs.append(log_data) - + # Process logs if they're already in dictionary format processed_logs = [] + for log_data in logs: # If log_data is already processed by the flusher, it will have these keys if "startTime" in log_data and "endTime" in log_data: @@ -96,19 +79,21 @@ def prepare_request(self, dto: Optional[Input] = None) -> Dict[str, Any]: if "end_time" in processed_log: processed_log["endTime"] = processed_log["end_time"].isoformat() if isinstance(processed_log["end_time"], datetime) and processed_log["end_time"] else None del processed_log["end_time"] - + # Extract parent ID if "parent" in processed_log and processed_log["parent"]: processed_log["parentId"] = processed_log["parent"]["id"] del processed_log["parent"] else: processed_log["parentId"] = None - + processed_logs.append(processed_log) - + # Create the request body body = { - "chainSlug": trace_data.get("chain_slug", trace_data.get("chainSlug")), + "featureSlug": trace_data.get("feature_slug", trace_data.get("featureSlug")), + "name": trace_data.get("name", trace_data.get("name")), + "experiment": {"id": trace_data.get("experiment", {}).id} if trace_data.get("experiment") else None, "input": trace_data.get("input"), "output": trace_data.get("output"), "metadata": trace_data.get("metadata"), @@ -116,15 +101,17 @@ def prepare_request(self, dto: Optional[Input] = None) -> Dict[str, Any]: "user": trace_data.get("user"), "startTime": trace_data.get("start_time", trace_data.get("startTime")), "endTime": trace_data.get("end_time", trace_data.get("endTime")), - "logs": processed_logs + "logs": processed_logs, + "evaluators": trace_data.get("evaluators"), + "evaluationConfig": trace_data.get("evaluationConfig") } - + # Convert dates to ISO format if they're datetime objects if isinstance(body["startTime"], datetime): body["startTime"] = body["startTime"].isoformat() if isinstance(body["endTime"], datetime): body["endTime"] = body["endTime"].isoformat() - + return { "method": "post", "path": "/monitor/trace", diff --git a/basalt/objects/base_log.py b/basalt/objects/base_log.py index 200a95b..a856512 100644 --- a/basalt/objects/base_log.py +++ b/basalt/objects/base_log.py @@ -1,16 +1,17 @@ from datetime import datetime -from typing import Dict, Optional, Any, TYPE_CHECKING +from typing import Dict, Optional, Any, List import uuid -if TYPE_CHECKING: - from .log import Log - from .trace import Trace +from ..ressources.monitor.base_log_types import BaseLogParams +from ..ressources.monitor.evaluator_types import Evaluator +from ..ressources.monitor.trace_types import Trace +from ..ressources.monitor.log_types import Log class BaseLog: """ Base class for logs and generations. """ - def __init__(self, params: Dict[str, Any]): + def __init__(self, params: BaseLogParams): self._id = f"log-{uuid.uuid4().hex[:8]}" self._type = params.get("type") self._name = params.get("name") @@ -19,6 +20,7 @@ def __init__(self, params: Dict[str, Any]): self._metadata = params.get("metadata") self._trace = params.get("trace") self._parent = params.get("parent") + self._evaluators = params.get("evaluators") # Add to trace's logs list if trace exists if self._trace: @@ -68,6 +70,11 @@ def metadata(self) -> Optional[Dict[str, Any]]: def trace(self) -> 'Trace': """Get the trace.""" return self._trace + + @property + def evaluators(self) -> List[Evaluator]: + """Get the evaluators.""" + return self._evaluators @trace.setter def trace(self, trace: 'Trace'): @@ -84,6 +91,13 @@ def set_metadata(self, metadata: Dict[str, Any]) -> 'BaseLog': self._metadata = metadata return self + def add_evaluator(self, evaluator: Evaluator) -> 'BaseLog': + if self._evaluators is None: + self._evaluators = [] + + self._evaluators.append(evaluator) + return self + def update(self, params: Dict[str, Any]) -> 'BaseLog': """Update the log.""" self._name = params.get("name", self._name) diff --git a/basalt/objects/experiment.py b/basalt/objects/experiment.py new file mode 100644 index 0000000..55c5e85 --- /dev/null +++ b/basalt/objects/experiment.py @@ -0,0 +1,22 @@ +from datetime import datetime +from ..ressources.monitor.experiment_types import Experiment as IExperiment + +class Experiment: + def __init__(self, experiment: IExperiment): + self._experiment = experiment + + @property + def id(self) -> str: + return self._experiment.id + + @property + def name(self) -> str: + return self._experiment.name + + @property + def feature_slug(self) -> str: + return self._experiment.feature_slug + + @property + def created_at(self) -> datetime: + return self._experiment.created_at diff --git a/basalt/objects/generation.py b/basalt/objects/generation.py index bbe887e..545718f 100644 --- a/basalt/objects/generation.py +++ b/basalt/objects/generation.py @@ -1,12 +1,13 @@ from typing import Dict, Optional, Any, List, Union from .base_log import BaseLog +from ..ressources.monitor.generation_types import GenerationParams class Generation(BaseLog): """ Class representing a generation in the monitoring system. """ - def __init__(self, params: Dict[str, Any]): + def __init__(self, params: GenerationParams): params_with_type = { "type": "generation", **params @@ -16,8 +17,8 @@ def __init__(self, params: Dict[str, Any]): self._prompt = params.get("prompt") self._input = params.get("input") self._output = params.get("output") - self._inputTokens = params.get("inputTokens") - self._outputTokens = params.get("outputTokens") + self._input_tokens = params.get("input_tokens") + self._output_tokens = params.get("output_tokens") self._cost = params.get("cost") # Convert variables to array format if needed @@ -50,14 +51,14 @@ def output(self) -> Optional[str]: return self._output @property - def inputTokens(self) -> Optional[int]: + def input_tokens(self) -> Optional[int]: """Get the generation input tokens.""" - return self._inputTokens + return self._input_tokens @property - def outputTokens(self) -> Optional[int]: + def output_tokens(self) -> Optional[int]: """Get the generation output tokens.""" - return self._outputTokens + return self._output_tokens @property def cost(self) -> Optional[float]: @@ -132,8 +133,8 @@ def update(self, params: Dict[str, Any]) -> 'Generation': self._input = params.get("input", self._input) self._output = params.get("output", self._output) self._prompt = params.get("prompt", self._prompt) - self._inputTokens = params.get("inputTokens", self._inputTokens) - self._outputTokens = params.get("outputTokens", self._outputTokens) + self._input_tokens = params.get("input_tokens", self._input_tokens) + self._output_tokens = params.get("output_tokens", self._output_tokens) self._cost = params.get("cost", self._cost) # Update variables if provided diff --git a/basalt/objects/log.py b/basalt/objects/log.py index 0d38db8..5102cea 100644 --- a/basalt/objects/log.py +++ b/basalt/objects/log.py @@ -1,15 +1,13 @@ -from typing import Dict, Optional, Any, TYPE_CHECKING - +from typing import Dict, Optional, Any +from ..ressources.monitor.log_types import LogParams from .base_log import BaseLog - -if TYPE_CHECKING: - from .generation import Generation +from .generation import Generation class Log(BaseLog): """ Class representing a log in the monitoring system. """ - def __init__(self, params: Dict[str, Any]): + def __init__(self, params: LogParams): super().__init__(params) self._input = params.get("input") self._output = None @@ -67,8 +65,18 @@ def append(self, generation: 'Generation') -> 'Log': Returns: Log: The log instance. """ - generation.parent = self + # Remove child log from the list of its previous trace + generation.trace.logs = [log for log in generation.trace.logs if log.id != generation.id] + + # Add child to the new trace list + self.trace.logs.append(generation) + + # Set the trace of the generation to the current log generation.trace = self.trace + generation.options = {"type": "multi"} + + # Set the parent of the generation to the current log + generation.parent = self return self diff --git a/basalt/objects/trace.py b/basalt/objects/trace.py index d9d32b3..76b52f0 100644 --- a/basalt/objects/trace.py +++ b/basalt/objects/trace.py @@ -1,18 +1,21 @@ from datetime import datetime -from typing import Dict, Optional, Any, List, TYPE_CHECKING +from typing import Dict, Optional, Any, List -if TYPE_CHECKING: - from .base_log import BaseLog - from .generation import Generation - from ..utils.flusher import Flusher + +from ..ressources.monitor.trace_types import TraceParams +from .base_log import BaseLog +from .generation import Generation +from ..utils.flusher import Flusher +from .experiment import Experiment +from ..utils.logger import Logger class Trace: """ Class representing a trace in the monitoring system. """ - def __init__(self, slug: str, params: Dict[str, Any], flusher: 'Flusher'): - self._chain_slug = slug - + def __init__(self, feature_slug: str, params: TraceParams, flusher: 'Flusher', logger: 'Logger'): + self._feature_slug = feature_slug + self._input = params.get("input") self._output = params.get("output") self._name = params.get("name") @@ -21,11 +24,29 @@ def __init__(self, slug: str, params: Dict[str, Any], flusher: 'Flusher'): self._user = params.get("user") self._organization = params.get("organization") self._metadata = params.get("metadata") - + self._logs: List['BaseLog'] = [] - + self._flusher = flusher - self._flushed_promise = None + self._is_ended = False + + self._evaluators = params.get("evaluators") + self._evaluation_config = params.get("evaluationConfig") + self._logger = logger + + if "experiment" in params: + experiment = params["experiment"] + if experiment is None: + self._logger.warn("Warning: Experiment is None. This experiment will be ignored.") + elif experiment.feature_slug != self._feature_slug: + self._logger.warn("Warning: Experiment feature slug does not match trace feature slug. This experiment will be ignored.") + else: + self._experiment = experiment + + @property + def name(self) -> Optional[str]: + """Get the trace name.""" + return self._name @property def input(self) -> Optional[str]: @@ -68,38 +89,53 @@ def logs(self, logs: List['BaseLog']): self._logs = logs @property - def chain_slug(self) -> str: - """Get the chain slug.""" - return self._chain_slug + def feature_slug(self) -> str: + """Get the feature slug.""" + return self._feature_slug @property def end_time(self) -> Optional[datetime]: """Get the end time.""" return self._end_time + @property + def experiment(self) -> Optional['Experiment']: + """Get the experiment.""" + return self._experiment + + @property + def evaluation_config(self) -> Optional[Dict[str, Any]]: + """Get the evaluation configuration.""" + return self._evaluation_config + + @property + def evaluators(self) -> Optional[List[Dict[str, Any]]]: + """Get the evaluators.""" + return self._evaluators + def start(self, input: Optional[str] = None) -> 'Trace': """ Start the trace with an optional input. - + Args: input (Optional[str]): The input to the trace. - + Returns: Trace: The trace instance. """ if input: self._input = input - + self._start_time = datetime.now() return self def identify(self, params: Dict[str, Any]) -> 'Trace': """ Set identification information for the trace. - + Args: params (Dict[str, Any]): Identification parameters. - + Returns: Trace: The trace instance. """ @@ -110,23 +146,65 @@ def identify(self, params: Dict[str, Any]) -> 'Trace': def set_metadata(self, metadata: Dict[str, Any]) -> 'Trace': """ Set metadata for the trace. - + Args: metadata (Dict[str, Any]): The metadata to set. - + Returns: Trace: The trace instance. """ self._metadata = metadata return self + def set_evaluation_config(self, config: Dict[str, Any]) -> 'Trace': + """ + Set the evaluation configuration for the trace. + + Args: + config (Dict[str, Any]): The evaluation configuration to set. + + Returns: + Trace: The trace instance. + """ + self._evaluation_config = config + return self + + def set_experiment(self, experiment: Dict[str, Any]) -> 'Trace': + """ + Set the experiment for the trace. + + Args: + experiment (Dict[str, Any]): The experiment to set. + + Returns: + Trace: The trace instance. + """ + self._experiment = experiment + return self + + def add_evaluator(self, evaluator: Evaluator) -> 'Trace': + """ + Add an evaluator to the trace. + + Args: + evaluator (Dict[str, Any]): The evaluator to add. + + Returns: + Trace: The trace instance. + """ + if self._evaluators is None: + self._evaluators = [] + + self._evaluators.append(evaluator) + return self + def update(self, params: Dict[str, Any]) -> 'Trace': """ Update the trace. - + Args: params (Dict[str, Any]): Parameters to update. - + Returns: Trace: The trace instance. """ @@ -135,15 +213,17 @@ def update(self, params: Dict[str, Any]) -> 'Trace': self._output = params.get("output", self._output) self._organization = params.get("organization", self._organization) self._user = params.get("user", self._user) - + if params.get("start_time"): self._start_time = params.get("start_time") - + if params.get("end_time"): self._end_time = params.get("end_time") - + self._name = params.get("name", self._name) - + self._evaluators = params.get("evaluators", self._evaluators) + self._evaluation_config = params.get("evaluationConfig", self._evaluation_config) + return self def append(self, generation: 'Generation') -> 'Trace': @@ -207,32 +287,33 @@ def create_log(self, params: Dict[str, Any]) -> 'BaseLog': **params, "trace": self }) - + return log def end(self, output: Optional[str] = None) -> 'Trace': """ End the trace with an optional output. - + Args: output (Optional[str]): The output of the trace. - + Returns: Trace: The trace instance. """ self._output = output if output is not None else self._output - self._end_time = datetime.now() - + # Send to the API using the flusher - if not self._flushed_promise: + if self._can_flush(): + self._end_time = datetime.now() + self._is_ended = True self._flusher.flush_trace(self) - - return self + + return self def to_dict(self) -> Dict[str, Any]: """Convert the trace to a dictionary for API serialization.""" return { - "chain_slug": self._chain_slug, + "feature_slug": self._feature_slug, "input": self._input, "output": self._output, "name": self._name, @@ -241,5 +322,20 @@ def to_dict(self) -> Dict[str, Any]: "user": self._user, "organization": self._organization, "metadata": self._metadata, - "logs": self._logs - } \ No newline at end of file + "logs": self._logs, + "experiment": self._experiment, + "evaluators": self._evaluators, + "evaluation_config": self._evaluation_config + } + + def _can_flush(self) -> bool: + """ + Check if the trace can be flushed. + + Returns: + bool: True if the trace can be flushed, False otherwise. + """ + if self._is_ended: + self._logger.warn('Trace already ended. This operation will be ignored.') + + return not self._is_ended \ No newline at end of file diff --git a/basalt/ressources/__init__.py b/basalt/ressources/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/basalt/ressources/monitor/__init__.py b/basalt/ressources/monitor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/basalt/ressources/monitor/base_log_types.py b/basalt/ressources/monitor/base_log_types.py new file mode 100644 index 0000000..95e41c2 --- /dev/null +++ b/basalt/ressources/monitor/base_log_types.py @@ -0,0 +1,118 @@ +from datetime import datetime +from typing import Dict, List, Optional, Union, Any, TYPE_CHECKING +from dataclasses import dataclass, field +from uuid import uuid4 + +from .evaluator_types import Evaluator +from .log_type import LogType + +if TYPE_CHECKING: + from .trace_types import Trace + from .log_types import Log + +@dataclass +class BaseLogParams: + """Base parameters for creating a log entry. + + Attributes: + name: Name of the log entry, describing what it represents. + start_time: When the log entry started, can be a datetime object or ISO string. + If not provided, defaults to the current time when created. + end_time: When the log entry ended, can be a datetime object or ISO string. + Can be set later using the end() method. + metadata: Additional contextual information about this log entry. + Can be any structured data relevant to the operation being logged. + parent: Optional parent span if this log is part of a larger operation. + Used to establish hierarchical relationships between operations. + trace: The trace this log belongs to, providing the overall context. + Every log must be associated with a trace. + evaluators: The evaluators to attach to the log. + """ + name: str + start_time: Optional[Union[datetime, str]] = None + end_time: Optional[Union[datetime, str]] = None + metadata: Optional[Dict[str, Any]] = None + parent: Optional['Log'] = None + trace: 'Trace' = None + evaluators: Optional[List[Evaluator]] = None + +@dataclass +class BaseLog: + """Base class for all log entries. + + Attributes: + id: Unique identifier for this log entry. + Automatically generated when the log is created. + type: The type of log entry (e.g., 'span', 'generation'). + Used to distinguish between different kinds of logs. + name: Name of the log entry, describing what it represents. + start_time: When the log entry started, can be a datetime object or ISO string. + If not provided, defaults to the current time when created. + end_time: When the log entry ended, can be a datetime object or ISO string. + Can be set later using the end() method. + metadata: Additional contextual information about this log entry. + Can be any structured data relevant to the operation being logged. + parent: Optional parent span if this log is part of a larger operation. + Used to establish hierarchical relationships between operations. + trace: The trace this log belongs to, providing the overall context. + Every log must be associated with a trace. + evaluators: List of evaluators attached to the log. + """ + id: str = field(default_factory=lambda: str(f'log-{uuid4().hex[:8]}')) + type: LogType = None + name: str = None + start_time: Optional[Union[datetime, str]] = None + end_time: Optional[Union[datetime, str]] = None + metadata: Optional[Dict[str, Any]] = None + parent: Optional['Log'] = None + trace: 'Trace' = None + evaluators: List[Evaluator] = field(default_factory=list) + + def start(self) -> 'BaseLog': + """Marks the log as started and sets the start time if not already set. + + Returns: + The log instance for method chaining. + """ + ... + + def set_metadata(self, metadata: Optional[Dict[str, Any]] = None) -> 'BaseLog': + """Sets the metadata for the log. + + Args: + metadata: The metadata to set for the log. + + Returns: + The log instance for method chaining. + """ + ... + + def add_evaluator(self, evaluator: Evaluator) -> 'BaseLog': + """Adds an evaluator to the log. + + Args: + evaluator: The evaluator to add to the log. + + Returns: + The log instance for method chaining. + """ + ... + + def update(self, **params) -> 'BaseLog': + """Updates the log with new parameters. + + Args: + **params: The parameters to update. + + Returns: + The log instance for method chaining. + """ + ... + + def end(self) -> 'BaseLog': + """Marks the log as ended. + + Returns: + The log instance for method chaining. + """ + ... diff --git a/basalt/ressources/monitor/evaluator_types.py b/basalt/ressources/monitor/evaluator_types.py new file mode 100644 index 0000000..237d376 --- /dev/null +++ b/basalt/ressources/monitor/evaluator_types.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass +from typing import Optional + +@dataclass +class Evaluator: + """ + Represents an evaluator configuration. + """ + slug: str + +@dataclass +class EvaluationConfig: + """ + Configuration for the evaluation of the trace and its logs. + """ + sample_rate: Optional[float] = None diff --git a/basalt/ressources/monitor/experiment_types.py b/basalt/ressources/monitor/experiment_types.py new file mode 100644 index 0000000..5aafc19 --- /dev/null +++ b/basalt/ressources/monitor/experiment_types.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass +from datetime import datetime + +@dataclass +class ExperimentParams: + """Parameters for creating an experiment.""" + name: str + +@dataclass +class Experiment: + id: str + name: str + feature_slug: str + created_at: datetime diff --git a/basalt/ressources/monitor/generation_types.py b/basalt/ressources/monitor/generation_types.py new file mode 100644 index 0000000..24f56e8 --- /dev/null +++ b/basalt/ressources/monitor/generation_types.py @@ -0,0 +1,156 @@ +from typing import Dict, Optional, Union, Any +from dataclasses import dataclass, field + +from .base_log_types import BaseLog, BaseLogParams, LogType + +@dataclass +class PromptReference: + """Reference to a prompt template. + + This class represents a reference to a prompt template used in AI model generations. + + Attributes: + slug (str): Unique identifier for the prompt template. + version (str): Version of the prompt template. + + Example: + ```python + # Basic prompt reference + prompt = PromptReference(slug="qa-prompt", version="2.1.0") + ``` + """ + slug: str + version: str + +@dataclass +class GenerationParams(BaseLogParams): + """Parameters for creating a new generation. + + This class defines the parameters that can be used to create a new generation, + either with or without a prompt reference. + + Attributes: + prompt (Optional[PromptReference]): Reference to the prompt template used. + input (Optional[str]): The input provided to the model. + output (Optional[str]): The output generated by the model. + variables (Optional[Dict[str, Any]]): Variables used in the prompt template. + + Example: + ```python + # Create generation parameters with a prompt reference + params = GenerationParams( + name="answer-generation", + prompt=PromptReference(slug="qa-prompt", version="2.1.0"), + input="What is the capital of France?", + variables={"style": "concise", "language": "en"} + ) + + # Create generation parameters without a prompt reference + params = GenerationParams( + name="text-completion", + input="Complete this sentence: The sky is", + output="The sky is blue and vast." + ) + ``` + """ + prompt: Optional[PromptReference] = None + input: Optional[str] = None + output: Optional[str] = None + variables: Optional[Dict[str, Any]] = None + +@dataclass +class Generation(BaseLog): + """Generation class representing an AI model generation within a trace. + + This class tracks interactions with AI models, including inputs, outputs, + and prompt information used for the generation. + + Attributes: + prompt (Optional[PromptReference]): Reference to the prompt template used. + input (Optional[str]): The input provided to the model. + output (Optional[str]): The output generated by the model. + variables (Optional[Dict[str, Any]]): Variables used in the prompt template. + type (str): The type of log, defaults to LogType.GENERATION. + input_tokens (Optional[int]): Number of tokens used for the input. + output_tokens (Optional[int]): Number of tokens used for the output. + cost (Optional[float]): Cost of the generation. + + Example: + ```python + # Create a generation with a prompt reference + generation = trace.create_generation( + name="answer-generation", + prompt=PromptReference(slug="qa-prompt", version="2.1.0"), + input="What is the capital of France?" + ) + + # Start the generation + generation.start() + + # End the generation with output + generation.end("The capital of France is Paris.") + + # Update generation metadata + generation.update(metadata={ + "model_version": "gpt-4", + "tokens_used": 42 + }) + ``` + """ + prompt: Optional[PromptReference] = None + input: Optional[str] = None + output: Optional[str] = None + variables: Optional[Dict[str, Any]] = None + type: str = field(default=LogType.GENERATION) + input_tokens: Optional[int] = None + output_tokens: Optional[int] = None + cost: Optional[float] = None + + def start(self, input: Optional[str] = None) -> 'Generation': + """Marks the generation as started and sets the input if provided. + + Args: + input (Optional[str]): Optional input data to associate with the generation. + + Returns: + Generation: The generation instance for method chaining. + + Example: + ```python + # Start a generation without input + generation.start() + + # Start a generation with input + generation.start("What is the capital of France?") + ``` + """ + ... + + def end(self, output: Optional[Union[str, Dict[str, Any]]] = None) -> 'Generation': + """Marks the generation as ended and sets the output if provided. + + Args: + output (Optional[Union[str, Dict[str, Any]]]): Optional output data from the model. + Can be either a string or a dictionary containing output parameters. + + Returns: + Generation: The generation instance for method chaining. + + Example: + ```python + # End a generation without output + generation.end() + + # End a generation with output as string + generation.end("The capital of France is Paris.") + + # End a generation with output params + generation.end({ + "output": "The capital of France is Paris.", + "input_tokens": 10, + "output_tokens": 10, + "cost": 0.01 + }) + ``` + """ + ... diff --git a/basalt/ressources/monitor/log_type.py b/basalt/ressources/monitor/log_type.py new file mode 100644 index 0000000..cd4c1f2 --- /dev/null +++ b/basalt/ressources/monitor/log_type.py @@ -0,0 +1,17 @@ +class LogType: + """Enum-like class for log types. + + Attributes: + SPAN: Represents a span log type + GENERATION: Represents a generation log type + FUNCTION: Represents a function log type + TOOL: Represents a tool log type + RETRIEVAL: Represents a retrieval log type + EVENT: Represents an event log type + """ + SPAN = 'span' + GENERATION = 'generation' + FUNCTION = 'function' + TOOL = 'tool' + RETRIEVAL = 'retrieval' + EVENT = 'event' \ No newline at end of file diff --git a/basalt/ressources/monitor/log_types.py b/basalt/ressources/monitor/log_types.py new file mode 100644 index 0000000..652559c --- /dev/null +++ b/basalt/ressources/monitor/log_types.py @@ -0,0 +1,188 @@ +from typing import Optional, TYPE_CHECKING +from dataclasses import dataclass + +from .base_log_types import BaseLog, BaseLogParams +from .log_type import LogType + +if TYPE_CHECKING: + from .generation_types import Generation, GenerationParams + +@dataclass +class LogParams(BaseLogParams): + """Parameters for creating or updating a log. + + This class defines the parameters needed to create or update a log entry, + including its type, input, and output data. + + Attributes: + type: The type of log entry (e.g., 'span', 'generation'). + Used to distinguish between different kinds of logs. + input: Optional input data for this operation. + output: Optional output data generated by the operation. + """ + type: LogType = None + input: Optional[str] = None + output: Optional[str] = None + +@dataclass +class Log(BaseLog): + """Log interface representing a specific operation or step within a trace. + + Logs are used to track discrete operations within a process flow, such as + data fetching, validation, or any other logical step. Logs can contain + generations and can be nested within other logs to create a hierarchical + structure of operations. + + Example: + ```python + # Create a log within a trace + log = trace.create_log({ + 'name': 'data-processing' + }) + + # Start the log with input + log.start('Raw user data') + + # Create a nested log for a sub-operation + validation_log = log.create_log({ + 'name': 'data-validation' + }) + + # Create a generation within the validation log + generation = validation_log.create_generation({ + 'name': 'validation-check', + 'prompt': {'slug': 'data-validator', 'version': '1.0.0'}, + 'input': 'Raw user data' + }) + + # End the generation with output + generation.end('Data is valid') + + # End the validation log + validation_log.end('Validation complete') + + # End the main log with processed output + log.end('Processed user data') + ``` + """ + input: Optional[str] = None + output: Optional[str] = None + + def start(self, input: Optional[str] = None) -> 'Log': + """Marks the log as started and sets the input if provided. + + Args: + input: Optional input data to associate with the log. + + Returns: + The log instance for method chaining. + + Example: + ```python + # Start a log without input + log.start() + + # Start a log with input + log.start('Raw user data to be processed') + ``` + """ + ... + + def end(self, output: Optional[str] = None) -> 'Log': + """Marks the log as ended and sets the output if provided. + + Args: + output: Optional output data to associate with the log. + + Returns: + The log instance for method chaining. + + Example: + ```python + # End a log without output + log.end() + + # End a log with output + log.end('Processed data: {"success": true, "items": 42}') + ``` + """ + ... + + def append(self, generation: 'Generation') -> 'Log': + """Adds a generation to this log. + + Args: + generation: The generation to add to this log. + + Returns: + The log instance for method chaining. + + Example: + ```python + # Create a generation separately + generation = monitor_sdk.create_generation({ + 'name': 'external-generation', + 'trace': trace + }) + + # Append the generation to this log + log.append(generation) + ``` + """ + ... + + def create_generation(self, params: 'GenerationParams') -> 'Generation': + """Creates a new generation within this log. + + Args: + params: Parameters for the generation. + + Returns: + A new Generation instance associated with this log. + + Example: + ```python + # Create a generation with a prompt reference + generation = log.create_generation({ + 'name': 'text-analysis', + 'prompt': {'slug': 'text-analyzer', 'version': '1.2.0'}, + 'input': 'Analyze this text for sentiment and key topics', + 'variables': {'language': 'en', 'mode': 'detailed'}, + 'metadata': {'priority': 'high'} + }) + + # Create a simple generation without a prompt reference + simple_generation = log.create_generation({ + 'name': 'quick-check', + 'input': 'Is this text appropriate?', + 'output': 'Yes, the text is appropriate for all audiences.' + }) + ``` + """ + ... + + def create_log(self, params: LogParams) -> 'Log': + """Creates a new nested log within this log. + + Args: + params: Parameters for the nested log. + + Returns: + A new Log instance associated with this log as its parent. + + Example: + ```python + # Create a basic nested log + nested_log = log.create_log({ + 'name': 'sub-operation' + }) + + # Create a detailed nested log + detailed_nested_log = log.create_log({ + 'name': 'data-transformation', + 'input': 'Raw data format', + 'metadata': {'transformType': 'json-to-xml', 'preserveOrder': True} + }) + ``` + """ + ... \ No newline at end of file diff --git a/basalt/ressources/monitor/monitorsdk_types.py b/basalt/ressources/monitor/monitorsdk_types.py new file mode 100644 index 0000000..dfc6fa6 --- /dev/null +++ b/basalt/ressources/monitor/monitorsdk_types.py @@ -0,0 +1,176 @@ +from typing import Protocol, Optional, Tuple +from .trace_types import TraceParams +from .experiment_types import ExperimentParams +from .experiment_types import Experiment +from .trace_types import Trace +from .generation_types import GenerationParams, Generation +from .log_types import LogParams, Log + +class IMonitorSDK(Protocol): + """Interface for interacting with Basalt monitoring. + + The MonitorSDK provides methods to create and manage traces, generations, and logs + for monitoring and tracking AI application flows. + + Examples: + ```python + # Example 1: Creating a trace + trace = basalt.monitor.create_trace('user-session', { + 'input': 'User started a new session', + 'metadata': {'userId': '123', 'sessionType': 'web'} + }) + + # Example 2: Creating a generation within a trace + generation = basalt.monitor.create_generation({ + 'name': 'text-completion', + 'prompt': {'slug': 'text-completion-prompt', 'version': '1.0.0'}, + 'input': 'Tell me a joke', + 'trace': trace + }) + + # Example 3: Creating a span for a processing step + span = basalt.monitor.create_log({ + 'type': 'span', + 'name': 'data-processing', + 'trace': trace, + 'metadata': {'processingType': 'text-analysis'} + }) + ``` + """ + + def create_trace(self, slug: str, params: Optional[TraceParams] = None) -> Trace: + """Creates a new trace to monitor a complete user interaction or process flow. + + Args: + slug: The unique identifier of the feature to which the trace belongs. + params: Optional parameters for the trace. + - output: Final output data for the trace. + - start_time: When the trace started (defaults to now if not provided). + - end_time: When the trace ended. + - user: User information associated with this trace. + - organization: Organization information associated with this trace. + - metadata: Additional contextual information for the trace. + + Examples: + ```python + # Create a basic trace + basic_trace = basalt.monitor.create_trace('user-query') + + # Create a trace with parameters + detailed_trace = basalt.monitor.create_trace('document-processing', { + 'input': 'Raw document text', + 'start_time': datetime.now(), + 'user': {'id': 'user-123', 'name': 'John Doe'}, + 'metadata': {'documentId': 'doc-456', 'documentType': 'invoice'} + }) + ``` + + Returns: + A Trace object that can be used to track the process flow. + """ + ... + + def create_generation(self, params: GenerationParams) -> Generation: + """Creates a new generation to track an AI model generation within a trace. + + Args: + params: Parameters for the generation. + - name: Name of the generation (required). + - trace: The parent trace this generation belongs to (required). + - prompt: Information about the prompt used for generation. + - slug: Prompt identifier. + - version: Prompt version. + - tag: Prompt tag. + - input: The input provided to the model. + - variables: Variables used in the prompt template. + - output: The output generated by the model. + - start_time: When the generation started. + - end_time: When the generation completed. + - metadata: Additional contextual information. + - parent: Optional parent log if this generation is part of a log. + + Examples: + ```python + # Create a generation with a prompt reference + generation = basalt.monitor.create_generation({ + 'name': 'answer-generation', + 'trace': trace, + 'prompt': {'slug': 'qa-prompt', 'version': '2.1.0'}, + 'input': 'What is the capital of France?', + 'variables': {'style': 'concise', 'language': 'en'}, + 'metadata': {'modelVersion': 'gpt-4'} + }) + + # Create a generation without a prompt reference + simple_generation = basalt.monitor.create_generation({ + 'name': 'text-completion', + 'trace': trace, + 'input': 'Complete this sentence: The sky is', + 'output': 'The sky is blue and vast.' + }) + ``` + + Returns: + A Generation object that can be used to track the AI generation. + """ + ... + + def create_log(self, params: LogParams) -> Log: + """Creates a new log to track a specific operation or step within a trace. + + Args: + params: Parameters for the log. + - name: Name of the log (required). + - trace: The parent trace this log belongs to (required). + - input: The input data for this operation. + - start_time: When the log started. + - end_time: When the log completed. + - metadata: Additional contextual information. + - parent: Optional parent log if this is a nested log. + + Examples: + ```python + # Create a basic log + basic_log = basalt.monitor.create_log({ + 'name': 'data-fetching', + 'trace': trace + }) + + # Create a detailed log + detailed_log = basalt.monitor.create_log({ + 'name': 'user-validation', + 'trace': trace, + 'input': 'user credentials', + 'metadata': {'validationRules': ['password-strength', 'email-format']}, + 'parent': parent_log # Another log this is nested under + }) + ``` + + Returns: + A Log object that can be used to track the operation. + """ + ... + + def create_experiment(self, feature_slug: str, params: ExperimentParams) -> Tuple[Optional[Exception], Optional[Experiment]]: + """Creates a new experiment to bundle multiple traces together in. + + You can pass this experiment to the create_trace method to add the generated traces to the experiment. + It's used mostly for local experimentations, to compare the performance between different versions of a workflow. + + Args: + feature_slug: The unique identifier of the feature to which the experiment belongs. + params: Parameters for the experiment. + - name: Name of the experiment (required). + + Examples: + ```python + experiment = basalt.monitor.create_experiment('user-query', {'name': 'my-experiment'}) + + # Create a trace and add it to the experiment + trace = basalt.monitor.create_trace('user-query', {'experiment': experiment}) + ``` + + Returns: + A tuple containing (Optional[Exception], Optional[Experiment]). The Experiment object can be used to track the AI generation. + """ + ... diff --git a/basalt/ressources/monitor/trace_types.py b/basalt/ressources/monitor/trace_types.py new file mode 100644 index 0000000..41cdfdc --- /dev/null +++ b/basalt/ressources/monitor/trace_types.py @@ -0,0 +1,301 @@ +from datetime import datetime +from typing import Dict, Optional, Any, List, TYPE_CHECKING +from dataclasses import dataclass, field +from .experiment_types import Experiment +from .evaluator_types import Evaluator, EvaluationConfig +from .log_type import LogType + +if TYPE_CHECKING: + from .log_types import Log, LogParams + from .generation_types import Generation, GenerationParams + from .base_log_types import BaseLog + +@dataclass +class User: + """User information associated with a trace.""" + id: str + name: str + +@dataclass +class Organization: + """Organization information associated with a trace.""" + id: str + name: str + +@dataclass +class TraceParams: + """Parameters for creating or updating a trace.""" + name: Optional[str] = None + input: Optional[str] = None + output: Optional[str] = None + start_time: Optional[datetime] = None + end_time: Optional[datetime] = None + user: Optional[User] = None + organization: Optional[Organization] = None + metadata: Optional[Dict[str, Any]] = None + experiment: Optional['Experiment'] = None + evaluators: Optional[List[Evaluator]] = None + evaluation_config: Optional[EvaluationConfig] = None + +@dataclass +class Trace(TraceParams): + """A trace represents a complete user interaction or process flow and serves as the top-level container for all monitoring activities. + + A trace provides methods to create and manage spans and generations within the process flow. + + Example: + ```python + # Create a basic trace + trace = monitor_sdk.create_trace('user-query') + + # Start the trace with input + trace.start('What is the capital of France?') + + # Create a span within the trace + processing_log = trace.create_log( + name='query-processing', + type='process' + ) + + # Create a generation within the span + generation = processing_log.create_generation( + name='answer-generation', + prompt={'slug': 'qa-prompt', 'version': '1.0.0'}, + input='What is the capital of France?' + ) + + # End the generation with output + generation.end('The capital of France is Paris.') + + # End the span + processing_log.end() + + # End the trace with final output + trace.end('Paris is the capital of France.') + ``` + """ + + start_time: datetime + logs: List['BaseLog'] = field(default_factory=list) + + def start(self, input: Optional[str] = None) -> 'Trace': + """Marks the trace as started and sets the input if provided. + + Args: + input: Optional input data to associate with the trace. + + Returns: + The trace instance for method chaining. + + Example: + ```python + # Start a trace without input + trace.start() + + # Start a trace with input + trace.start('User query: What is the capital of France?') + ``` + """ + ... + + def set_metadata(self, metadata: Dict[str, Any]) -> 'Trace': + """Sets or updates the metadata for this trace. + + Args: + metadata: The metadata to associate with this trace. + + Returns: + The trace instance for method chaining. + + Example: + ```python + # Add metadata to the trace + trace.set_metadata({ + 'user_id': 'user-123', + 'session_id': 'session-456', + 'source': 'web-app' + }) + ``` + """ + ... + + def set_evaluation_config(self, config: EvaluationConfig) -> 'Trace': + """Sets the evaluation configuration for the trace. + + Args: + config: The evaluation configuration to set. + + Returns: + The trace instance for method chaining. + """ + ... + + def set_experiment(self, experiment: Experiment) -> 'Trace': + """Sets the experiment for the trace. + + Args: + experiment: The experiment to set. + + Returns: + The trace instance for method chaining. + """ + ... + + def update(self, params: TraceParams) -> 'Trace': + """Updates the trace with new parameters. + The new parameters given in this method will override the existing ones. + + Args: + params: The parameters to update. + + Returns: + The trace instance for method chaining. + + Example: + ```python + # Update trace parameters + trace.update({ + 'name': 'Updated trace name', + 'metadata': {'priority': 'high'} + }) + ``` + """ + ... + + def add_evaluator(self, evaluator: Evaluator) -> 'Trace': + """Adds an evaluator to the trace. + + Args: + evaluator: The evaluator to add to the trace. + + Returns: + The trace instance for method chaining. + """ + ... + + def append(self, log: 'BaseLog') -> 'Trace': + """Adds a log (span or generation) to this trace. + + Args: + log: The log to add to this trace. + + Returns: + The trace instance for method chaining. + + Example: + ```python + # Create a generation separately and append it to the trace + generation = monitor_sdk.create_generation( + name='external-generation', + trace=another_trace + ) + + # Append the generation to this trace + trace.append(generation) + ``` + """ + ... + + def identify(self, user: Optional[User] = None, organization: Optional[Organization] = None) -> 'Trace': + """Associates user information with this trace. + + Args: + user: The user information to associate with this trace. + organization: The organization information to associate with this trace. + + Returns: + The trace instance for method chaining. + + Example: + ```python + # Identify a user with user and organization information + trace.identify( + user=User( + id='user-123', + name='John Doe' + ), + organization=Organization( + id='org-123', + name='Acme Corporation' + ) + ) + ``` + """ + ... + + def create_generation(self, params: 'GenerationParams') -> 'Generation': + """Creates a new generation within this trace. + + Args: + params: Parameters for the generation. + + Returns: + A new Generation instance associated with this trace. + + Example: + ```python + # Create a generation with a prompt reference + generation = trace.create_generation({ + 'name': 'answer-generation', + 'prompt': {'slug': 'qa-prompt', 'version': '2.1.0'}, + 'input': 'What is the capital of France?', + 'variables': {'style': 'concise', 'language': 'en'}, + 'metadata': {'model_version': 'gpt-4'} + }) + + # Create a generation without a prompt reference + simple_generation = trace.create_generation({ + 'name': 'text-completion', + 'input': 'Complete this sentence: The sky is', + 'output': 'The sky is blue and vast.' + }) + ``` + """ + ... + + def create_log(self, params: 'LogParams') -> 'Log': + """Creates a new span within this trace. + + Args: + params: Parameters for the span. + + Returns: + A new Log instance associated with this trace. + + Example: + ```python + # Create a basic span + basic_log = trace.create_log({ + 'name': 'data-fetching', + 'type': 'io' + }) + + # Create a detailed span + detailed_log = trace.create_log({ + 'name': 'user-validation', + 'input': 'user credentials', + 'metadata': {'validation_rules': ['...word-strength', 'email-format']} + }) + ``` + """ + ... + + def end(self, output: Optional[str] = None) -> 'Trace': + """Marks the trace as ended and sets the output if provided. + + Args: + output: Optional output data to associate with the trace. + + Returns: + The trace instance for method chaining. + + Example: + ```python + # End a trace without output + trace.end() + + # End a trace with output + trace.end('The capital of France is Paris.') + ``` + """ + ... diff --git a/basalt/sdk/monitorsdk.py b/basalt/sdk/monitorsdk.py index a410752..6277296 100644 --- a/basalt/sdk/monitorsdk.py +++ b/basalt/sdk/monitorsdk.py @@ -1,12 +1,16 @@ -from typing import Dict, Optional, Any +from typing import Dict, Optional, Any, Tuple from ..utils.protocols import IApi, ILogger -from ..utils.dtos import TraceParams, GenerationParams, LogParams - +from ..ressources.monitor.trace_types import TraceParams +from ..ressources.monitor.experiment_types import ExperimentParams +from ..ressources.monitor.generation_types import GenerationParams +from ..ressources.monitor.log_types import LogParams from ..objects.trace import Trace from ..objects.generation import Generation from ..objects.log import Log +from ..objects.experiment import Experiment from ..utils.flusher import Flusher +from ..endpoints.monitor.create_experiment import CreateExperimentEndpoint, CreateExperimentDTO class MonitorSDK: """ @@ -20,25 +24,44 @@ def __init__( self._api = api self._logger = logger + def create_experiment( + self, + feature_slug: str, + params: ExperimentParams + ) -> Tuple[Optional[Exception], Optional[Experiment]]: + """ + Creates a new experiment for monitoring. + + Args: + feature_slug (str): The feature slug for the experiment. + params (Dict[str, Any]): Parameters for the experiment. + + Returns: + Experiment: A new Experiment instance. + """ + return self._create_experiment(feature_slug, params) + + def create_trace( self, slug: str, - params: Optional[Dict[str, Any]] = None + params: Optional[TraceParams] = None ) -> Trace: """ Creates a new trace for monitoring. Args: slug (str): The unique identifier for the trace. - params (Optional[Dict[str, Any]]): Optional parameters for the trace. + params (TraceParams): Parameters for the trace. Returns: Trace: A new Trace instance. """ if params is None: params = {} - + trace_params = TraceParams(**params) + return self._create_trace(slug, trace_params) def create_generation( @@ -73,6 +96,35 @@ def create_log( log_params = LogParams(**params) return self._create_log(log_params) + def _create_experiment( + self, + feature_slug: str, + params: ExperimentParams + ) -> Tuple[Optional[Exception], Optional[Experiment]]: + """ + Internal implementation for creating an experiment. + + Args: + feature_slug (str): The feature slug for the experiment. + params (ExperimentParams): Parameters for the experiment. + + Returns: + Experiment: A new Experiment instance. + """ + dto = CreateExperimentDTO( + feature_slug=feature_slug, + name=params.get("name"), + ) + + # Call the API endpoint + err, result = self._api.invoke(CreateExperimentEndpoint, dto) + + if err is None: + return None, Experiment(result.experiment) + + return err, None + + def _create_trace( self, slug: str, @@ -98,9 +150,12 @@ def _create_trace( "end_time": params.end_time, "user": params.user, "organization": params.organization, - "metadata": params.metadata + "metadata": params.metadata, + "experiment": params.experiment, + "evaluators": params.evaluators, + "evaluationConfig": params.evaluation_config } - trace = Trace(slug, params_dict, flusher) + trace = Trace(slug, params_dict, flusher, self._logger) return trace def _create_generation( @@ -156,4 +211,4 @@ def _create_log( "start_time": params.start_time, "end_time": params.end_time } - return Log(params_dict) \ No newline at end of file + return Log(params_dict) \ No newline at end of file diff --git a/basalt/sdk/promptsdk.py b/basalt/sdk/promptsdk.py index 5da4cbe..85c9e05 100644 --- a/basalt/sdk/promptsdk.py +++ b/basalt/sdk/promptsdk.py @@ -1,6 +1,6 @@ from typing import Optional, Dict, Tuple, Any -from ..utils.dtos import GetPromptDTO, PromptResponse, DescribePromptResponse, DescribePromptDTO, GetResult, DescribeResult, ListResult, PromptListResponse +from ..utils.dtos import GetPromptDTO, PromptResponse, DescribePromptResponse, DescribePromptDTO, GetResult, DescribeResult, ListResult, PromptListResponse, PromptListDTO from ..utils.protocols import ICache, IApi, ILogger from ..endpoints.get_prompt import GetPromptEndpoint @@ -118,7 +118,7 @@ def _prepare_monitoring( trace = Trace(slug, { "input": original_prompt_text or prompt.text, "start_time": datetime.now() - }, flusher) + }, flusher, self._logger) # Create a generation generation = Generation({ @@ -177,8 +177,10 @@ def describe( return err, None - def list(self) -> ListResult: - err, result = self._api.invoke(ListPromptsEndpoint) + def list(self, feature_slug: Optional[str] = None) -> ListResult: + dto = PromptListDTO(featureSlug=feature_slug) + + err, result = self._api.invoke(ListPromptsEndpoint, dto) if err is not None: return err, None @@ -194,14 +196,19 @@ def list(self) -> ListResult: def _replace_vars(self, prompt: PromptResponse, variables: Dict[str, str] = {}): missing_vars, replaced = replace_variables(prompt.text, variables) + missing_system_vars, replaced_system = replace_variables(prompt.systemText or "", variables) if missing_vars: self._logger.warn(f"""Basalt Warning: Some variables are missing in the prompt text: {", ".join(map(str, missing_vars))}""") + if missing_system_vars: + self._logger.warn(f"""Basalt Warning: Some variables are missing in the prompt systemText: + {", ".join(map(str, missing_system_vars))}""") + return None, PromptResponse( text=replaced, - systemText=prompt.systemText, + systemText=replaced_system, version=prompt.version, model=prompt.model ) \ No newline at end of file diff --git a/basalt/utils/api.py b/basalt/utils/api.py index b19a1e9..7bb928c 100644 --- a/basalt/utils/api.py +++ b/basalt/utils/api.py @@ -20,7 +20,7 @@ class Api: def __init__(self, root_url: str, networker: INetworker, api_key: str, sdk_version: str, sdk_type: str, logger: Optional[ILogger] = None): """ Initialize the Api class with the given parameters. - + Args: root_url (str): The root URL of the API. networker (INetworker): The networker instance to handle network requests. diff --git a/basalt/utils/dtos.py b/basalt/utils/dtos.py index d9b2031..2de0621 100644 --- a/basalt/utils/dtos.py +++ b/basalt/utils/dtos.py @@ -1,6 +1,8 @@ from dataclasses import dataclass from typing import Optional, Dict, Any, List, Tuple +from ..ressources.monitor.generation_types import Generation + from .utils import pick_typed, pick_number # ------------------------------ Get Prompt ----------------------------- # @@ -68,7 +70,7 @@ class GetPromptDTO: tag: Optional[str] = None version: Optional[str] = None -GetResult = Tuple[Optional[Exception], Optional[PromptResponse]] +GetResult = Tuple[Optional[Exception], Optional[PromptResponse], Optional[Generation]] # ------------------------------ Describe Prompt ----------------------------- # @dataclass(frozen=True) @@ -122,46 +124,10 @@ def from_dict(cls, data: Dict[str, Any]): available_tags=pick_typed(data, "availableTags", list), ) -ListResult = Tuple[Optional[Exception], Optional[List[PromptListResponse]]] +@dataclass(frozen=True) +class PromptListDTO: + featureSlug: Optional[str] = None -# ------------------------------ Monitor ----------------------------- # -@dataclass -class TraceParams: - """Parameters for creating a trace.""" - input: Optional[str] = None - output: Optional[str] = None - name: Optional[str] = None - start_time: Optional[Any] = None - end_time: Optional[Any] = None - user: Optional[Dict[str, Any]] = None - organization: Optional[Dict[str, Any]] = None - metadata: Optional[Dict[str, Any]] = None -@dataclass -class GenerationParams: - """Parameters for creating a generation.""" - name: str - trace: Any - prompt: Optional[Dict[str, Any]] = None - input: Optional[str] = None - output: Optional[str] = None - variables: Optional[Dict[str, Any]] = None - parent: Optional[Any] = None - metadata: Optional[Dict[str, Any]] = None - start_time: Optional[Any] = None - end_time: Optional[Any] = None - options: Optional[Dict[str, Any]] = None -@dataclass -class LogParams: - """Parameters for creating a log.""" - name: str - trace: Any - input: Optional[str] = None - output: Optional[str] = None - parent: Optional[Any] = None - metadata: Optional[Dict[str, Any]] = None - start_time: Optional[Any] = None - end_time: Optional[Any] = None - -MonitorResult = Tuple[Optional[Exception], Optional[Any]] \ No newline at end of file +ListResult = Tuple[Optional[Exception], Optional[List[PromptListResponse]]] \ No newline at end of file diff --git a/basalt/utils/flusher.py b/basalt/utils/flusher.py index ca11c85..e25e23d 100644 --- a/basalt/utils/flusher.py +++ b/basalt/utils/flusher.py @@ -30,7 +30,7 @@ def _trace_to_dict(self, trace: 'Trace') -> Dict[str, Any]: if output is not None and not isinstance(output, str): output = json.dumps(output) return { - "chain_slug": trace.chain_slug, + "feature_slug": trace.feature_slug, "input": trace.input, "output": output, "name": trace._name, @@ -39,7 +39,10 @@ def _trace_to_dict(self, trace: 'Trace') -> Dict[str, Any]: "user": trace.user, "organization": trace.organization, "metadata": trace.metadata, - "logs": [self._log_to_dict(log) for log in trace.logs] if trace.logs else [] + "logs": [self._log_to_dict(log) for log in trace.logs] if trace.logs else [], + "experiment": trace.experiment, + "evaluators": trace.evaluators, + "evaluationConfig": trace.evaluation_config } def _log_to_dict(self, log: Any) -> Dict[str, Any]: @@ -88,26 +91,24 @@ def flush_trace(self, trace: 'Trace') -> None: """ try: if not self._api: - self._logger.warn("Cannot flush trace: no API instance available") + self._logger.error("Cannot flush trace: no API instance available") return - + # Create an endpoint instance endpoint = SendTraceEndpoint() - + # Convert trace to dictionary trace_dict = self._trace_to_dict(trace) - + # Create the DTO with the trace dictionary dto = {"trace": trace_dict} - + # Invoke the API with the endpoint and DTO error, result = self._api.invoke(endpoint, dto) - + if error: - self._logger.warn(f"Failed to flush trace: {error}") + self._logger.error(f"Failed to flush trace {trace.feature_slug}: {error}") return - - self._logger.warn(f"Successfully flushed trace {trace.chain_slug} to the API") - + except Exception as e: - self._logger.warn(f"Exception while flushing trace: {str(e)}") \ No newline at end of file + self._logger.error(f"Exception while flushing trace: {str(e)}") \ No newline at end of file diff --git a/basalt/utils/logger.py b/basalt/utils/logger.py index e49e25d..e9c3a11 100644 --- a/basalt/utils/logger.py +++ b/basalt/utils/logger.py @@ -1,19 +1,26 @@ -from .protocols import ILogger +from .protocols import ILogger, LogLevel class Logger(ILogger): - def __init__(self, log_level: str = 'all'): + def __init__(self, log_level: LogLevel = 'all'): self._log_level = log_level def warn(self, *args): if self._can_warn(): print(*args) - def debug(self, *args): - if self._can_debug(): + def info(self, *args): + if self._can_info(): + print(*args) + + def error(self, *args): + if self._can_error(): print(*args) def _can_warn(self): - return self._log_level in ['all', 'warning', 'debug'] + return self._log_level == 'all' or self._log_level == 'warning' + + def _can_info(self): + return self._log_level == 'all' - def _can_debug(self): - return self._log_level in ['debug'] + def _can_error(self): + return True diff --git a/basalt/utils/networker.py b/basalt/utils/networker.py index b89e8b1..64a9b47 100644 --- a/basalt/utils/networker.py +++ b/basalt/utils/networker.py @@ -9,8 +9,8 @@ class Networker(INetworker): Networker class that implements the INetworker protocol. Provides a method to fetch data from a given URL using HTTP methods. """ - def __init__(self, logger: Optional[ILogger] = None): - self._logger = logger + def __init__(self): + pass def fetch( self, @@ -36,12 +36,6 @@ def fetch( - (FetchError, None) """ try: - if self._logger: - self._logger.debug(f"[DEBUG] Making request to: {url}") - self._logger.debug(f"[DEBUG] Method: {method}") - self._logger.debug(f"[DEBUG] Headers: {headers}") - self._logger.debug(f"[DEBUG] Body: {body}") - response = requests.request( method, url, @@ -50,15 +44,8 @@ def fetch( headers=headers ) - if self._logger: - self._logger.debug(f"[DEBUG] Response status: {response.status_code}") - self._logger.debug(f"[DEBUG] Response headers: {response.headers}") - json_response = response.json() - if self._logger: - self._logger.debug(f"[DEBUG] Response body: {json_response}") - if response.status_code == 400: return BadRequest(json_response.get('error', 'Bad Request')), None @@ -70,12 +57,10 @@ def fetch( if response.status_code == 404: return NotFound(json_response.get('error', 'Not Found')), None - + response.raise_for_status() return None, json_response except Exception as e: - if self._logger: - self._logger.debug(f"[DEBUG] Error: {str(e)}") return NetworkBaseError(str(e)), None diff --git a/basalt/utils/protocols.py b/basalt/utils/protocols.py index d98c903..5e8ac8a 100644 --- a/basalt/utils/protocols.py +++ b/basalt/utils/protocols.py @@ -1,5 +1,7 @@ -from typing import Any, Optional, Protocol, Hashable, Tuple, TypeVar, Dict, Mapping -from .dtos import GetResult, DescribeResult, ListResult, MonitorResult +from typing import Any, Optional, Protocol, Hashable, Tuple, TypeVar, Dict, Mapping, Literal +from .dtos import GetResult, DescribeResult, ListResult + +from ..ressources.monitor.monitorsdk_types import IMonitorSDK Input = TypeVar('Input') Output = TypeVar('Output') @@ -27,12 +29,7 @@ def fetch(self, class IPromptSDK(Protocol): def get(self, slug: str, tag: Optional[str] = None, version: Optional[str] = None, variables: Dict[str, str] = {}, cache_enabled: bool = True) -> GetResult: ... def describe(self, slug: str, tag: Optional[str] = None, version: Optional[str] = None) -> DescribeResult: ... - def list(self) -> ListResult: ... - -class IMonitorSDK(Protocol): - def create_trace(self, slug: str, params: Optional[Dict[str, Any]] = None) -> Any: ... - def create_generation(self, params: Dict[str, Any]) -> Any: ... - def create_log(self, params: Dict[str, Any]) -> Any: ... + def list(self, feature_slug: Optional[str] = None) -> ListResult: ... class IBasaltSDK(Protocol): @property @@ -42,3 +39,9 @@ def monitor(self) -> IMonitorSDK: ... class ILogger: def warn(self, message: str): ... + def info(self, message: str): ... + def error(self, message: str): ... + + +LogLevel = Literal["all", "warning", "none"] + diff --git a/tests/test_monitor_sdk.py b/tests/test_monitor_sdk.py index 60f5f9f..0a8bd31 100644 --- a/tests/test_monitor_sdk.py +++ b/tests/test_monitor_sdk.py @@ -67,7 +67,7 @@ def test_create_trace(self): self.assertEqual(trace.user, self.user) self.assertEqual(trace.organization, {"id": "org-123", "name": "Basalt"}) self.assertEqual(trace.metadata, {"property1": "value1", "property2": "value2"}) - self.assertEqual(trace.chain_slug, "test-slug") + self.assertEqual(trace.feature_slug, "test-slug") def test_create_log(self): """Test creating a log within a trace.""" diff --git a/tests/test_send_trace_endpoint.py b/tests/test_send_trace_endpoint.py index a35c3f4..b5a1bf8 100644 --- a/tests/test_send_trace_endpoint.py +++ b/tests/test_send_trace_endpoint.py @@ -14,7 +14,7 @@ def test_prepare_request_with_empty_dto(self): def test_prepare_request_with_full_trace(self): # Create a mock trace with all required fields trace = { - "chain_slug": "test-chain", + "feature_slug": "test-feature", "input": {"query": "test"}, "output": {"response": "test-response"}, "metadata": {"test": "metadata"}, @@ -47,7 +47,7 @@ def test_prepare_request_with_full_trace(self): # Verify the body contains all required fields body = result["body"] - self.assertEqual(body["chainSlug"], "test-chain") + self.assertEqual(body["featureSlug"], "test-feature") self.assertEqual(body["input"], {"query": "test"}) self.assertEqual(body["output"], {"response": "test-response"}) self.assertEqual(body["metadata"], {"test": "metadata"})