Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 9 additions & 57 deletions .github/workflows/python-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: Python SDK Tests

on:
pull_request:

# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:

Expand All @@ -12,27 +12,27 @@ jobs:
pull-requests: write

runs-on: ubuntu-latest

strategy:
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']

steps:
- uses: actions/checkout@v3

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

- name: Install dependencies
working-directory: ./
run: |
python -m pip install --upgrade pip
pip install pytest pytest-cov
pip install -e .
pip install -r dev-requirements.txt

- name: Run tests
working-directory: ./
run: |
Expand All @@ -41,64 +41,16 @@ jobs:
echo "COVERAGE_PCT=$(python -c "import xml.etree.ElementTree as ET; tree = ET.parse('coverage.xml'); root = tree.getroot(); print(f'{float(root.attrib[\"line-rate\"]) * 100:.2f}')")" >> $GITHUB_OUTPUT
echo "TEST_COUNT=$(python -c "import xml.etree.ElementTree as ET; tree = ET.parse('coverage.xml'); root = tree.getroot(); print(root.find('.//metrics').attrib['tests'])")" >> $GITHUB_OUTPUT
id: test_results

- name: Create result file
run: |
mkdir -p test-results
echo "${{ steps.test_results.outputs.COVERAGE_PCT }}" > test-results/coverage.txt
echo "${{ steps.test_results.outputs.TEST_COUNT }}" > test-results/test-count.txt

- name: Upload test results
uses: actions/upload-artifact@v4
with:
name: test-results-${{ matrix.python-version }}
path: test-results/

comment:
needs: test
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'

steps:
- name: Download all artifacts
uses: actions/download-artifact@v4

- name: Prepare comment
id: prepare_comment
run: |
echo "COMMENT_BODY<<EOF" >> $GITHUB_ENV
echo "## Python SDK Test Results" >> $GITHUB_ENV
echo "" >> $GITHUB_ENV
echo "| Python Version | Status | Coverage | Tests Run |" >> $GITHUB_ENV
echo "| -------------- | ------ | -------- | --------- |" >> $GITHUB_ENV

for version in 3.8 3.9 3.10 3.11 3.12; do
version_path="test-results-$version"
if [ -d "$version_path" ] && [ -f "$version_path/coverage.txt" ]; then
coverage=$(cat "$version_path/coverage.txt")
test_count=$(cat "$version_path/test-count.txt")
echo "| Python $version | ✅ Passed | $coverage% | $test_count |" >> $GITHUB_ENV
else
echo "| Python $version | ❌ Failed or not run | - | - |" >> $GITHUB_ENV
fi
done

echo "" >> $GITHUB_ENV
echo "*Last updated: $(date -u '+%Y-%m-%d %H:%M:%S UTC')*" >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV

- name: Find Comment
uses: peter-evans/find-comment@v3
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: 'github-actions[bot]'
body-includes: Python SDK Test Results

- name: Create or update comment
uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: ${{ env.COMMENT_BODY }}
edit-mode: replace
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,6 @@ build/
dist/
__pycache__/
test.py
.DS_Store
.DS_Store
.idea/
venv/
2 changes: 1 addition & 1 deletion basalt/_version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.3.0"
__version__ = "0.4.0"
2 changes: 1 addition & 1 deletion basalt/endpoints/monitor/send_trace.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def prepare_request(self, dto: Optional[Input] = None) -> Dict[str, Any]:
processed_log["startTime"] = processed_log["start_time"].isoformat() if isinstance(processed_log["start_time"], datetime) else processed_log["start_time"]
del processed_log["start_time"]
if "end_time" in processed_log:
processed_log["endTime"] = processed_log["end_time"].isoformat() if isinstance(processed_log["end_time"], datetime) and processed_log["end_time"] else None
processed_log["endTime"] = processed_log["end_time"].isoformat() if isinstance(processed_log["end_time"], datetime) else processed_log["end_time"]
del processed_log["end_time"]

# Extract parent ID
Expand Down
17 changes: 9 additions & 8 deletions basalt/objects/base_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@
from typing import Dict, Optional, Any, List
import uuid

from ..ressources.monitor.base_log_types import BaseLogParams
from ..ressources.monitor.base_log_types import BaseLogParams, LogType
from ..ressources.monitor.evaluator_types import Evaluator
from ..ressources.monitor.trace_types import Trace
from ..ressources.monitor.log_types import Log


class BaseLog:
"""
Base class for logs and generations.
Expand All @@ -15,7 +16,7 @@ def __init__(self, params: BaseLogParams):
self._id = f"log-{uuid.uuid4().hex[:8]}"
self._type = params.get("type")
self._name = params.get("name")
self._start_time = params.get("start_time", datetime.now())
self._start_time = params.get("start_time") if params.get("start_time") is not None else datetime.now()
self._end_time = params.get("end_time")
self._metadata = params.get("metadata")
self._trace = params.get("trace")
Expand Down Expand Up @@ -43,7 +44,7 @@ def parent(self, parent: 'Log'):
self._parent = parent

@property
def type(self) -> str:
def type(self) -> LogType:
"""Get the log type."""
return self._type

Expand Down Expand Up @@ -76,7 +77,7 @@ def metadata(self) -> Optional[Dict[str, Any]]:
def trace(self) -> 'Trace':
"""Get the trace."""
return self._trace

@property
def evaluators(self) -> List[Evaluator]:
"""Get the evaluators."""
Expand Down Expand Up @@ -113,13 +114,13 @@ def update(self, params: Dict[str, Any]) -> 'BaseLog':
"""Update the log."""
self._name = params.get("name", self._name)
self._metadata = params.get("metadata", self._metadata)

if params.get("start_time"):
self._start_time = params.get("start_time")

if params.get("end_time"):
self._end_time = params.get("end_time")

return self

def end(self) -> 'BaseLog':
Expand All @@ -138,4 +139,4 @@ def to_dict(self) -> Dict[str, Any]:
"end_time": self._end_time,
"metadata": self._metadata,
"parent": {"id": self._parent.id} if self._parent else None,
}
}
53 changes: 32 additions & 21 deletions basalt/objects/generation.py
Original file line number Diff line number Diff line change
@@ -1,26 +1,37 @@
from datetime import datetime
from typing import Dict, Optional, Any, List, Union

from .base_log import BaseLog
from ..ressources.monitor.generation_types import GenerationParams
from ..ressources.monitor.base_log_types import BaseLogParams, LogType


class Generation(BaseLog):
"""
Class representing a generation in the monitoring system.
"""
def __init__(self, params: GenerationParams):
params_with_type = {
"type": "generation",
**params
base_log_params = {
"name": params.get("name"),
"ideal_output": params.get("ideal_output"),
"start_time": params.get("start_time"),
"end_time": params.get("end_time"),
"metadata": params.get("metadata"),
"parent": params.get("parent"),
"trace": params.get("trace"),
"evaluators": params.get("evaluators"),
"type": LogType.GENERATION,
}
super().__init__(params_with_type)


super().__init__(base_log_params)

self._prompt = params.get("prompt")
self._input = params.get("input")
self._output = params.get("output")
self._input_tokens = params.get("input_tokens")
self._output_tokens = params.get("output_tokens")
self._cost = params.get("cost")

# Convert variables to array format if needed
variables = params.get("variables")
if variables is not None:
Expand All @@ -32,7 +43,7 @@ def __init__(self, params: GenerationParams):
self._variables = []
else:
self._variables = []

self._options = params.get("options")

@property
Expand Down Expand Up @@ -83,50 +94,50 @@ def options(self, options: Dict[str, Any]):
def start(self, input: Optional[str] = None) -> 'Generation':
"""
Start the generation with an optional input.

Args:
input (Optional[str]): The input to the generation.

Returns:
Generation: The generation instance.
"""
if input:
self._input = input

super().start()
return self

def end(self, output: Optional[Union[str, Dict[str, Any]]] = None) -> 'Generation':
"""
End the generation with an optional output or update parameters.

Args:
output (Optional[Union[str, Dict[str, Any]]]): The output of the generation
or a dictionary of parameters to update.

Returns:
Generation: The generation instance.
"""
super().end()

if isinstance(output, dict):
self.update(output)
elif isinstance(output, str):
self._output = output

# If this is a single generation, end the trace as well
if self._options and self._options.get("type") == "single":
self.trace.end(self._output)
self.trace.end_sync(self._output)

return self

def update(self, params: Dict[str, Any]) -> 'Generation':
"""
Update the generation.

Args:
params (Dict[str, Any]): Parameters to update.

Returns:
Generation: The generation instance.
"""
Expand All @@ -136,7 +147,7 @@ def update(self, params: Dict[str, Any]) -> 'Generation':
self._input_tokens = params.get("input_tokens", self._input_tokens)
self._output_tokens = params.get("output_tokens", self._output_tokens)
self._cost = params.get("cost", self._cost)

# Update variables if provided
variables = params.get("variables")
if variables is not None:
Expand All @@ -146,6 +157,6 @@ def update(self, params: Dict[str, Any]) -> 'Generation':
self._variables = [{"label": str(v.get("label")), "value": str(v.get("value"))} for v in variables if v.get("label")]
else:
self._variables = []

super().update(params)
return self
return self
Loading