diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2f35942 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + - name: Ruff lint + run: ruff check . + - name: Ruff format check + run: ruff format --check . + - name: Mypy + run: mypy agentargus + - name: Pytest + run: pytest --cov=agentargus --cov-report=xml --cov-report=term-missing + + e2e: + # End-to-end demo runs separately and does not block the merge gate (spec §10). + runs-on: ubuntu-latest + needs: test + if: github.event_name == 'push' + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install + run: pip install -e ".[dev]" + - name: Run demo (placeholder until Module 10) + run: echo "e2e demo lands in Module 10" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cbcf7af --- /dev/null +++ b/.gitignore @@ -0,0 +1,41 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.eggs/ +build/ +dist/ +*.egg + +# Virtual envs +.venv/ +venv/ +env/ + +# Testing / coverage +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +htmlcov/ +.coverage +coverage.xml + +# Secrets & local config +.env +.env.* +!.env.example + +# Editors / OS +.vscode/ +.idea/ +.DS_Store +Thumbs.db + +# Project artifacts +*.sqlite +*.sqlite3 +dead_letter*.jsonl + +# Internal planning / study docs (not for the public repo) +IMPLEMENTAION.md +module_notes/ diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000..9459cfa --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,25 @@ +target-version = "py310" +line-length = 100 +src = ["agentargus", "tests"] + +[lint] +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort + "W", # pycodestyle warnings + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # comprehensions + "SIM", # simplify + "T20", # flake8-print -> bans print() in library code (§7) +] + +[lint.per-file-ignores] +# Examples and CLI are allowed to print; tests may too. +"examples/**" = ["T20"] +"tests/**" = ["T20"] +"benchmarks/**" = ["T20"] + +[format] +quote-style = "double" diff --git a/AgentArgus/.gitignore b/AgentArgus/.gitignore deleted file mode 100644 index e825e1e..0000000 --- a/AgentArgus/.gitignore +++ /dev/null @@ -1,23 +0,0 @@ -# Python -__pycache__/ -*.py[cod] -*.egg-info/ -.eggs/ - -# Build -dist/ -build/ - -# Virtual environment -venv/ -.venv/ -env/ - -# Credentials — NEVER commit these -.pypirc -*.env -.env* - -# IDE -.vscode/ -.idea/ \ No newline at end of file diff --git a/AgentArgus/README.md b/AgentArgus/README.md deleted file mode 100644 index 92a8ffb..0000000 --- a/AgentArgus/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# AgentArgus - -Production-grade evaluation, observability, and reliability for AI agents. - -> Under active development. Follow progress at https://github.com/mohdcodes/AgentArgus \ No newline at end of file diff --git a/AgentArgus/agentargus/__init__.py b/AgentArgus/agentargus/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/AgentArgus/pyproject.toml b/AgentArgus/pyproject.toml deleted file mode 100644 index 396ff03..0000000 --- a/AgentArgus/pyproject.toml +++ /dev/null @@ -1,117 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -# ── Project metadata ────────────────────────────────────────────────────────── - -[project] -name = "agentargus" -version = "0.0.0" -description = "Production-grade evaluation, observability, and reliability for AI agents" -readme = "README.md" -requires-python = ">=3.10" -license = { text = "MIT" } -authors = [ - { name = "Mohd Arbaaz Siddiqui", email = "arbaazcode@gmail.com" } -] -keywords = [ - "llm", - "agents", - "evaluation", - "observability", - "reliability", - "langgraph", - "opentelemetry", - "rag", -] -classifiers = [ - "Development Status :: 1 - Planning", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Software Development :: Libraries :: Python Modules", -] - -# Runtime dependencies (empty for v0.0.0 stub — fill these in during Week 2+) -dependencies = [] - -# Optional dependency groups -[project.optional-dependencies] -dev = [ - "pytest>=8.0", - "pytest-cov>=4.1", - "pytest-asyncio>=0.23", - "ruff>=0.5", - "mypy>=1.10", - "build>=1.0", - "twine>=5.0", -] -docs = [ - "pdoc>=14.0", -] - -# ── Project URLs ────────────────────────────────────────────────────────────── - -[project.urls] -Repository = "https://github.com/mohdcodes/AgentArgus" -Issues = "https://github.com/mohdcodes/AgentArgus/issues" - -# ── Hatchling build config ──────────────────────────────────────────────────── - -[tool.hatch.build.targets.wheel] -packages = ["agentargus"] - -# ── Ruff (linter + formatter — replaces black, flake8, isort) ──────────────── - -[tool.ruff] -line-length = 100 -target-version = "py310" - -[tool.ruff.lint] -select = ["E", "F", "W", "I", "B", "UP"] -ignore = ["E501"] - -[tool.ruff.lint.isort] -known-first-party = ["agentargus"] - -# ── Pytest ──────────────────────────────────────────────────────────────────── - -[tool.pytest.ini_options] -minversion = "8.0" -testpaths = ["tests"] -python_files = ["test_*.py"] -addopts = "-v --tb=short --cov=agentargus" -asyncio_mode = "auto" -filterwarnings = [ - "ignore::DeprecationWarning", -] - -# ── Coverage ────────────────────────────────────────────────────────────────── - -[tool.coverage.run] -source = ["agentargus"] -omit = ["*/tests/*", "*/test_*.py"] - -[tool.coverage.report] -exclude_lines = [ - "pragma: no cover", - "def __repr__", - "raise AssertionError", - "raise NotImplementedError", - "if __name__ == .__main__.:", -] -precision = 2 -fail_under = 80 - -# ── Mypy ────────────────────────────────────────────────────────────────────── - -[tool.mypy] -python_version = "3.10" -warn_return_any = true -warn_unused_configs = true -ignore_missing_imports = true \ No newline at end of file diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 0000000..e6a5dd0 --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,36 @@ +# AgentArgus — Design, Scope, Non-Goals + +> Owner-authored document. Claude Code seeds it from spec §1; the owner refines +> and defends each point. + +## Problem +Teams ship LLM agents to production with almost no operational rigour. There is +no single place to answer: *Did the agent do the right thing? What did it cost? +Why did it fail? Can it recover?* RAGAS covers RAG eval only; observability +tools cover traces only; reliability is hand-rolled per project. AgentArgus is a +framework-agnostic, single-package answer. + +## What AgentArgus is +A production-grade Python library that wraps *any* agent and uniformly adds +evaluation, observability, reliability, human-in-the-loop, and orchestration +helpers, producing a rich `RunResult` that evaluation consumes. + +## The core design bet (checkpoint 1.1) +**Framework-agnostic wrapping.** We wrap a callable / LangGraph graph / BaseAgent +behind one `Agent` facade. The cost of that generality: we cannot piggyback any +one framework's trace schema, so we define our own (`RunResult` + GenAI-convention +spans), and we forgo framework-specific optimizations. The payoff: one API works +everywhere and nothing is locked to a vendor. + +## Non-goals (defend these) +- **Not a model-serving / inference engine** — that's vLLM/TGI's job. +- **Not a vector DB or a RAG framework** — AgentArgus *evaluates* RAG; it does + not *do* retrieval for you. +- **Not tied to any one agent framework** — LangGraph is *supported*, never + *required*. +- **Not a hosted product in v0.1.0** — it is a library. A dashboard/app is a + *consumer* of it. + +## Key decisions log +See [DESIGN_LOG.md](DESIGN_LOG.md) for the per-module decision record and +[HARD_QUESTIONS.md](HARD_QUESTIONS.md) for the review questions. diff --git a/DESIGN_LOG.md b/DESIGN_LOG.md new file mode 100644 index 0000000..cffc5f5 --- /dev/null +++ b/DESIGN_LOG.md @@ -0,0 +1,121 @@ +# AgentArgus — DESIGN_LOG + +A per-module decision record, written by Claude Code, so the owner learns the +system by reviewing decisions rather than typing every line. Newest entries at +the top. + +--- + +## Module 0 — Core (`RunResult`, config, logging, scaffold) — 2026-07-19 + +### 1. What was built +- **`agentargus/core/results.py`** — the canonical `RunResult` plus its value + objects `Span`, `ToolCall`, `Step`, `ErrorRecord`, `CostBreakdown`. All are + frozen dataclasses. `RunResult` carries `output`, `trace_id`, `spans`, `cost`, + `tool_calls`, `steps`, `errors`, `scores`, `metadata`, plus `with_scores`, + `to_dict`, and `from_dict`. +- **`agentargus/config.py`** — `AgentArgusConfig` (env + kwargs, `from_env`) and + the `Judge` protocol (the LLM-as-judge seam). +- **`agentargus/logging.py`** — `get_logger` factory, `ColorFormatter`, + `JsonFormatter`, `configure_logging`, and `contextvars`-based trace + correlation (`set_trace_id` / `get_trace_id`). +- **`agentargus/__init__.py`** — the (deliberately small) public API surface. +- Scaffolding: `pyproject.toml` (hatchling), `.ruff.toml` (with `T20` to ban + `print()` in library code), `.gitignore`, CI matrix on 3.10/3.11/3.12, + README/LICENSE. + +### 2. Why this shape +- **Deep immutability via tuples, not just `frozen=True`.** Every collection + field is coerced to a `tuple` in `__post_init__`, and every mapping to a + `MappingProxyType`. `frozen=True` alone leaves `result.spans.append(...)` + legal — a real hole. This was decided explicitly with the owner and directly + answers the §11 HARD_QUESTION "is `spans` actually immutable?" with a provable + *yes*. Cost: a caller who wants to build a result incrementally must assemble + the collections first, then construct once. Acceptable — `RunResult` is an + *output* aggregate, not a builder. +- **`with_scores` returns a new object.** Checkpoint 2.1 tempts a mutate-in-place + `scores`. We reversed that in favour of immutability: eval consumers get a + fresh `RunResult`, so a result handed to two evaluators can't be corrupted by + one. `dataclasses.replace` keeps it a one-liner. +- **`Judge` as an injected protocol, no bundled client.** Base install stays + dependency-light (spec §1/§9) and provider-agnostic. A concrete Anthropic + adapter will live behind the `[dev]` extra for tests/demo. Second possible + implementation (proving the abstraction earns its keep, checkpoint 6.1): a + recorded/replay judge for deterministic CI, or a local-model judge. +- **`contextvars` for trace correlation, not a passed-around logger.** The + trace_id is set once by `Agent.run()` and read by the log filter. It survives + async boundaries and keeps trace_id out of every signature (checkpoint 7.1). +- **Raw ANSI over `colorama`.** Modern Windows terminals handle ANSI natively + and we only colorize on a TTY, so the dependency isn't justified. Color is + gated by three independent conditions: config flag AND not `NO_COLOR` AND + `stream.isatty()`. + +### 3. Reuse points introduced +- **`_freeze_mapping`** — the single home for turning a dict into a read-only + mapping; used by every value object. No duplicated freezing logic. +- **`get_logger`** — the *only* sanctioned logger entry point. `logging.getLogger` + is never called elsewhere in library code (enforced socially + by review). +- **`_TraceIdFilter`** — one place that injects trace_id onto records; both + formatters just read `record.trace_id`. +- **`AgentArgusConfig`** — the single home for env access; no module reads + `os.environ` directly. + +### 4. methodoverload decision +- **Not used in Module 0**, correctly. Module 0's inputs are not type-dispatched + — there is no "same operation, different runtime types" site here. Forcing it + would be decorative. Its designated sites are Modules 3, 5, 6 (and cautiously + 1). +- **Studied the library end-to-end before relying on it** (owner's explicit ask: + use it *gracefully*, not forcefully). Read the installed source and PyPI + metadata; wrote a verified reference at `docs/concepts/methodoverload.md`. + Findings that change how we'll use it: + - **Public API is exactly three names**: `overload`, `OverloadedFunction`, + `NoMatchingOverloadError`. The spec's §4 instruction to import `OverloadMeta` + is wrong for v0.1.7 — `OverloadMeta` exists only as an *internal, unexported* + class in `metaclass.py`. We will **not** reach into it; `@overload` alone + handles methods via frame inspection + the `__get__` descriptor, verified. + - **Dispatch is `isinstance`**, subclasses match their base, **no generics**, + **first-match-wins**, and there is a real **subtype ordering trap** (`bool` + is a subclass of `int`: register the most specific type first). All verified + empirically. Documented so every use site orders its overloads correctly. + - **Callables have no distinct `isinstance` class** — confirmed. So + `Agent.wrap` (site #3) overloads on `BaseAgent` only and routes plain + callables through a non-overloaded fallback, exactly per spec §4.3. We record + *why* instead of pretending it dispatched. + +### 4a. Post-review improvements (from HARD_QUESTIONS answers) +The owner's answers to the Module 0 HARD_QUESTIONS proposed four behaviours +better than the first cut. All were implemented before closing the gate: +- **#5** — `configure_logging` now takes a `threading.Lock` and installs the new + handler via an atomic list swap (build-then-swap), so it can never be observed + half-configured even under concurrent misuse. +- **#6** — batched judging is exposed via a `batch_complete(judge, prompts)` + helper that probes for an optional `complete_batch`; the `Judge` protocol keeps + `complete` as its *only* required member so minimal adapters still satisfy + `isinstance` (adding `complete_batch` to the protocol would have broken that — + caught by a failing test and reverted). +- **#7** — `to_dict` now runs `output` and each tool `result` through a + `_jsonable` coercion that tries JSON, then `.to_dict()`, then raises a + `SerializationError` naming the exact field. No silent lossy fallback. +- **#9** — a malformed `AGENTARGUS_COST_CEILING_USD` now raises `ConfigError` + (fail-fast) instead of silently defaulting — correct for a safety limit. +Introduced `agentargus/_internal/exceptions.py` (`AgentArgusError`, +`ConfigError`, `SerializationError`) as the one home for library error types. + +### 5. Failure modes +- If a caller stores a **mutable object inside** an otherwise-frozen field (e.g. + a list as a `metadata` value), the top-level mapping is read-only but the + nested list is still mutable. We freeze one level deep, not recursively. This + is a deliberate, documented boundary (it's a HARD_QUESTION). +- `configure_logging` replaces handlers each call — fine for tests, but two + threads calling it concurrently could race on handler list mutation. Not a + concern in practice (configured once at startup) but noted. +- `from_dict` trusts its input shape; malformed dicts raise `KeyError`/`TypeError` + rather than a friendly error. Acceptable for an internal round-trip helper. + +### 6. The one thing most likely to be asked in review +"You froze the top level and the collections, but a `dict` *value* inside +`metadata` is still mutable. So is `RunResult` immutable or not?" — Answer: it is +immutable at the structural level (you cannot swap or grow any field); it is not +*recursively* deep-frozen for arbitrary nested user data, by design, because +recursively freezing unknown user payloads is expensive and surprising. diff --git a/HARD_QUESTIONS.md b/HARD_QUESTIONS.md new file mode 100644 index 0000000..09994ab --- /dev/null +++ b/HARD_QUESTIONS.md @@ -0,0 +1,56 @@ +# AgentArgus — HARD_QUESTIONS + +Questions a skeptical staff engineer / interviewer would ask about each module. +**Claude Code writes the questions; the owner writes the answers.** A module's +gate does not close until the owner can answer its batch in their own words. + +--- + +## Module 0 — Core (`RunResult`, config, logging) + +1. `RunResult` is `frozen=True` and stores collections as tuples — but a `dict` + *value* nested inside `metadata` is still mutable. Is `RunResult` actually + immutable? Where exactly does the immutability guarantee stop, and why did you + choose to stop it there instead of deep-freezing recursively? + +2. `with_scores` returns a new object instead of mutating. Prove that this + actually prevents a class of bug — give a concrete two-evaluator scenario + where in-place mutation of `scores` would corrupt state, and explain the + memory cost of the copy-on-write approach when `spans` is large. + +3. Trace correlation uses a `contextvars.ContextVar`. What happens to the + trace_id when work is dispatched to a `ThreadPoolExecutor`? Does the child + thread see it? What about an `asyncio.create_task`? Prove you know the + difference and say which one AgentArgus relies on. + +4. You chose raw ANSI codes over `colorama`. On an old `cmd.exe` without VT + processing enabled, what does a colored log line look like? Is that a bug or + acceptable, and what specifically prevents it from happening in practice? + +5. `configure_logging` removes and re-adds handlers on every call. Is that + thread-safe? What breaks if two modules call it concurrently at import time, + and why is (or isn't) that a real risk? + +6. The `Judge` protocol has a single synchronous `complete(prompt) -> str` + method. RAG metrics may need to fire many judge calls per dataset row. Does a + sync-only judge interface bottleneck batch eval? How would you add concurrency + later without breaking the protocol's existing implementers? + +7. `to_dict`/`from_dict` is a hand-written round-trip. `output` and tool + `result` are typed `Any`. What happens when `output` is a non-JSON-serializable + object (e.g. a numpy array or a custom class)? Where does that fail, and is + failing there the right call? + +8. Why is `RunResult` the single coupling point for the whole system? Defend it + against the alternative where each module (tracer, cost, eval) defines its own + result shape. What is the cost of this central aggregate when a future module + needs a field none of the others care about? + +9. `AgentArgusConfig.from_env` silently falls back to defaults on a malformed + `AGENTARGUS_COST_CEILING_USD`. Is silent fallback the right behaviour for a + *cost ceiling* — a safety limit? Argue both sides. + +10. `slots=True` on the dataclasses — what did that buy you, and what did it cost + you (think: pickling, multiple inheritance, adding attributes in + `__post_init__` via `object.__setattr__`)? Why does `object.__setattr__` + still work with slots + frozen? diff --git a/AgentArgus/LICENCE b/LICENSE similarity index 99% rename from AgentArgus/LICENCE rename to LICENSE index 866557d..76b4fbc 100644 --- a/AgentArgus/LICENCE +++ b/LICENSE @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..bf2a2ff --- /dev/null +++ b/README.md @@ -0,0 +1,30 @@ +# AgentArgus + +Framework-agnostic **evaluation, observability, and reliability** for LLM agents — in one package. + +Wrap any agent (a callable, a LangGraph graph, a `BaseAgent`) and get, uniformly: + +- **Evaluation** — RAG metrics (faithfulness, answer relevance, context precision/recall) *and* agent metrics (tool-use accuracy, plan coherence, error-recovery), with regression detection. +- **Observability** — OpenTelemetry traces (GenAI semantic conventions) + accurate token/cost accounting. +- **Reliability** — retry/backoff, model fallback chains, circuit breaker, dead-letter queue. +- **Human-in-the-loop** — approval checkpoints that can pause a run. +- **Orchestration helpers** — supervisor/worker and agent handoff patterns. + +> **Status:** early development (`0.1.0.dev0`). Built module-by-module per the [implementation spec](IMPLEMENTAION.md). See [DESIGN_LOG.md](DESIGN_LOG.md) for the decision record. + +## Install + +```bash +pip install agentargus # minimal runtime +pip install "agentargus[dev]" # + test/lint/judge-adapter tooling +``` + +## Design docs + +- [DESIGN.md](DESIGN.md) — problem, scope, non-goals. +- [DESIGN_LOG.md](DESIGN_LOG.md) — per-module decision log. +- [HARD_QUESTIONS.md](HARD_QUESTIONS.md) — review questions per module. + +## License + +MIT © Mohd Arbaaz Siddiqui diff --git a/agentargus/__init__.py b/agentargus/__init__.py new file mode 100644 index 0000000..7b8f461 --- /dev/null +++ b/agentargus/__init__.py @@ -0,0 +1,45 @@ +"""AgentArgus — evaluation, observability, and reliability for LLM agents. + +The public API surface is intentionally small (spec Appendix A). Module 0 +exposes the core result object, configuration, and the logging factory; later +modules add ``Agent``, ``EvalSuite``, ``EvalRunner``, ``ReliabilityPolicy``, +and the metrics. +""" + +from __future__ import annotations + +from agentargus._internal.exceptions import ( + AgentArgusError, + ConfigError, + SerializationError, +) +from agentargus.config import AgentArgusConfig, Judge, batch_complete +from agentargus.core import ( + CostBreakdown, + ErrorRecord, + RunResult, + Span, + Step, + ToolCall, +) +from agentargus.logging import configure_logging, get_logger + +__version__ = "0.1.0.dev0" + +__all__ = [ + "__version__", + "RunResult", + "Span", + "ToolCall", + "Step", + "ErrorRecord", + "CostBreakdown", + "AgentArgusConfig", + "Judge", + "batch_complete", + "AgentArgusError", + "ConfigError", + "SerializationError", + "get_logger", + "configure_logging", +] diff --git a/agentargus/_internal/__init__.py b/agentargus/_internal/__init__.py new file mode 100644 index 0000000..e35c48b --- /dev/null +++ b/agentargus/_internal/__init__.py @@ -0,0 +1 @@ +"""Private internals. Nothing here is part of the public API.""" diff --git a/agentargus/_internal/exceptions.py b/agentargus/_internal/exceptions.py new file mode 100644 index 0000000..b71a7e8 --- /dev/null +++ b/agentargus/_internal/exceptions.py @@ -0,0 +1,31 @@ +"""Internal exception hierarchy for AgentArgus. + +One home for the library's own error types. ``NoMatchingOverloadError`` is NOT +redefined here — it is imported directly from ``methodoverload`` at its use +sites (spec §9 is explicit about this). +""" + +from __future__ import annotations + +__all__ = ["AgentArgusError", "ConfigError", "SerializationError"] + + +class AgentArgusError(Exception): + """Base class for all AgentArgus-raised errors.""" + + +class ConfigError(AgentArgusError): + """Raised when configuration is invalid — fail fast rather than boot wrong. + + Used for safety-critical settings (e.g. a malformed cost ceiling) where a + silent fallback to a default could let an app run with the wrong limit and + only surface the mistake after money has been spent. + """ + + +class SerializationError(AgentArgusError): + """Raised when a value cannot be serialized, naming the offending field. + + Preferred over a silent lossy fallback (e.g. ``str(obj)``), which would hide + real data loss from the caller. + """ diff --git a/agentargus/config.py b/agentargus/config.py new file mode 100644 index 0000000..df304a6 --- /dev/null +++ b/agentargus/config.py @@ -0,0 +1,118 @@ +"""Central configuration object for AgentArgus. + +``AgentArgusConfig`` is populated from environment variables with explicit +keyword overrides taking precedence. It encapsulates cross-cutting settings so +no module reaches into ``os.environ`` directly — one home for configuration. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import Any, Protocol, runtime_checkable + +from agentargus._internal.exceptions import ConfigError + +__all__ = ["AgentArgusConfig", "Judge", "batch_complete"] + + +@runtime_checkable +class Judge(Protocol): + """The seam for LLM-as-judge evaluation. + + AgentArgus ships **no** provider client in the base package (spec §1/§9 — + framework-agnostic, minimal deps). Metrics that need an LLM depend on this + protocol; the user injects any implementation. A thin Anthropic adapter is + provided behind the ``[dev]`` extra for tests and the demo. + """ + + def complete(self, prompt: str) -> str: + """Return the model's text completion for ``prompt``. + + This is the ONLY required member. An adapter may *optionally* also define + ``complete_batch(prompts: list[str]) -> list[str]`` to parallelize large + eval datasets — but it is intentionally NOT part of this protocol, so + that a minimal ``complete``-only adapter still satisfies ``isinstance`` + checks (HARD_QUESTIONS #6). Call sites use the ``batch_complete`` helper, + which probes for ``complete_batch`` and falls back to looping. + """ + ... + + +def batch_complete(judge: Judge, prompts: list[str]) -> list[str]: + """Complete many prompts via a judge, using ``complete_batch`` if provided. + + This is the single call-site seam for batched judging (one home). If the + adapter implements ``complete_batch`` it is used (potentially concurrent); + otherwise we loop ``complete`` so simple adapters still work. Later modules + (eval) call this rather than deciding batching per metric. + """ + batch = getattr(judge, "complete_batch", None) + if callable(batch): + result = batch(prompts) + # A Protocol member that is present but unimplemented may return the + # Ellipsis sentinel; fall back to the loop in that degenerate case. + if isinstance(result, list): + return result + return [judge.complete(p) for p in prompts] + + +def _env_bool(name: str, default: bool) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +def _env_float_strict(name: str, default: float | None) -> float | None: + """Parse a float env var, or raise ConfigError on a malformed value. + + Fail-fast is deliberate for safety-critical settings like a cost ceiling: a + silent fallback could let an app run with the wrong spend limit and only + surface the mistake after money has been spent (HARD_QUESTIONS #9). + """ + raw = os.environ.get(name) + if raw is None: + return default + try: + return float(raw) + except ValueError as exc: + raise ConfigError( + f"{name} must be a number, got {raw!r}. " + f"Refusing to start with an ambiguous safety limit." + ) from exc + + +@dataclass +class AgentArgusConfig: + """Cross-cutting configuration, resolved from env + explicit kwargs. + + Precedence: an explicitly-passed value wins; otherwise the environment is + consulted via ``from_env``; otherwise a sane default is used. + """ + + judge_model: str = "claude-opus-4-8" + judge: Judge | None = None + cost_ceiling_usd: float | None = None + tracer_exporter: str = "console" # "console" | "otlp" | "memory" + log_level: str = "INFO" + log_color: bool = True + log_json: bool = False + extra: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_env(cls, **overrides: Any) -> AgentArgusConfig: + """Build a config from the environment, then apply explicit overrides.""" + base = cls( + judge_model=os.environ.get("AGENTARGUS_JUDGE_MODEL", "claude-opus-4-8"), + cost_ceiling_usd=_env_float_strict("AGENTARGUS_COST_CEILING_USD", None), + tracer_exporter=os.environ.get("AGENTARGUS_TRACER_EXPORTER", "console"), + log_level=os.environ.get("AGENTARGUS_LOG_LEVEL", "INFO"), + log_color=_env_bool("AGENTARGUS_LOG_COLOR", True), + log_json=_env_bool("AGENTARGUS_LOG_JSON", False), + ) + for key, value in overrides.items(): + if not hasattr(base, key): + raise TypeError(f"Unknown config option: {key!r}") + setattr(base, key, value) + return base diff --git a/agentargus/core/__init__.py b/agentargus/core/__init__.py new file mode 100644 index 0000000..03f7f68 --- /dev/null +++ b/agentargus/core/__init__.py @@ -0,0 +1,19 @@ +"""Core domain objects shared across every AgentArgus module.""" + +from agentargus.core.results import ( + CostBreakdown, + ErrorRecord, + RunResult, + Span, + Step, + ToolCall, +) + +__all__ = [ + "RunResult", + "Span", + "ToolCall", + "Step", + "ErrorRecord", + "CostBreakdown", +] diff --git a/agentargus/core/results.py b/agentargus/core/results.py new file mode 100644 index 0000000..dc0a0c5 --- /dev/null +++ b/agentargus/core/results.py @@ -0,0 +1,266 @@ +"""Core domain objects for AgentArgus. + +This module defines ``RunResult`` — the single canonical data object that is the +spine of the whole system (see spec §2) — and its supporting value objects. + +Design posture (decided in the Module 0 discussion, recorded in DESIGN_LOG): + +* Every object here is a **deeply immutable** frozen dataclass. Collection + fields are stored as ``tuple`` (never ``list``) so a caller cannot mutate + ``result.spans`` in place. This directly answers the HARD_QUESTION + "``spans`` is a list — is it actually immutable?" with a provable *yes*. +* ``RunResult.scores`` is populated *after* evaluation. Rather than mutate the + object in place, ``with_scores`` returns a **new** ``RunResult`` — resolving + checkpoint 2.1 in favour of immutability. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field, replace +from types import MappingProxyType +from typing import Any + +from agentargus._internal.exceptions import SerializationError + +__all__ = [ + "Span", + "ToolCall", + "Step", + "ErrorRecord", + "CostBreakdown", + "RunResult", +] + + +def _freeze_mapping(m: Mapping[str, Any]) -> Mapping[str, Any]: + """Return a read-only view of a mapping so dict fields cannot be mutated.""" + return MappingProxyType(dict(m)) + + +def _jsonable(value: Any, *, field_name: str) -> Any: + """Coerce an arbitrary value into a JSON-serializable form, or fail loudly. + + Strategy (HARD_QUESTIONS #7): try plain JSON first; if that fails, honour a + ``.to_dict()`` method if the object defines one; otherwise raise a + ``SerializationError`` naming the exact field and type. A silent lossy + fallback (e.g. ``str(value)``) is deliberately rejected — it would hide real + data loss from the caller. + """ + try: + json.dumps(value) + return value + except (TypeError, ValueError): + pass + to_dict = getattr(value, "to_dict", None) + if callable(to_dict): + candidate = to_dict() + try: + json.dumps(candidate) + return candidate + except (TypeError, ValueError): + pass + raise SerializationError( + f"Field {field_name!r} holds a value of type " + f"{type(value).__name__!r} that is not JSON-serializable and has no " + f"usable .to_dict(). Convert it before building the RunResult." + ) + + +@dataclass(frozen=True, slots=True) +class Span: + """A single structured execution step, emitted by the tracer (spec §2).""" + + name: str + span_id: str + start_time: float + end_time: float + attributes: Mapping[str, Any] = field(default_factory=dict) + parent_id: str | None = None + + def __post_init__(self) -> None: + # Freeze the mapping so ``span.attributes["x"] = ...`` raises. + object.__setattr__(self, "attributes", _freeze_mapping(self.attributes)) + + @property + def duration(self) -> float: + return self.end_time - self.start_time + + +@dataclass(frozen=True, slots=True) +class ToolCall: + """A single tool invocation: name, args, result, success, latency (spec §2).""" + + name: str + args: Mapping[str, Any] + result: Any + success: bool + latency: float + error: str | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "args", _freeze_mapping(self.args)) + + +@dataclass(frozen=True, slots=True) +class Step: + """An ordered reasoning/action step, used by plan-coherence eval (spec §2).""" + + index: int + kind: str # e.g. "reason" | "action" | "observation" + content: str + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "metadata", _freeze_mapping(self.metadata)) + + +@dataclass(frozen=True, slots=True) +class ErrorRecord: + """Anything caught by the reliability layer (spec §2).""" + + error_type: str + message: str + recovered: bool + attempt: int = 0 + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "metadata", _freeze_mapping(self.metadata)) + + +@dataclass(frozen=True, slots=True) +class CostBreakdown: + """Token + dollar accounting for a run (produced by CostTracker, spec §2).""" + + input_tokens: int = 0 + output_tokens: int = 0 + input_cost: float = 0.0 + output_cost: float = 0.0 + + @property + def total_tokens(self) -> int: + return self.input_tokens + self.output_tokens + + @property + def total_cost(self) -> float: + return self.input_cost + self.output_cost + + def __add__(self, other: CostBreakdown) -> CostBreakdown: + if not isinstance(other, CostBreakdown): + return NotImplemented + return CostBreakdown( + input_tokens=self.input_tokens + other.input_tokens, + output_tokens=self.output_tokens + other.output_tokens, + input_cost=self.input_cost + other.input_cost, + output_cost=self.output_cost + other.output_cost, + ) + + +@dataclass(frozen=True, slots=True) +class RunResult: + """The canonical result of one wrapped agent run — the spine of AgentArgus. + + Collection fields are stored as tuples for genuine immutability. Use the + typed sequence accessors; the underlying storage is never a mutable list. + ``scores`` starts empty and is filled by ``with_scores`` after eval, which + returns a *new* object rather than mutating this one. + """ + + output: Any + trace_id: str + spans: Sequence[Span] = () + cost: CostBreakdown = field(default_factory=CostBreakdown) + tool_calls: Sequence[ToolCall] = () + steps: Sequence[Step] = () + errors: Sequence[ErrorRecord] = () + scores: Mapping[str, float] = field(default_factory=dict) + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + # Coerce every collection to an immutable form, regardless of what the + # caller passed (list, generator, tuple). One home for the coercion. + object.__setattr__(self, "spans", tuple(self.spans)) + object.__setattr__(self, "tool_calls", tuple(self.tool_calls)) + object.__setattr__(self, "steps", tuple(self.steps)) + object.__setattr__(self, "errors", tuple(self.errors)) + object.__setattr__(self, "scores", _freeze_mapping(self.scores)) + object.__setattr__(self, "metadata", _freeze_mapping(self.metadata)) + + def with_scores(self, scores: Mapping[str, float]) -> RunResult: + """Return a NEW RunResult with ``scores`` merged in (immutability).""" + merged = {**self.scores, **scores} + return replace(self, scores=merged) + + def to_dict(self) -> dict[str, Any]: + """Serialize to a plain dict (JSON-friendly). Round-trips via from_dict.""" + return { + "output": _jsonable(self.output, field_name="output"), + "trace_id": self.trace_id, + "spans": [ + { + "name": s.name, + "span_id": s.span_id, + "start_time": s.start_time, + "end_time": s.end_time, + "attributes": dict(s.attributes), + "parent_id": s.parent_id, + } + for s in self.spans + ], + "cost": { + "input_tokens": self.cost.input_tokens, + "output_tokens": self.cost.output_tokens, + "input_cost": self.cost.input_cost, + "output_cost": self.cost.output_cost, + }, + "tool_calls": [ + { + "name": t.name, + "args": dict(t.args), + "result": _jsonable(t.result, field_name=f"tool_calls[{i}].result"), + "success": t.success, + "latency": t.latency, + "error": t.error, + } + for i, t in enumerate(self.tool_calls) + ], + "steps": [ + { + "index": st.index, + "kind": st.kind, + "content": st.content, + "metadata": dict(st.metadata), + } + for st in self.steps + ], + "errors": [ + { + "error_type": e.error_type, + "message": e.message, + "recovered": e.recovered, + "attempt": e.attempt, + "metadata": dict(e.metadata), + } + for e in self.errors + ], + "scores": dict(self.scores), + "metadata": dict(self.metadata), + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> RunResult: + """Reconstruct a RunResult from ``to_dict`` output.""" + cost_data = data.get("cost", {}) + return cls( + output=data["output"], + trace_id=data["trace_id"], + spans=tuple(Span(**s) for s in data.get("spans", [])), + cost=CostBreakdown(**cost_data), + tool_calls=tuple(ToolCall(**t) for t in data.get("tool_calls", [])), + steps=tuple(Step(**st) for st in data.get("steps", [])), + errors=tuple(ErrorRecord(**e) for e in data.get("errors", [])), + scores=dict(data.get("scores", {})), + metadata=dict(data.get("metadata", {})), + ) diff --git a/agentargus/logging.py b/agentargus/logging.py new file mode 100644 index 0000000..2e173c9 --- /dev/null +++ b/agentargus/logging.py @@ -0,0 +1,168 @@ +"""The AgentArgus logging system (spec §7). + +Goals: + * One factory ``get_logger(name)`` — never call ``logging.getLogger`` + elsewhere (single home / reuse point). + * Colorized human output for dev, level-based colors, auto-disabled when + output is not a TTY, when ``NO_COLOR`` is set, or via config. + * A JSON formatter for production, selectable via config. + * Every log line inside a run carries the ``trace_id``, propagated through a + ``contextvars.ContextVar`` (survives async boundaries; keeps trace_id out + of every function signature). This is the logs<->traces bridge. + +Color strategy: raw ANSI codes, no dependency. ``colorama`` was considered for +Windows but modern Windows terminals (Windows Terminal, VS Code, PS 7) support +ANSI natively, and we only emit color when attached to a TTY anyway — so the +extra dependency is not worth it. Recorded in DESIGN_LOG. +""" + +from __future__ import annotations + +import json +import logging +import os +import sys +import threading +from contextvars import ContextVar +from typing import Any + +__all__ = ["get_logger", "configure_logging", "set_trace_id", "get_trace_id"] + +# --------------------------------------------------------------------------- # +# Trace correlation via contextvars +# --------------------------------------------------------------------------- # +_trace_id_var: ContextVar[str | None] = ContextVar("agentargus_trace_id", default=None) + + +def set_trace_id(trace_id: str | None) -> object: + """Bind ``trace_id`` for the current context; return a reset token.""" + return _trace_id_var.set(trace_id) + + +def get_trace_id() -> str | None: + """Return the trace_id bound to the current context, if any.""" + return _trace_id_var.get() + + +class _TraceIdFilter(logging.Filter): + """Injects the current context's trace_id onto every record.""" + + def filter(self, record: logging.LogRecord) -> bool: + record.trace_id = _trace_id_var.get() + return True + + +# --------------------------------------------------------------------------- # +# Formatters +# --------------------------------------------------------------------------- # +_RESET = "\033[0m" +_LEVEL_COLORS = { + logging.DEBUG: "\033[90m", # grey + logging.INFO: "\033[32m", # green + logging.WARNING: "\033[33m", # yellow + logging.ERROR: "\033[31m", # red + logging.CRITICAL: "\033[1;31m", # bold red +} + + +class ColorFormatter(logging.Formatter): + """Human-readable formatter with optional level-based ANSI color.""" + + def __init__(self, use_color: bool) -> None: + super().__init__() + self._use_color = use_color + + def format(self, record: logging.LogRecord) -> str: + trace_id = getattr(record, "trace_id", None) + trace_part = f" [{trace_id[:8]}]" if trace_id else "" + base = ( + f"{self.formatTime(record, '%H:%M:%S')} " + f"{record.levelname:<8}{trace_part} " + f"{record.name}: {record.getMessage()}" + ) + if record.exc_info: + base += "\n" + self.formatException(record.exc_info) + if self._use_color: + color = _LEVEL_COLORS.get(record.levelno, "") + return f"{color}{base}{_RESET}" + return base + + +class JsonFormatter(logging.Formatter): + """Machine-parseable structured formatter for production.""" + + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, Any] = { + "timestamp": self.formatTime(record, "%Y-%m-%dT%H:%M:%S"), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + "trace_id": getattr(record, "trace_id", None), + } + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + return json.dumps(payload) + + +# --------------------------------------------------------------------------- # +# Factory / configuration +# --------------------------------------------------------------------------- # +_ROOT_NAME = "agentargus" +_configured = False +_config_lock = threading.Lock() + + +def _should_use_color(color_flag: bool, stream: Any) -> bool: + """Color only when requested AND NO_COLOR unset AND the stream is a TTY.""" + if not color_flag: + return False + if os.environ.get("NO_COLOR") is not None: + return False + return bool(getattr(stream, "isatty", lambda: False)()) + + +def configure_logging( + level: str = "INFO", + *, + color: bool = True, + json_format: bool = False, + stream: Any | None = None, +) -> None: + """Configure the ``agentargus`` root logger. + + The contract is "call once at startup". A lock plus build-then-swap makes + even a concurrent misuse safe: the fully-built handler is installed + atomically, so logging can never be observed half-configured + (HARD_QUESTIONS #5). + """ + global _configured + stream = stream if stream is not None else sys.stderr + + # Build the new handler completely BEFORE touching the logger. + handler = logging.StreamHandler(stream) + handler.addFilter(_TraceIdFilter()) + if json_format: + handler.setFormatter(JsonFormatter()) + else: + handler.setFormatter(ColorFormatter(_should_use_color(color, stream))) + + with _config_lock: + logger = logging.getLogger(_ROOT_NAME) + logger.setLevel(level.upper()) + logger.propagate = False + # Atomic swap: replace the handler list in one assignment rather than + # remove-then-add as two separately-visible steps. + logger.handlers = [handler] + _configured = True + + +def get_logger(name: str | None = None) -> logging.Logger: + """Return a namespaced child of the ``agentargus`` logger. + + This is the ONLY sanctioned way to obtain a logger in library code. + """ + if not _configured: + configure_logging() + if name is None or name == _ROOT_NAME: + return logging.getLogger(_ROOT_NAME) + return logging.getLogger(f"{_ROOT_NAME}.{name}") diff --git a/docs/concepts/methodoverload.md b/docs/concepts/methodoverload.md new file mode 100644 index 0000000..1e9a538 --- /dev/null +++ b/docs/concepts/methodoverload.md @@ -0,0 +1,104 @@ +# Using `methodoverload` in AgentArgus (verified reference) + +`methodoverload` is the owner's own library (`pip install methodoverload`, +**v0.1.7**, released 2026-01-20). This page records the **verified** API and +dispatch behaviour — read from the installed source and confirmed empirically — +so AgentArgus uses it *gracefully* (where type-dispatch is genuinely cleaner) +rather than *forcefully*. It supersedes the spec's §4, which is out of date. + +Repo: · Summary: "Runtime function and +method overloading for Python." + +## The real public API (exactly three names) + +```python +from methodoverload import overload, OverloadedFunction, NoMatchingOverloadError +``` + +That is the entire exported surface (`methodoverload.__all__`). **There is no +`OverloadMeta` in the public API.** A metaclass *does* exist in the package +(`methodoverload/metaclass.py`) as an internal alternative, but it is **not +exported and not documented** — so AgentArgus does not use it. The spec's +instruction to `from methodoverload import overload, OverloadMeta` is wrong for +v0.1.7; importing `OverloadMeta` from the top level raises `ImportError`. + +## How overloading works + +`@overload` decorates multiple same-named callables. At definition time it uses +**frame inspection** (`inspect.currentframe().f_back.f_locals`) to find the +sibling of the same name in the *defining namespace* and merge them into one +`OverloadedFunction` dispatcher. This works identically at module scope and +**inside a class body** — which is precisely why no metaclass is needed for +instance/class/static methods. + +### Free functions + +```python +@overload +def add(a: int, b: int): + return a + b + +@overload +def add(a: str, b: str): + return f"{a} {b}" + +add(1, 2) # 3 +add("Hello", "World") # "Hello World" +``` + +### Instance methods — no metaclass required (verified) + +```python +class EvalDataset: + @overload + def load(self, source: str): ... # path + @overload + def load(self, source: list): ... # in-memory records + @overload + def load(self, source: dict): ... # single record +``` + +`OverloadedFunction.__get__` (the descriptor protocol) binds `self`/`cls` +correctly. `classmethod`/`staticmethod` are supported by placing `@overload` +**outermost**, then `@classmethod`/`@staticmethod`. + +## Dispatch semantics (the rules we must design around) + +1. **Dispatch is `isinstance(value, annotation)`** on each annotated parameter. + Parameters named `self`/`cls` are skipped; unannotated parameters are skipped. +2. **Subclasses match their base** — `wrap(inner: BaseAgent)` matches any + `BaseAgent` subclass. (Verified.) This is why site #3 can dispatch on + `BaseAgent`. +3. **No generic types.** `list[int]` vs `list[str]` cannot be told apart — only + the bare origin (`list`) participates in `isinstance`. Never annotate an + overload with a parameterized generic and expect dispatch on the parameter. +4. **First matching overload wins**, in definition order. Combined with rule 2 + this creates a **subtype ordering trap**: because `bool` is a subclass of + `int`, if `int` is registered before `bool`, calling with `True` resolves to + the `int` overload. (Verified: it returned `'int'`.) **Rule for AgentArgus: + register the most specific type first.** +5. **`NoMatchingOverloadError`** is raised when nothing matches. Catch it + explicitly where a fallback is offered. It subclasses an internal + `OverloadError`; import it from `methodoverload`, do **not** redefine it + (spec §9 agrees). +6. **Caching:** `OverloadedFunction` memoises resolutions by `(name, args, + kwargs)`. Fine for our sites; be wary in hot inner loops (spec §4 "measure"). + +## Where AgentArgus uses it (and why it fits) + +| Site | Method | Dispatch types | Why overload beats an `isinstance` ladder | +|------|--------|----------------|--------------------------------------------| +| #1 | `EvalDataset.load(source)` | `str` / `list` / `dict` | Three genuinely distinct runtime shapes; open/closed — add a new source type by adding an overload, not editing a branch. | +| #2 | `CostTracker.add_usage(usage)` | `dict` / `Usage` / provider response | Different callers pass different shapes; removes a growing `isinstance` chain. | +| #3 | `Agent.wrap(inner)` | `BaseAgent` (+ fallback) | Subclass dispatch is clean for `BaseAgent`; **callables have no distinct `isinstance` class**, so the plain-callable case is a documented fallback, not an overload (see below). | +| #4 | `Metric.compute(x)` | `RunResult` / `dict` | Lets metrics accept a full result in prod and a lightweight dict trace in unit tests. | + +### The honest non-fit: callables (site #3 caveat) + +There is no clean `isinstance` class for "an arbitrary callable" (`function`, +`lambda`, a class with `__call__`, a bound method are all just `object`). +Verified: you cannot write `wrap(inner: Callable)` and have it dispatch away +from `object`. So `Agent.wrap` overloads on `BaseAgent` only, and everything +else falls through to a single non-overloaded path that wraps the callable +directly. This is exactly the "do not force methodoverload where it doesn't fit" +guidance in spec §4.3 — and we record *why* rather than pretending it dispatched. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..151a24e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,66 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "agentargus" +version = "0.1.0.dev0" +description = "Framework-agnostic evaluation, observability, and reliability for LLM agents." +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "Mohd Arbaaz Siddiqui", email = "arbaazcode@gmail.com" }] +keywords = ["llm", "agents", "evaluation", "observability", "reliability"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dependencies = [ + "opentelemetry-api>=1.20", + "opentelemetry-sdk>=1.20", + "jinja2>=3.0", + "methodoverload>=0.1.7", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4", + "pytest-cov>=4.1", + "pytest-asyncio>=0.23", + "pytest-recording>=0.13", + "ruff>=0.4", + "mypy>=1.8", + "build>=1.0", + "twine>=5.0", + "anthropic>=0.34", +] +langgraph = ["langgraph>=0.2"] +docs = ["pdoc>=14.0"] + +[project.urls] +Homepage = "https://github.com/mohdcodes/agentargus" +Repository = "https://github.com/mohdcodes/agentargus" + +[tool.hatch.build.targets.wheel] +packages = ["agentargus"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" +addopts = "-ra" + +[tool.coverage.run] +source = ["agentargus"] +branch = true + +[tool.mypy] +python_version = "3.10" +strict = true +warn_unused_configs = true +disallow_untyped_defs = true +ignore_missing_imports = true diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..4db9681 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,31 @@ +"""Shared pytest fixtures for AgentArgus tests.""" + +from __future__ import annotations + +import pytest +from agentargus.core import CostBreakdown, ErrorRecord, RunResult, Span, Step, ToolCall + + +@pytest.fixture +def sample_run_result() -> RunResult: + """A representative RunResult populated across every field.""" + return RunResult( + output="the answer is 42", + trace_id="trace-abc-123", + spans=[ + Span( + name="agent.run", + span_id="s1", + start_time=0.0, + end_time=1.5, + attributes={"gen_ai.system": "anthropic"}, + ) + ], + cost=CostBreakdown(input_tokens=100, output_tokens=50, input_cost=0.001, output_cost=0.002), + tool_calls=[ + ToolCall(name="search", args={"q": "meaning"}, result="42", success=True, latency=0.3) + ], + steps=[Step(index=0, kind="reason", content="think about it")], + errors=[ErrorRecord(error_type="TimeoutError", message="slow", recovered=True, attempt=1)], + metadata={"user": "test"}, + ) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..dabbe6e --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,86 @@ +"""Tests for AgentArgusConfig (env resolution + overrides + Judge protocol).""" + +from __future__ import annotations + +import pytest +from agentargus._internal.exceptions import ConfigError +from agentargus.config import AgentArgusConfig, Judge, batch_complete + + +class TestFromEnv: + def test_defaults_when_env_empty(self, monkeypatch: pytest.MonkeyPatch) -> None: + for k in list(dict(__import__("os").environ)): + if k.startswith("AGENTARGUS_"): + monkeypatch.delenv(k, raising=False) + cfg = AgentArgusConfig.from_env() + assert cfg.judge_model == "claude-opus-4-8" + assert cfg.cost_ceiling_usd is None + assert cfg.log_level == "INFO" + assert cfg.log_color is True + + def test_reads_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AGENTARGUS_JUDGE_MODEL", "claude-sonnet-5") + monkeypatch.setenv("AGENTARGUS_COST_CEILING_USD", "1.5") + monkeypatch.setenv("AGENTARGUS_LOG_COLOR", "false") + monkeypatch.setenv("AGENTARGUS_LOG_JSON", "true") + cfg = AgentArgusConfig.from_env() + assert cfg.judge_model == "claude-sonnet-5" + assert cfg.cost_ceiling_usd == 1.5 + assert cfg.log_color is False + assert cfg.log_json is True + + def test_overrides_win_over_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AGENTARGUS_JUDGE_MODEL", "from-env") + cfg = AgentArgusConfig.from_env(judge_model="explicit") + assert cfg.judge_model == "explicit" + + def test_unknown_override_raises(self) -> None: + with pytest.raises(TypeError): + AgentArgusConfig.from_env(nonexistent=True) + + def test_malformed_cost_ceiling_fails_fast(self, monkeypatch: pytest.MonkeyPatch) -> None: + # #9: a safety limit must not silently fall back to a default. + monkeypatch.setenv("AGENTARGUS_COST_CEILING_USD", "not-a-number") + with pytest.raises(ConfigError): + AgentArgusConfig.from_env() + + +class TestJudgeProtocol: + def test_duck_typed_implementation_satisfies_protocol(self) -> None: + class FakeJudge: + def complete(self, prompt: str) -> str: + return "ok" + + assert isinstance(FakeJudge(), Judge) + + def test_missing_method_fails_protocol(self) -> None: + class NotAJudge: + pass + + assert not isinstance(NotAJudge(), Judge) + + +class TestBatchComplete: + def test_falls_back_to_loop_without_complete_batch(self) -> None: + # #6: simple adapters without complete_batch still work via the loop. + calls: list[str] = [] + + class SimpleJudge: + def complete(self, prompt: str) -> str: + calls.append(prompt) + return prompt.upper() + + out = batch_complete(SimpleJudge(), ["a", "b"]) + assert out == ["A", "B"] + assert calls == ["a", "b"] + + def test_uses_complete_batch_when_available(self) -> None: + class BatchingJudge: + def complete(self, prompt: str) -> str: # pragma: no cover - not used + raise AssertionError("should not be called") + + def complete_batch(self, prompts: list[str]) -> list[str]: + return [p * 2 for p in prompts] + + out = batch_complete(BatchingJudge(), ["x", "y"]) + assert out == ["xx", "yy"] diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py new file mode 100644 index 0000000..4884b22 --- /dev/null +++ b/tests/unit/test_logging.py @@ -0,0 +1,124 @@ +"""Tests for the logging system (spec §7: color gating, JSON, trace correlation).""" + +from __future__ import annotations + +import io +import json +import logging + +import pytest +from agentargus.logging import ( + ColorFormatter, + JsonFormatter, + _should_use_color, + configure_logging, + get_logger, + get_trace_id, + set_trace_id, +) + + +class _FakeTTY(io.StringIO): + def isatty(self) -> bool: + return True + + +class TestColorGating: + def test_no_color_when_not_tty(self) -> None: + assert _should_use_color(True, io.StringIO()) is False + + def test_color_when_tty(self) -> None: + assert _should_use_color(True, _FakeTTY()) is True + + def test_no_color_env_wins(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NO_COLOR", "1") + assert _should_use_color(True, _FakeTTY()) is False + + def test_flag_off_disables(self) -> None: + assert _should_use_color(False, _FakeTTY()) is False + + +class TestFormatters: + def _record(self, msg: str = "hello", level: int = logging.INFO) -> logging.LogRecord: + rec = logging.LogRecord("agentargus.test", level, __file__, 1, msg, None, None) + rec.trace_id = "trace-1234567890" + return rec + + def test_color_formatter_includes_ansi_when_enabled(self) -> None: + out = ColorFormatter(use_color=True).format(self._record()) + assert "\033[" in out + assert "trace-12" in out # first 8 chars of trace_id + + def test_color_formatter_plain_when_disabled(self) -> None: + out = ColorFormatter(use_color=False).format(self._record()) + assert "\033[" not in out + + def test_json_formatter_is_valid_json(self) -> None: + out = JsonFormatter().format(self._record()) + parsed = json.loads(out) + assert parsed["level"] == "INFO" + assert parsed["trace_id"] == "trace-1234567890" + assert parsed["message"] == "hello" + + +class TestTraceCorrelation: + def test_set_and_get(self) -> None: + token = set_trace_id("abc") + try: + assert get_trace_id() == "abc" + finally: + import agentargus.logging as lg + + lg._trace_id_var.reset(token) + + def test_trace_id_appears_in_output(self) -> None: + stream = io.StringIO() + configure_logging(level="DEBUG", color=False, json_format=True, stream=stream) + token = set_trace_id("corr-999") + try: + get_logger("x").info("with trace") + finally: + import agentargus.logging as lg + + lg._trace_id_var.reset(token) + line = stream.getvalue().strip().splitlines()[-1] + assert json.loads(line)["trace_id"] == "corr-999" + + +class TestFactory: + def test_namespaced(self) -> None: + assert get_logger("agents").name == "agentargus.agents" + + def test_root(self) -> None: + assert get_logger().name == "agentargus" + + def test_concurrent_configure_never_half_configured(self) -> None: + # #5: lock + atomic swap => the handler list is never observed empty. + import threading + + stream = io.StringIO() + errors: list[str] = [] + + def worker() -> None: + for _ in range(50): + configure_logging(level="INFO", color=False, stream=stream) + handlers = logging.getLogger("agentargus").handlers + if len(handlers) != 1: + errors.append(f"saw {len(handlers)} handlers") + + threads = [threading.Thread(target=worker) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + assert errors == [] + + def test_levels_emit(self) -> None: + stream = io.StringIO() + configure_logging(level="WARNING", color=False, stream=stream) + log = get_logger("lvl") + log.info("should not appear") + log.warning("should appear") + output = stream.getvalue() + assert "should not appear" not in output + assert "should appear" in output diff --git a/tests/unit/test_results.py b/tests/unit/test_results.py new file mode 100644 index 0000000..cb68890 --- /dev/null +++ b/tests/unit/test_results.py @@ -0,0 +1,103 @@ +"""Tests for RunResult and supporting dataclasses (spec §8: immutability, round-trip).""" + +from __future__ import annotations + +import dataclasses + +import pytest +from agentargus._internal.exceptions import SerializationError +from agentargus.core import CostBreakdown, RunResult, Span, ToolCall + + +class TestImmutability: + def test_top_level_frozen(self, sample_run_result: RunResult) -> None: + with pytest.raises(dataclasses.FrozenInstanceError): + sample_run_result.output = "changed" # type: ignore[misc] + + def test_collections_are_tuples_not_lists(self, sample_run_result: RunResult) -> None: + # Passed as lists in the fixture; stored as tuples => cannot .append(). + assert isinstance(sample_run_result.spans, tuple) + assert isinstance(sample_run_result.tool_calls, tuple) + assert isinstance(sample_run_result.steps, tuple) + assert isinstance(sample_run_result.errors, tuple) + with pytest.raises(AttributeError): + sample_run_result.spans.append(None) # type: ignore[attr-defined] + + def test_span_attributes_are_read_only(self) -> None: + span = Span(name="x", span_id="s", start_time=0.0, end_time=1.0, attributes={"a": 1}) + with pytest.raises(TypeError): + span.attributes["a"] = 2 # type: ignore[index] + + def test_scores_mapping_read_only(self, sample_run_result: RunResult) -> None: + with pytest.raises(TypeError): + sample_run_result.scores["x"] = 1.0 # type: ignore[index] + + +class TestWithScores: + def test_returns_new_object(self, sample_run_result: RunResult) -> None: + updated = sample_run_result.with_scores({"faithfulness": 0.9}) + assert updated is not sample_run_result + assert updated.scores["faithfulness"] == 0.9 + + def test_original_unchanged(self, sample_run_result: RunResult) -> None: + _ = sample_run_result.with_scores({"faithfulness": 0.9}) + assert "faithfulness" not in sample_run_result.scores + + def test_merges_with_existing(self, sample_run_result: RunResult) -> None: + first = sample_run_result.with_scores({"a": 1.0}) + second = first.with_scores({"b": 2.0}) + assert second.scores == {"a": 1.0, "b": 2.0} + + +class TestSerialization: + def test_round_trip(self, sample_run_result: RunResult) -> None: + data = sample_run_result.to_dict() + restored = RunResult.from_dict(data) + assert restored.output == sample_run_result.output + assert restored.trace_id == sample_run_result.trace_id + assert restored.cost.total_tokens == sample_run_result.cost.total_tokens + assert restored.tool_calls[0].name == "search" + assert restored.spans[0].attributes["gen_ai.system"] == "anthropic" + assert restored.errors[0].recovered is True + + def test_round_trip_is_json_safe(self, sample_run_result: RunResult) -> None: + import json + + json.dumps(sample_run_result.to_dict()) # must not raise + + def test_unserializable_output_raises_named_error(self) -> None: + # #7: fail loudly, naming the field, rather than silently mangling. + rr = RunResult(output=object(), trace_id="t") + with pytest.raises(SerializationError, match="output"): + rr.to_dict() + + def test_output_with_to_dict_is_honoured(self) -> None: + class Custom: + def to_dict(self) -> dict[str, int]: + return {"n": 1} + + rr = RunResult(output=Custom(), trace_id="t") + assert rr.to_dict()["output"] == {"n": 1} + + def test_unserializable_tool_result_names_index(self) -> None: + rr = RunResult( + output="ok", + trace_id="t", + tool_calls=[ToolCall(name="f", args={}, result=object(), success=True, latency=0.1)], + ) + with pytest.raises(SerializationError, match=r"tool_calls\[0\]\.result"): + rr.to_dict() + + +class TestCostBreakdown: + def test_totals(self) -> None: + c = CostBreakdown(input_tokens=10, output_tokens=5, input_cost=0.1, output_cost=0.2) + assert c.total_tokens == 15 + assert c.total_cost == pytest.approx(0.3) + + def test_add(self) -> None: + a = CostBreakdown(input_tokens=10, input_cost=0.1) + b = CostBreakdown(output_tokens=5, output_cost=0.2) + total = a + b + assert total.total_tokens == 15 + assert total.total_cost == pytest.approx(0.3)