From 7fa6f218be3f2eabb5defc5ea6b7dd8079570923 Mon Sep 17 00:00:00 2001 From: Mohd Arbaaz Siddiqui Date: Sun, 19 Jul 2026 15:12:55 +0530 Subject: [PATCH 1/4] Module 0: scaffold + RunResult, config, logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - RunResult and value objects: deep-immutable frozen dataclasses (tuples + read-only mappings); with_scores returns a new object. - AgentArgusConfig (env + kwargs) and injectable Judge protocol (no bundled provider client). - Logging: get_logger factory, color + JSON formatters, contextvars trace correlation, TTY/NO_COLOR gating, print() banned via ruff T20. - Tooling: hatchling pyproject, ruff, mypy strict, CI matrix 3.10/3.11/3.12. - 30 tests, 96% coverage; ruff + mypy clean. - DESIGN_LOG + HARD_QUESTIONS for Module 0. NOTE: methodoverload 0.1.7 has no OverloadMeta (spec §4 out of date); @overload works on methods with no metaclass. Documented in DESIGN_LOG. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 48 ++++ .gitignore | 37 +++ .ruff.toml | 25 ++ DESIGN.md | 36 +++ DESIGN_LOG.md | 91 ++++++ HARD_QUESTIONS.md | 56 ++++ IMPLEMENTAION.md | 456 +++++++++++++++++++++++++++++++ LICENSE | 21 ++ README.md | 30 ++ agentargus/__init__.py | 36 +++ agentargus/_internal/__init__.py | 1 + agentargus/config.py | 81 ++++++ agentargus/core/__init__.py | 19 ++ agentargus/core/results.py | 234 ++++++++++++++++ agentargus/logging.py | 159 +++++++++++ pyproject.toml | 66 +++++ tests/conftest.py | 31 +++ tests/unit/test_config.py | 58 ++++ tests/unit/test_logging.py | 103 +++++++ tests/unit/test_results.py | 79 ++++++ 20 files changed, 1667 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 .ruff.toml create mode 100644 DESIGN.md create mode 100644 DESIGN_LOG.md create mode 100644 HARD_QUESTIONS.md create mode 100644 IMPLEMENTAION.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 agentargus/__init__.py create mode 100644 agentargus/_internal/__init__.py create mode 100644 agentargus/config.py create mode 100644 agentargus/core/__init__.py create mode 100644 agentargus/core/results.py create mode 100644 agentargus/logging.py create mode 100644 pyproject.toml create mode 100644 tests/conftest.py create mode 100644 tests/unit/test_config.py create mode 100644 tests/unit/test_logging.py create mode 100644 tests/unit/test_results.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2f35942 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + - name: Ruff lint + run: ruff check . + - name: Ruff format check + run: ruff format --check . + - name: Mypy + run: mypy agentargus + - name: Pytest + run: pytest --cov=agentargus --cov-report=xml --cov-report=term-missing + + e2e: + # End-to-end demo runs separately and does not block the merge gate (spec §10). + runs-on: ubuntu-latest + needs: test + if: github.event_name == 'push' + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install + run: pip install -e ".[dev]" + - name: Run demo (placeholder until Module 10) + run: echo "e2e demo lands in Module 10" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e121450 --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.eggs/ +build/ +dist/ +*.egg + +# Virtual envs +.venv/ +venv/ +env/ + +# Testing / coverage +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +htmlcov/ +.coverage +coverage.xml + +# Secrets & local config +.env +.env.* +!.env.example + +# Editors / OS +.vscode/ +.idea/ +.DS_Store +Thumbs.db + +# Project artifacts +*.sqlite +*.sqlite3 +dead_letter*.jsonl diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000..9459cfa --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,25 @@ +target-version = "py310" +line-length = 100 +src = ["agentargus", "tests"] + +[lint] +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort + "W", # pycodestyle warnings + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # comprehensions + "SIM", # simplify + "T20", # flake8-print -> bans print() in library code (§7) +] + +[lint.per-file-ignores] +# Examples and CLI are allowed to print; tests may too. +"examples/**" = ["T20"] +"tests/**" = ["T20"] +"benchmarks/**" = ["T20"] + +[format] +quote-style = "double" diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 0000000..e6a5dd0 --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,36 @@ +# AgentArgus — Design, Scope, Non-Goals + +> Owner-authored document. Claude Code seeds it from spec §1; the owner refines +> and defends each point. + +## Problem +Teams ship LLM agents to production with almost no operational rigour. There is +no single place to answer: *Did the agent do the right thing? What did it cost? +Why did it fail? Can it recover?* RAGAS covers RAG eval only; observability +tools cover traces only; reliability is hand-rolled per project. AgentArgus is a +framework-agnostic, single-package answer. + +## What AgentArgus is +A production-grade Python library that wraps *any* agent and uniformly adds +evaluation, observability, reliability, human-in-the-loop, and orchestration +helpers, producing a rich `RunResult` that evaluation consumes. + +## The core design bet (checkpoint 1.1) +**Framework-agnostic wrapping.** We wrap a callable / LangGraph graph / BaseAgent +behind one `Agent` facade. The cost of that generality: we cannot piggyback any +one framework's trace schema, so we define our own (`RunResult` + GenAI-convention +spans), and we forgo framework-specific optimizations. The payoff: one API works +everywhere and nothing is locked to a vendor. + +## Non-goals (defend these) +- **Not a model-serving / inference engine** — that's vLLM/TGI's job. +- **Not a vector DB or a RAG framework** — AgentArgus *evaluates* RAG; it does + not *do* retrieval for you. +- **Not tied to any one agent framework** — LangGraph is *supported*, never + *required*. +- **Not a hosted product in v0.1.0** — it is a library. A dashboard/app is a + *consumer* of it. + +## Key decisions log +See [DESIGN_LOG.md](DESIGN_LOG.md) for the per-module decision record and +[HARD_QUESTIONS.md](HARD_QUESTIONS.md) for the review questions. diff --git a/DESIGN_LOG.md b/DESIGN_LOG.md new file mode 100644 index 0000000..8280ed6 --- /dev/null +++ b/DESIGN_LOG.md @@ -0,0 +1,91 @@ +# AgentArgus — DESIGN_LOG + +A per-module decision record, written by Claude Code, so the owner learns the +system by reviewing decisions rather than typing every line. Newest entries at +the top. + +--- + +## Module 0 — Core (`RunResult`, config, logging, scaffold) — 2026-07-19 + +### 1. What was built +- **`agentargus/core/results.py`** — the canonical `RunResult` plus its value + objects `Span`, `ToolCall`, `Step`, `ErrorRecord`, `CostBreakdown`. All are + frozen dataclasses. `RunResult` carries `output`, `trace_id`, `spans`, `cost`, + `tool_calls`, `steps`, `errors`, `scores`, `metadata`, plus `with_scores`, + `to_dict`, and `from_dict`. +- **`agentargus/config.py`** — `AgentArgusConfig` (env + kwargs, `from_env`) and + the `Judge` protocol (the LLM-as-judge seam). +- **`agentargus/logging.py`** — `get_logger` factory, `ColorFormatter`, + `JsonFormatter`, `configure_logging`, and `contextvars`-based trace + correlation (`set_trace_id` / `get_trace_id`). +- **`agentargus/__init__.py`** — the (deliberately small) public API surface. +- Scaffolding: `pyproject.toml` (hatchling), `.ruff.toml` (with `T20` to ban + `print()` in library code), `.gitignore`, CI matrix on 3.10/3.11/3.12, + README/LICENSE. + +### 2. Why this shape +- **Deep immutability via tuples, not just `frozen=True`.** Every collection + field is coerced to a `tuple` in `__post_init__`, and every mapping to a + `MappingProxyType`. `frozen=True` alone leaves `result.spans.append(...)` + legal — a real hole. This was decided explicitly with the owner and directly + answers the §11 HARD_QUESTION "is `spans` actually immutable?" with a provable + *yes*. Cost: a caller who wants to build a result incrementally must assemble + the collections first, then construct once. Acceptable — `RunResult` is an + *output* aggregate, not a builder. +- **`with_scores` returns a new object.** Checkpoint 2.1 tempts a mutate-in-place + `scores`. We reversed that in favour of immutability: eval consumers get a + fresh `RunResult`, so a result handed to two evaluators can't be corrupted by + one. `dataclasses.replace` keeps it a one-liner. +- **`Judge` as an injected protocol, no bundled client.** Base install stays + dependency-light (spec §1/§9) and provider-agnostic. A concrete Anthropic + adapter will live behind the `[dev]` extra for tests/demo. Second possible + implementation (proving the abstraction earns its keep, checkpoint 6.1): a + recorded/replay judge for deterministic CI, or a local-model judge. +- **`contextvars` for trace correlation, not a passed-around logger.** The + trace_id is set once by `Agent.run()` and read by the log filter. It survives + async boundaries and keeps trace_id out of every signature (checkpoint 7.1). +- **Raw ANSI over `colorama`.** Modern Windows terminals handle ANSI natively + and we only colorize on a TTY, so the dependency isn't justified. Color is + gated by three independent conditions: config flag AND not `NO_COLOR` AND + `stream.isatty()`. + +### 3. Reuse points introduced +- **`_freeze_mapping`** — the single home for turning a dict into a read-only + mapping; used by every value object. No duplicated freezing logic. +- **`get_logger`** — the *only* sanctioned logger entry point. `logging.getLogger` + is never called elsewhere in library code (enforced socially + by review). +- **`_TraceIdFilter`** — one place that injects trace_id onto records; both + formatters just read `record.trace_id`. +- **`AgentArgusConfig`** — the single home for env access; no module reads + `os.environ` directly. + +### 4. methodoverload decision +- **Not used in Module 0**, correctly. Module 0's inputs are not type-dispatched + — there is no "same operation, different runtime types" site here. Forcing it + would be decorative. Its designated sites are Modules 3, 5, 6 (and cautiously + 1). **Important verified finding:** the spec's §4 says to import `OverloadMeta` + and use it as a metaclass for method overloading. **`OverloadMeta` does not + exist in `methodoverload` 0.1.7.** Empirically, `@overload` works on instance + methods with *no metaclass at all* (verified with a live test). Module 1+ will + use the real API; the spec's §4 is out of date on this point. Also confirmed + the §4.3 caution: callables have no distinct `isinstance` class, so the + `Agent.wrap` overload will dispatch on `BaseAgent` + a fallback. + +### 5. Failure modes +- If a caller stores a **mutable object inside** an otherwise-frozen field (e.g. + a list as a `metadata` value), the top-level mapping is read-only but the + nested list is still mutable. We freeze one level deep, not recursively. This + is a deliberate, documented boundary (it's a HARD_QUESTION). +- `configure_logging` replaces handlers each call — fine for tests, but two + threads calling it concurrently could race on handler list mutation. Not a + concern in practice (configured once at startup) but noted. +- `from_dict` trusts its input shape; malformed dicts raise `KeyError`/`TypeError` + rather than a friendly error. Acceptable for an internal round-trip helper. + +### 6. The one thing most likely to be asked in review +"You froze the top level and the collections, but a `dict` *value* inside +`metadata` is still mutable. So is `RunResult` immutable or not?" — Answer: it is +immutable at the structural level (you cannot swap or grow any field); it is not +*recursively* deep-frozen for arbitrary nested user data, by design, because +recursively freezing unknown user payloads is expensive and surprising. diff --git a/HARD_QUESTIONS.md b/HARD_QUESTIONS.md new file mode 100644 index 0000000..09994ab --- /dev/null +++ b/HARD_QUESTIONS.md @@ -0,0 +1,56 @@ +# AgentArgus — HARD_QUESTIONS + +Questions a skeptical staff engineer / interviewer would ask about each module. +**Claude Code writes the questions; the owner writes the answers.** A module's +gate does not close until the owner can answer its batch in their own words. + +--- + +## Module 0 — Core (`RunResult`, config, logging) + +1. `RunResult` is `frozen=True` and stores collections as tuples — but a `dict` + *value* nested inside `metadata` is still mutable. Is `RunResult` actually + immutable? Where exactly does the immutability guarantee stop, and why did you + choose to stop it there instead of deep-freezing recursively? + +2. `with_scores` returns a new object instead of mutating. Prove that this + actually prevents a class of bug — give a concrete two-evaluator scenario + where in-place mutation of `scores` would corrupt state, and explain the + memory cost of the copy-on-write approach when `spans` is large. + +3. Trace correlation uses a `contextvars.ContextVar`. What happens to the + trace_id when work is dispatched to a `ThreadPoolExecutor`? Does the child + thread see it? What about an `asyncio.create_task`? Prove you know the + difference and say which one AgentArgus relies on. + +4. You chose raw ANSI codes over `colorama`. On an old `cmd.exe` without VT + processing enabled, what does a colored log line look like? Is that a bug or + acceptable, and what specifically prevents it from happening in practice? + +5. `configure_logging` removes and re-adds handlers on every call. Is that + thread-safe? What breaks if two modules call it concurrently at import time, + and why is (or isn't) that a real risk? + +6. The `Judge` protocol has a single synchronous `complete(prompt) -> str` + method. RAG metrics may need to fire many judge calls per dataset row. Does a + sync-only judge interface bottleneck batch eval? How would you add concurrency + later without breaking the protocol's existing implementers? + +7. `to_dict`/`from_dict` is a hand-written round-trip. `output` and tool + `result` are typed `Any`. What happens when `output` is a non-JSON-serializable + object (e.g. a numpy array or a custom class)? Where does that fail, and is + failing there the right call? + +8. Why is `RunResult` the single coupling point for the whole system? Defend it + against the alternative where each module (tracer, cost, eval) defines its own + result shape. What is the cost of this central aggregate when a future module + needs a field none of the others care about? + +9. `AgentArgusConfig.from_env` silently falls back to defaults on a malformed + `AGENTARGUS_COST_CEILING_USD`. Is silent fallback the right behaviour for a + *cost ceiling* — a safety limit? Argue both sides. + +10. `slots=True` on the dataclasses — what did that buy you, and what did it cost + you (think: pickling, multiple inheritance, adding attributes in + `__post_init__` via `object.__setattr__`)? Why does `object.__setattr__` + still work with slots + frozen? diff --git a/IMPLEMENTAION.md b/IMPLEMENTAION.md new file mode 100644 index 0000000..4688b71 --- /dev/null +++ b/IMPLEMENTAION.md @@ -0,0 +1,456 @@ +# AgentArgus — Claude Code End-to-End Build Specification + +**Author / Owner:** Mohd Arbaaz Siddiqui (`arbaazcode@gmail.com`) +**Repo:** `github.com/mohdcodes/agentargus` +**PyPI target:** `agentargus` +**Python:** 3.10+ (develop on 3.11) +**License:** MIT + +> **What this file is.** This is the authoritative build spec for Claude Code to implement AgentArgus end-to-end. It is not a tutorial and not the old 12-week tactical plan. It defines *architecture, class contracts, the OOP rationale, logging design, the methodoverload integration, testing requirements, and the open-discussion checkpoints* that make the owner able to defend every design decision. +> +> **Non-negotiable working agreement (read before writing any code):** +> 1. **Build in the module order in §5.** Do not jump ahead. Each module must be green (tests + lint + types) before the next begins. +> 2. **Every module ends with a `DESIGN_LOG.md` entry** (see §11) written by you, Claude Code, explaining *what you built and why, what you rejected, and what would break it.* This is how the owner learns the system without hand-typing it. +> 3. **Every module ends with a `HARD_QUESTIONS.md` batch** (see §11) — 6–10 questions a skeptical senior engineer or interviewer would ask about that module. Do NOT answer them. The owner answers them. +> 4. **Reuse ruthlessly.** No copy-pasted logic across modules. If two modules need the same behaviour, it lives in one place (`_internal/` or a shared base class) and both import it. Call this out explicitly in the DESIGN_LOG whenever you extract something. +> 5. **Do not invent library APIs.** For `methodoverload` (the owner's own library) use only the verified API in §4. For any third-party API you are unsure about, stop and flag it rather than guessing. +> 6. **No secrets in code.** All keys via environment / `.env` (gitignored). Never hardcode tokens. + +--- + +## Table of Contents + +1. [Problem, Scope, and Non-Goals](#problem) +2. [Architectural Overview & Data Flow](#architecture) +3. [OOP Design Pillars — Where Each One Lives](#oop) +4. [methodoverload Integration Contract](#methodoverload) +5. [Module Build Order (dependency-ordered)](#build-order) +6. [Detailed Module Specifications](#module-specs) +7. [The Logging System (color, structured, correlated)](#logging) +8. [Testing Requirements (internal test cases per module)](#testing) +9. [Repository Layout](#repo) +10. [CI/CD, Packaging, Release](#cicd) +11. [DESIGN_LOG & HARD_QUESTIONS Protocol](#protocol) +12. [Definition of Done (per module + whole project)](#dod) + +--- + + +## 1. Problem, Scope, and Non-Goals + +### The problem +Teams ship LLM agents to production with almost no operational rigour. There is no single place to answer: *Did the agent do the right thing? What did it cost? Why did it fail? Can it recover?* RAGAS covers RAG evaluation only. Observability tools cover traces only. Reliability is hand-rolled per project. Nobody has a **framework-agnostic, single-package** answer. + +### What AgentArgus is +A production-grade Python library that wraps *any* agent (a callable, a LangGraph graph, an arbitrary function) and gives it, uniformly: + +- **Evaluation** — RAG metrics (faithfulness, answer relevance, context precision/recall) AND agent metrics (tool-use accuracy, plan coherence, error-recovery rate), scored against datasets with regression detection. +- **Observability** — OpenTelemetry traces following GenAI semantic conventions, plus accurate token/cost accounting. +- **Reliability** — retry with backoff, model fallback chains, circuit breaker, dead-letter queue. +- **Human-in-the-loop** — checkpoints that can pause a run for approval. +- **Orchestration helpers** — supervisor/worker and agent-to-agent handoff patterns. + +### Non-goals (write these in DESIGN.md and defend them) +- Not a model-serving/inference engine (that's vLLM/TGI's job). +- Not a vector DB or a RAG framework itself — it *evaluates* RAG, it doesn't *do* retrieval for you (the demo app may, but the library doesn't mandate it). +- Not tied to any one agent framework. LangGraph is *supported*, never *required*. +- Not a hosted product in v0.1.0 — it's a library. A dashboard/app is a *consumer* of it (see the separate Jira-agent project). + +**Open-discussion checkpoint 1.1:** Be ready to explain why "framework-agnostic wrapping" is the core design bet, and what the cost of that generality is (you lose framework-specific optimizations; you must define your own trace schema instead of piggybacking one). + +--- + + +## 2. Architectural Overview & Data Flow + +### The central abstraction +Everything hangs off one wrap: `Agent(inner, ...)`. `inner` is whatever the user already has. AgentArgus decorates its execution with observability, reliability, and (optionally) HITL, and produces a rich `RunResult` that eval consumes. + +``` + ┌──────────────────────────────────────────┐ + user request ─────▶ │ Agent (facade) │ + │ wraps: callable | LangGraph | BaseAgent │ + └───────┬───────────────────────┬───────────┘ + │ │ + reliability │ │ observability + (retry / ▼ ▼ (tracer + cost) + fallback / ┌───────────────┐ ┌────────────────┐ + breaker / │ ReliabilityᴾLCY│ │ Tracer (OTel) │ + DLQ) └──────┬────────┘ │ CostTracker │ + │ └───────┬────────┘ + ▼ │ + ┌────────────────┐ │ + HITL ◀── │ inner.run() │ │ emits spans + cost + checkpoint └───────┬────────┘ │ onto RunResult + │ │ + ▼ ▼ + ┌──────────────────────────────────────┐ + │ RunResult │ + │ output, trace_id, spans, cost, steps, │ + │ tool_calls, errors, scores(after eval)│ + └───────────────────┬──────────────────┘ + │ + ▼ + ┌──────────────────────────────────────┐ + │ EvalRunner ──uses──▶ EvalSuite │ + │ (batch over EvalDataset) │ + │ EvalSuite = [Metric, Metric, ...] │ + └───────────────────┬──────────────────┘ + ▼ + EvalReport (summary, regressions, to_html) +``` + +### The one canonical data object: `RunResult` +This is the spine of the whole system. Define it **first** (Week/Module 1), get it right, and everything else consumes it. Fields: + +- `output: Any` — the agent's final answer +- `trace_id: str` — correlation id linking to spans +- `spans: list[Span]` — structured execution steps (from tracer) +- `cost: CostBreakdown` — tokens + dollars per model call (from CostTracker) +- `tool_calls: list[ToolCall]` — name, args, result, success/failure, latency +- `steps: list[Step]` — ordered reasoning/action steps (for plan-coherence eval) +- `errors: list[ErrorRecord]` — anything caught by reliability layer +- `scores: dict[str, float]` — populated *after* eval runs (empty at run time) +- `metadata: dict` — freeform + +**Open-discussion checkpoint 2.1:** Be ready to explain why `RunResult` is the coupling point and why that's acceptable coupling (it's the domain's natural aggregate; the alternative — each module defining its own result shape — creates N×N adapters). Also be ready to defend `scores` being mutated post-hoc vs. returning a new object (immutability trade-off — see §3). + +--- + + +## 3. OOP Design Pillars — Where Each One Lives + +The owner explicitly wants all four OOP pillars used *meaningfully* (not decoratively). For each, this is where it MUST appear and why. Claude Code: implement exactly here, and justify in DESIGN_LOG. + +### Abstraction +- `Metric` (ABC) — `compute(run_result) -> float`. Callers depend on the interface, never a concrete metric. +- `BaseAgent` (ABC) — defines the `run()` contract; concrete agents/wrappers implement it. +- `ReliabilityStrategy` (ABC) — retry, fallback, circuit-breaker, DLQ all implement a common `execute(callable, context)` interface. +- `SpanExporter` seam — depend on OTel's exporter interface, not a concrete backend. + +### Encapsulation +- `CostTracker` hides the pricing tables and token-accounting math behind `add_usage(...)` / `total()`. Pricing tables live in `_internal/pricing.py` and are **private** — never import them outside the observability package. +- `CircuitBreaker` hides its state machine (`CLOSED → OPEN → HALF_OPEN`) behind `allow()` / `record_success()` / `record_failure()`. Internal counters are name-mangled/private. +- `OverloadCache` behaviour (from methodoverload) is used but never exposed. + +### Inheritance +- Concrete metrics inherit `Metric`: `Faithfulness`, `AnswerRelevance`, `ContextPrecision`, `ContextRecall`, `ToolUseAccuracy`, `PlanCoherence`, `ErrorRecoveryRate`. +- Concrete reliability strategies inherit `ReliabilityStrategy`. +- **Rule:** inheritance is for *is-a* only. If you're tempted to inherit for code reuse without an is-a relationship, use composition instead and note it in DESIGN_LOG. (E.g. `Agent` does NOT inherit from `Tracer`; it *has* a tracer.) + +### Polymorphism +- `EvalSuite` iterates `list[Metric]` and calls `metric.compute(run_result)` without knowing the concrete type — classic runtime polymorphism. +- `ReliabilityPolicy` composes multiple `ReliabilityStrategy` objects and applies them uniformly. +- **methodoverload-based polymorphism** (see §4): where a single conceptual operation legitimately takes different input *types*, use `@overload` for type-dispatched implementations instead of `isinstance` ladders. + +**Open-discussion checkpoint 3.1:** Be ready to explain the difference between the inheritance-based polymorphism (EvalSuite over Metric) and the overload-based polymorphism (methodoverload), and *why each is used where it is*. A common interview trap: "why not just isinstance-branch?" Answer must cover open/closed principle and testability. + +--- + + +## 4. methodoverload Integration Contract + +`methodoverload` is the owner's own PyPI library (`pip install methodoverload`, v0.1.7). **Verified API — use only these:** + +```python +from methodoverload import overload, OverloadMeta, NoMatchingOverloadError +``` + +- `@overload` — decorates multiple same-named implementations; dispatch is by argument **type** at runtime via `isinstance()`. +- `OverloadMeta` — metaclass required for overloading **instance/class/static methods** inside a class: + `class Foo(metaclass=OverloadMeta): ...` +- Works with `@classmethod` and `@staticmethod` (place `@overload` **outermost**, then `@classmethod`). +- `NoMatchingOverloadError` — raised when no signature matches; catch it explicitly where you offer a fallback. + +**Verified limitations (design around these — do not fight them):** +- Dispatch uses `isinstance()`. **No generic types** — `List[int]` won't dispatch; only `list` will. +- Only positional/keyword args, no compile-time checking. +- First matching overload wins. + +### Where methodoverload MUST be used (genuine fits, distinct runtime types) +1. **`EvalDataset` loading** — `load(source)` overloaded on `source: str` (path), `source: list` (in-memory records), `source: dict` (single record). Distinct types, clean dispatch. +2. **`CostTracker.add_usage(...)`** — overload on a raw usage `dict` vs. a typed `Usage` object vs. a provider-specific response object. Different callers pass different shapes; overload removes the isinstance ladder. +3. **`Agent` construction / wrapping target** — `wrap(inner)` overloaded on `inner: BaseAgent`, `inner: `... **CAUTION:** callables are not a distinct `isinstance` class in a clean way; verify behaviour with a test before relying on it. If it doesn't dispatch cleanly, fall back to a single method with an internal type check and record WHY in DESIGN_LOG. **Do not force methodoverload where it doesn't fit.** +4. **Metric input adaptation** — a metric's `compute` accepting either a full `RunResult` or a lightweight `dict` trace (useful for unit tests). Overload on `RunResult` vs `dict`. + +### Where NOT to use it +- Anywhere dispatch would depend on generic parameterization (`list[str]` vs `list[int]`) — it can't tell them apart. +- Hot inner loops where the caching still adds overhead vs. a direct call — measure if unsure. + +**Open-discussion checkpoint 4.1:** Be ready to explain, for each usage site, *why overload beats an isinstance ladder there* — and to honestly name the one or two places you tried it and backed off. "I used my own library everywhere" is a weaker answer than "I used it where type-dispatch was genuinely cleaner and here's where I decided it wasn't." + +--- + + +## 5. Module Build Order (dependency-ordered) + +Build strictly in this order. Each is a hard gate. + +| # | Module | Depends on | Gate before moving on | +|---|--------|-----------|----------------------| +| 0 | Repo scaffold + `RunResult` + logging + config | — | CI green, `RunResult` frozen, logger prints color | +| 1 | `agents/` — `BaseAgent`, `Agent` facade | 0 | wrap a trivial callable, `run()` returns `RunResult` | +| 2 | `observability/tracer.py` | 1 | every run emits OTel spans visible in Jaeger | +| 3 | `observability/cost.py` + `_internal/pricing.py` | 2 | cost within 5% of a hand-computed fixture | +| 4 | `reliability/` (retry, fallback, breaker, DLQ) + `ReliabilityPolicy` | 1 | `Agent(reliability=...)` recovers from injected failures | +| 5 | `eval/metrics/base.py` + RAG metrics | 1 | `EvalSuite([...]).run(result)` returns scores | +| 6 | `eval/dataset.py` + `eval/runner.py` + `EvalReport` | 5 | batch eval over JSONL → HTML report + regressions | +| 7 | `eval` agent metrics (tool-use, plan-coherence, recovery) | 6, 2 | unified suite handles RAG + agent metrics | +| 8 | `agents/patterns.py` (supervisor, handoff) + checkpointer | 1 | 3-agent supervisor runs, state persists (SQLite) | +| 9 | `hitl/checkpoint.py` | 4, 8 | a run can pause and resume on approval | +| 10 | `examples/deep_research_agent/` | all | end-to-end demo runs, produces eval report | +| 11 | docs + benchmarks + release | all | v0.1.0 on TestPyPI then PyPI | + +**Rationale to defend (checkpoint 5.1):** why `RunResult` and logging come *before* any feature (everything depends on them), why observability precedes reliability (you can't verify recovery without traces), and why eval agent-metrics come *after* tracing (they read the trace). + +--- + + +## 6. Detailed Module Specifications + +For each module: the classes, their responsibilities, the key methods with signatures, and the reuse points. Claude Code implements these contracts; the owner reviews before you proceed. + +### 6.0 Core (`agentargus/__init__.py`, `core/`, `config.py`, `logging.py`) +- `RunResult` — frozen dataclass per §2. Provide a `with_scores(scores)` method that returns a **new** RunResult (immutability; do not mutate in place). This is deliberately the opposite of the "mutate scores" temptation in checkpoint 2.1 — resolve it in favour of immutability and explain the reversal. +- Supporting dataclasses: `Span`, `CostBreakdown`, `ToolCall`, `Step`, `ErrorRecord`. +- `AgentArgusConfig` — central config object, populated from env + explicit kwargs. Encapsulates: default judge model, cost ceiling, tracer exporter choice, log level/color toggle. +- `logging.py` — see §7. + +### 6.1 `agents/` +- `BaseAgent(ABC)` — `run(self, input: Any) -> RunResult` (abstract). `arun` async variant. +- `Agent(BaseAgent)` — the **facade**. Composition, not inheritance, for its collaborators: + - `self._inner` (wrapped target) + - `self._tracer` (Tracer | None) + - `self._cost` (CostTracker | None) + - `self._reliability` (ReliabilityPolicy | None) + - `self._hitl` (Checkpoint | None) + - `run()` orchestration order: open trace span → apply reliability wrapper around `inner` call → (HITL pause point) → collect cost → assemble `RunResult`. +- `wrap(inner)` — methodoverload usage site #3 (with the documented caution). + +### 6.2 `observability/tracer.py` + `conventions.py` +- `Tracer` — thin OO wrapper over `opentelemetry-sdk`. Encapsulates exporter setup. Method `span(name, **attrs)` context manager; auto-instrument decorator `@traced`. +- `conventions.py` — constants for GenAI semantic-convention attribute keys (`gen_ai.system`, `gen_ai.request.model`, `gen_ai.usage.input_tokens`, etc.). One source of truth; every span-attribute write imports from here (reuse point — no string literals scattered). + +### 6.3 `observability/cost.py` + `_internal/pricing.py` +- `CostTracker` — `add_usage(...)` (overload site #2), `total() -> CostBreakdown`, cost-ceiling guard raising `CostCeilingExceeded`. +- `_internal/pricing.py` — private per-provider/per-model price tables `{model: (input_per_1k, output_per_1k)}`. Encapsulated; never imported outside `observability/`. + +### 6.4 `reliability/` +- `ReliabilityStrategy(ABC)` — `execute(self, fn, ctx)`. +- `RetryWithBackoff(ReliabilityStrategy)` — exponential backoff + jitter; configurable max attempts. +- `FallbackChain(ReliabilityStrategy)` — ordered list of alternative callables/models; on failure try next; catch `NoMatchingOverloadError` too if relevant. +- `CircuitBreaker(ReliabilityStrategy)` — encapsulated `CLOSED/OPEN/HALF_OPEN` state machine. +- `DeadLetterQueue` — persists permanently-failed inputs (start with JSONL sink; interface allows swapping). +- `ReliabilityPolicy` — **composes** strategies and applies them in order. Polymorphism site. `Agent(reliability=ReliabilityPolicy(retries=3, fallback_models=[...]))`. + +### 6.5 `eval/` +- `Metric(ABC)` — `compute(self, run_result) -> float`; `name: str`. Overload site #4 (accept `RunResult` or `dict`). +- RAG metrics (LLM-judge based; judge model from config): `Faithfulness` (claim decomposition), `AnswerRelevance`, `ContextPrecision`, `ContextRecall`. +- Agent metrics (read the trace/steps): `ToolUseAccuracy` (expected vs actual tool calls), `PlanCoherence` (LLM judge over `steps`), `ErrorRecoveryRate` (from `errors` + recovery outcomes). +- `EvalSuite` — holds `list[Metric]`; `run(run_result) -> dict[str,float]`; polymorphic iteration. +- `EvalDataset` — `load(source)` overload site #1; `from_jsonl`, validation. +- `EvalRunner` — `run(agent, dataset, suite) -> EvalReport`, batch, async-capable. +- `EvalReport` — `summary()`, `regressions(baseline)`, `to_html()` (Jinja template). + +### 6.6 `agents/patterns.py` +- `SupervisorAgent(BaseAgent)` — routes to `workers: list[BaseAgent]`; polymorphic over workers. +- Handoff primitive — structured transfer of control + context between agents. +- SQLite checkpointer for persistence. + +### 6.7 `hitl/checkpoint.py` +- `Checkpoint` — a pause point; `require_approval(context) -> Decision`. Pluggable approval backend (console/callback in v0.1.0). Integrates with reliability (a rejected checkpoint is a controlled failure, not a crash). + +**Open-discussion checkpoint 6.1:** For every ABC, be ready to name a concrete second implementation that *could* exist — proving the abstraction earns its keep (e.g. a non-LLM heuristic metric, a Redis-backed DLQ, a Slack approval backend). If you can't name one, the abstraction may be premature. + +--- + + +## 7. The Logging System (color, structured, correlated) + +The owner specifically wants a strong logging system with color and good detail. Requirements: + +- **One logger factory** `get_logger(name)` in `agentargus/logging.py`. Never call `logging.getLogger` directly elsewhere (reuse point). +- **Colorized console output** for local dev: level-based colors (DEBUG grey, INFO green, WARNING yellow, ERROR red, CRITICAL bold red). Use ANSI codes directly (no heavy dep) OR `colorama` for Windows safety — pick one and justify. Color must be **auto-disabled** when output is not a TTY (piped/CI) and via a config flag. Respect `NO_COLOR` env convention. +- **Structured logs** — support a JSON formatter for production (machine-parseable) selectable via config; human+color formatter for dev. +- **Correlation** — every log line inside an agent run carries the `trace_id` (use a `contextvars.ContextVar` set by `Agent.run()` / the tracer, read by the formatter). This is the bridge between logs and traces — call it out in DESIGN_LOG as a deliberate observability design choice. +- **No print()** anywhere in library code. Ever. (Examples/CLI may print.) +- **Sensible levels:** span open/close = DEBUG, cost totals = INFO, retries/circuit trips = WARNING, dead-letter = ERROR. + +**Open-discussion checkpoint 7.1:** Be ready to explain how logs and traces are correlated (the `contextvars` trace_id), and why you didn't just pass the logger around explicitly (contextvars survive across async boundaries and don't pollute signatures). + +--- + + +## 8. Testing Requirements (internal test cases per module) + +**The rule (from the owner's plan, kept):** every module gets its tests written *in the same PR as the implementation*. Untested code does not pass the module gate. + +### Test pyramid +- **Unit (most):** one class/function; mock all LLM/network calls; <1s each. +- **Integration (some):** module interactions; recorded LLM responses via `vcr.py`/`pytest-recording`. +- **End-to-end (few):** the demo app; run in a dedicated CI job, not every push. + +### Determinism strategies for LLM code +1. Mock the judge/LLM with a fixture returning canned JSON. +2. Snapshot/record real responses once, replay after. +3. Statistical asserts for e2e ("score>0.7 on ≥90% of dataset"), never exact equality. + +### Mandatory internal test cases (minimum — Claude Code adds more) +- **RunResult:** immutability (`with_scores` returns new object, original unchanged); serialization round-trip. +- **Agent:** wraps callable; `run()` populates trace_id, cost, tool_calls; async path works. +- **Tracer:** spans emitted with correct GenAI convention keys; in-memory exporter assertions; nested spans nest. +- **CostTracker:** cost math correct to fixture; `add_usage` overload dispatches on all input types (**explicit test that methodoverload picks the right impl**, incl. a `NoMatchingOverloadError` case); cost ceiling raises. +- **Reliability:** retry succeeds on Nth attempt (injected transient failure); fallback moves to next model on failure; circuit breaker opens after threshold and half-opens after cooldown; DLQ captures permanent failure. +- **Metrics:** each metric high/low score with mocked judge; metric accepts both `RunResult` and `dict` (overload test). +- **EvalRunner:** batch aggregates; `regressions(baseline)` flags a deliberately worsened score; `to_html` produces valid HTML. +- **Patterns:** supervisor routes to correct worker; handoff preserves context; checkpointer persists+restores across process restart (temp SQLite). +- **HITL:** approval resumes; rejection produces controlled failure recorded in `errors`. +- **methodoverload sites (cross-cutting):** a dedicated `tests/unit/test_overload_sites.py` asserting each overloaded method dispatches correctly per type AND raises `NoMatchingOverloadError` on an unsupported type. + +### Coverage +- Target **80% line coverage**; do not chase 100%. Report in CI (`--cov=agentargus --cov-report=xml`). + +**Open-discussion checkpoint 8.1:** Be ready to explain how you test non-deterministic LLM behaviour without flaky CI, and why exact-match asserts on LLM output are a smell. + +--- + + +## 9. Repository Layout + +``` +agentargus/ +├── .github/workflows/{ci.yml, publish.yml} +├── agentargus/ +│ ├── __init__.py # public API surface + __version__ +│ ├── config.py # AgentArgusConfig +│ ├── logging.py # get_logger, formatters, color, contextvars +│ ├── core/ +│ │ ├── __init__.py +│ │ └── results.py # RunResult, Span, CostBreakdown, ToolCall, Step, ErrorRecord +│ ├── agents/ +│ │ ├── base.py # BaseAgent (ABC) +│ │ ├── agent.py # Agent facade (wrap = overload site) +│ │ ├── patterns.py # SupervisorAgent, handoff +│ │ └── orchestrator.py +│ ├── eval/ +│ │ ├── suite.py # EvalSuite +│ │ ├── dataset.py # EvalDataset (load = overload site) +│ │ ├── runner.py # EvalRunner +│ │ ├── report.py # EvalReport (+ jinja html template) +│ │ └── metrics/{base.py, rag.py, agent.py} +│ ├── observability/ +│ │ ├── tracer.py +│ │ ├── conventions.py # GenAI semantic-convention keys (single source) +│ │ └── cost.py # CostTracker (add_usage = overload site) +│ ├── reliability/ +│ │ ├── base.py # ReliabilityStrategy (ABC) +│ │ ├── retry.py +│ │ ├── fallback.py +│ │ ├── circuit_breaker.py +│ │ ├── dead_letter.py +│ │ └── policy.py # ReliabilityPolicy (composition) +│ ├── hitl/checkpoint.py +│ └── _internal/ +│ ├── pricing.py # PRIVATE price tables +│ └── exceptions.py # NoMatchingOverloadError re-exported? NO — import from methodoverload +├── tests/{unit/, integration/, fixtures/golden_dataset.jsonl, conftest.py} +├── examples/deep_research_agent/{README.md, agent.py, run.py} +├── docs/{getting_started.md, architecture.md, concepts/} +├── benchmarks/{README.md, deep_research_baseline.py} +├── DESIGN.md # problem/scope/non-goals/decisions (owner-authored w/ your input) +├── DESIGN_LOG.md # per-module decision log (see §11) +├── HARD_QUESTIONS.md # per-module interview questions (see §11) +├── pyproject.toml, .ruff.toml, .gitignore +├── README.md, LICENSE, CHANGELOG.md, CONTRIBUTING.md +``` + +### `pyproject.toml` key points +- Build backend: `hatchling`. +- Runtime deps: `opentelemetry-sdk`, `opentelemetry-api`, `jinja2`, `methodoverload>=0.1.7`. Keep the runtime dep list **minimal**; put judge-LLM clients behind optional extras. +- Optional extras: `dev` (pytest, pytest-cov, pytest-asyncio, ruff, mypy, build, twine, pytest-recording), `langgraph` (langgraph support), `docs` (pdoc). +- Author = Mohd Arbaaz Siddiqui; keywords include `llm, agents, evaluation, observability`. + +--- + + +## 10. CI/CD, Packaging, Release + +- **`ci.yml`:** matrix Python 3.10/3.11/3.12 → install `.[dev]` → `ruff check .` → `ruff format --check .` → `mypy agentargus` → `pytest --cov=agentargus`. e2e demo runs in a separate, non-blocking job. +- **`publish.yml`:** on tag `v*` → build → `twine upload` using `PYPI_API_TOKEN` secret. Prefer Trusted Publishing (OIDC) if set up. +- **Release flow:** bump `__version__` in `__init__.py` + `pyproject.toml` → build → upload TestPyPI → install from TestPyPI in a clean venv and smoke test → tag → real PyPI → GitHub release from CHANGELOG. +- **The `agentargus` PyPI name should already be reserved** with a v0.0.0 stub per the original plan. If not yet reserved, do that first (it's cheap insurance). + +--- + + +## 11. DESIGN_LOG & HARD_QUESTIONS Protocol (how the owner learns without hand-typing) + +This is the mechanism that replaces "typing every line" with "understanding every decision." **Mandatory after each module.** + +### `DESIGN_LOG.md` — you (Claude Code) write this +Per module, append a dated entry with: +1. **What was built** — the classes/functions and their responsibilities, in plain English. +2. **Why this shape** — the key design decisions and the alternatives rejected (e.g. "composition over inheritance for Agent's tracer because…"). +3. **Reuse points introduced** — what got extracted so nothing is duplicated, and who consumes it. +4. **methodoverload decision** — used here / not used here, and the honest reason. +5. **Failure modes** — what would break this module and how it degrades. +6. **The one thing most likely to be asked in review.** + +### `HARD_QUESTIONS.md` — you write the questions, owner writes the answers +Per module, 6–10 questions a skeptical staff engineer would ask. Examples of the *level* expected: +- "Your circuit breaker shares state across threads — is `record_failure` thread-safe? Prove it." +- "Faithfulness uses an LLM judge — how do you stop the judge's bias from silently inflating scores? What's your calibration story?" +- "`RunResult` is frozen but `spans` is a list — is it actually immutable? What stops a caller mutating the list in place?" +- "Cost is 'within 5% of invoice' — where does the other 5% go, and when would it be worse?" + +**Do NOT answer these.** Leave them for the owner. This is the explain-back test in written form. + +### The owner's loop per module +1. Read your DESIGN_LOG entry. +2. Answer the HARD_QUESTIONS in their own words. +3. Anything they can't answer → they open a discussion (with you or with me) until they can. +4. Only then does the module gate close. + +--- + + +## 12. Definition of Done + +### Per module +- [ ] Code implements the §6 contract for that module. +- [ ] OOP pillar(s) assigned to it in §3 are present and justified. +- [ ] methodoverload used at its designated site OR explicitly waived in DESIGN_LOG with reason. +- [ ] All mandatory internal test cases (§8) pass; coverage ≥80% for the module. +- [ ] `ruff check`, `ruff format --check`, `mypy` all clean. +- [ ] No `print()`; logging used with correct levels + trace correlation. +- [ ] DESIGN_LOG entry written (by you). +- [ ] HARD_QUESTIONS batch written (by you), answered (by owner). +- [ ] CI green on 3.10/3.11/3.12. + +### Whole project (v0.1.0) +- [ ] Modules 0–9 done; demo app (10) runs end-to-end and emits a real eval report. +- [ ] Demo app produces: traces in Jaeger, an accurate cost breakdown, an HTML eval report with RAG + agent metrics, and at least one deliberately-injected failure that reliability recovers from. +- [ ] `docs/architecture.md` has a diagram matching §2. +- [ ] `docs/getting_started.md` gets a new user to a first traced+evaluated run in <5 minutes. +- [ ] Benchmarks: naive baseline vs AgentArgus-instrumented (overhead quantified — be honest about the cost of instrumentation). +- [ ] Published to TestPyPI, smoke-tested in clean venv, then to real PyPI as v0.1.0. +- [ ] README badges (PyPI version, Python versions, CI, license, coverage). + +--- + +## Appendix A — Instruction to Claude Code on AI-assistance discipline + +The owner is deliberately using you (Claude Code) to build fast. That is fine and expected. To keep this a *portfolio-grade, defensible* project rather than an opaque generated blob, honour these: + +- **Narrate decisions, not just code.** When you make a non-obvious choice, say why in the DESIGN_LOG, not just in a code comment. +- **Prefer the owner's own primitives.** `methodoverload` is theirs — use it where it genuinely fits (§4) and say where it doesn't. +- **Flag, don't fabricate.** If you're unsure of a third-party API (OTel semantic-convention keys, LangGraph checkpointer API, a provider's usage-response shape), stop and say "verify this" rather than inventing a plausible-looking call. +- **Keep the public API small.** `from agentargus import Agent, EvalSuite, EvalRunner, ReliabilityPolicy` and the metrics should be most of it. Everything else is internal. +- **One behaviour, one home.** Any time you'd duplicate logic, extract it and note the extraction. + +## Appendix B — First actions for Claude Code (do these in order) + +1. Confirm/scaffold repo structure per §9. Set up `pyproject.toml`, `.ruff.toml`, CI. +2. Implement **Module 0** completely: `RunResult` + supporting dataclasses, `AgentArgusConfig`, `logging.py` (color + contextvars + JSON formatter). Full tests. Gate. +3. Write the Module 0 DESIGN_LOG entry and HARD_QUESTIONS batch. **Stop and hand back to the owner** before Module 1. +4. Proceed module by module through §5, gating at each step, never batching multiple modules without owner sign-off. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..76b4fbc --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Mohd Arbaaz Siddiqui + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..bf2a2ff --- /dev/null +++ b/README.md @@ -0,0 +1,30 @@ +# AgentArgus + +Framework-agnostic **evaluation, observability, and reliability** for LLM agents — in one package. + +Wrap any agent (a callable, a LangGraph graph, a `BaseAgent`) and get, uniformly: + +- **Evaluation** — RAG metrics (faithfulness, answer relevance, context precision/recall) *and* agent metrics (tool-use accuracy, plan coherence, error-recovery), with regression detection. +- **Observability** — OpenTelemetry traces (GenAI semantic conventions) + accurate token/cost accounting. +- **Reliability** — retry/backoff, model fallback chains, circuit breaker, dead-letter queue. +- **Human-in-the-loop** — approval checkpoints that can pause a run. +- **Orchestration helpers** — supervisor/worker and agent handoff patterns. + +> **Status:** early development (`0.1.0.dev0`). Built module-by-module per the [implementation spec](IMPLEMENTAION.md). See [DESIGN_LOG.md](DESIGN_LOG.md) for the decision record. + +## Install + +```bash +pip install agentargus # minimal runtime +pip install "agentargus[dev]" # + test/lint/judge-adapter tooling +``` + +## Design docs + +- [DESIGN.md](DESIGN.md) — problem, scope, non-goals. +- [DESIGN_LOG.md](DESIGN_LOG.md) — per-module decision log. +- [HARD_QUESTIONS.md](HARD_QUESTIONS.md) — review questions per module. + +## License + +MIT © Mohd Arbaaz Siddiqui diff --git a/agentargus/__init__.py b/agentargus/__init__.py new file mode 100644 index 0000000..1a61c42 --- /dev/null +++ b/agentargus/__init__.py @@ -0,0 +1,36 @@ +"""AgentArgus — evaluation, observability, and reliability for LLM agents. + +The public API surface is intentionally small (spec Appendix A). Module 0 +exposes the core result object, configuration, and the logging factory; later +modules add ``Agent``, ``EvalSuite``, ``EvalRunner``, ``ReliabilityPolicy``, +and the metrics. +""" + +from __future__ import annotations + +from agentargus.config import AgentArgusConfig, Judge +from agentargus.core import ( + CostBreakdown, + ErrorRecord, + RunResult, + Span, + Step, + ToolCall, +) +from agentargus.logging import configure_logging, get_logger + +__version__ = "0.1.0.dev0" + +__all__ = [ + "__version__", + "RunResult", + "Span", + "ToolCall", + "Step", + "ErrorRecord", + "CostBreakdown", + "AgentArgusConfig", + "Judge", + "get_logger", + "configure_logging", +] diff --git a/agentargus/_internal/__init__.py b/agentargus/_internal/__init__.py new file mode 100644 index 0000000..e35c48b --- /dev/null +++ b/agentargus/_internal/__init__.py @@ -0,0 +1 @@ +"""Private internals. Nothing here is part of the public API.""" diff --git a/agentargus/config.py b/agentargus/config.py new file mode 100644 index 0000000..3f052c5 --- /dev/null +++ b/agentargus/config.py @@ -0,0 +1,81 @@ +"""Central configuration object for AgentArgus. + +``AgentArgusConfig`` is populated from environment variables with explicit +keyword overrides taking precedence. It encapsulates cross-cutting settings so +no module reaches into ``os.environ`` directly — one home for configuration. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import Any, Protocol, runtime_checkable + +__all__ = ["AgentArgusConfig", "Judge"] + + +@runtime_checkable +class Judge(Protocol): + """The seam for LLM-as-judge evaluation. + + AgentArgus ships **no** provider client in the base package (spec §1/§9 — + framework-agnostic, minimal deps). Metrics that need an LLM depend on this + protocol; the user injects any implementation. A thin Anthropic adapter is + provided behind the ``[dev]`` extra for tests and the demo. + """ + + def complete(self, prompt: str) -> str: + """Return the model's text completion for ``prompt``.""" + ... + + +def _env_bool(name: str, default: bool) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +def _env_float(name: str, default: float | None) -> float | None: + raw = os.environ.get(name) + if raw is None: + return default + try: + return float(raw) + except ValueError: + return default + + +@dataclass +class AgentArgusConfig: + """Cross-cutting configuration, resolved from env + explicit kwargs. + + Precedence: an explicitly-passed value wins; otherwise the environment is + consulted via ``from_env``; otherwise a sane default is used. + """ + + judge_model: str = "claude-opus-4-8" + judge: Judge | None = None + cost_ceiling_usd: float | None = None + tracer_exporter: str = "console" # "console" | "otlp" | "memory" + log_level: str = "INFO" + log_color: bool = True + log_json: bool = False + extra: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_env(cls, **overrides: Any) -> AgentArgusConfig: + """Build a config from the environment, then apply explicit overrides.""" + base = cls( + judge_model=os.environ.get("AGENTARGUS_JUDGE_MODEL", "claude-opus-4-8"), + cost_ceiling_usd=_env_float("AGENTARGUS_COST_CEILING_USD", None), + tracer_exporter=os.environ.get("AGENTARGUS_TRACER_EXPORTER", "console"), + log_level=os.environ.get("AGENTARGUS_LOG_LEVEL", "INFO"), + log_color=_env_bool("AGENTARGUS_LOG_COLOR", True), + log_json=_env_bool("AGENTARGUS_LOG_JSON", False), + ) + for key, value in overrides.items(): + if not hasattr(base, key): + raise TypeError(f"Unknown config option: {key!r}") + setattr(base, key, value) + return base diff --git a/agentargus/core/__init__.py b/agentargus/core/__init__.py new file mode 100644 index 0000000..03f7f68 --- /dev/null +++ b/agentargus/core/__init__.py @@ -0,0 +1,19 @@ +"""Core domain objects shared across every AgentArgus module.""" + +from agentargus.core.results import ( + CostBreakdown, + ErrorRecord, + RunResult, + Span, + Step, + ToolCall, +) + +__all__ = [ + "RunResult", + "Span", + "ToolCall", + "Step", + "ErrorRecord", + "CostBreakdown", +] diff --git a/agentargus/core/results.py b/agentargus/core/results.py new file mode 100644 index 0000000..8083650 --- /dev/null +++ b/agentargus/core/results.py @@ -0,0 +1,234 @@ +"""Core domain objects for AgentArgus. + +This module defines ``RunResult`` — the single canonical data object that is the +spine of the whole system (see spec §2) — and its supporting value objects. + +Design posture (decided in the Module 0 discussion, recorded in DESIGN_LOG): + +* Every object here is a **deeply immutable** frozen dataclass. Collection + fields are stored as ``tuple`` (never ``list``) so a caller cannot mutate + ``result.spans`` in place. This directly answers the HARD_QUESTION + "``spans`` is a list — is it actually immutable?" with a provable *yes*. +* ``RunResult.scores`` is populated *after* evaluation. Rather than mutate the + object in place, ``with_scores`` returns a **new** ``RunResult`` — resolving + checkpoint 2.1 in favour of immutability. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field, replace +from types import MappingProxyType +from typing import Any + +__all__ = [ + "Span", + "ToolCall", + "Step", + "ErrorRecord", + "CostBreakdown", + "RunResult", +] + + +def _freeze_mapping(m: Mapping[str, Any]) -> Mapping[str, Any]: + """Return a read-only view of a mapping so dict fields cannot be mutated.""" + return MappingProxyType(dict(m)) + + +@dataclass(frozen=True, slots=True) +class Span: + """A single structured execution step, emitted by the tracer (spec §2).""" + + name: str + span_id: str + start_time: float + end_time: float + attributes: Mapping[str, Any] = field(default_factory=dict) + parent_id: str | None = None + + def __post_init__(self) -> None: + # Freeze the mapping so ``span.attributes["x"] = ...`` raises. + object.__setattr__(self, "attributes", _freeze_mapping(self.attributes)) + + @property + def duration(self) -> float: + return self.end_time - self.start_time + + +@dataclass(frozen=True, slots=True) +class ToolCall: + """A single tool invocation: name, args, result, success, latency (spec §2).""" + + name: str + args: Mapping[str, Any] + result: Any + success: bool + latency: float + error: str | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "args", _freeze_mapping(self.args)) + + +@dataclass(frozen=True, slots=True) +class Step: + """An ordered reasoning/action step, used by plan-coherence eval (spec §2).""" + + index: int + kind: str # e.g. "reason" | "action" | "observation" + content: str + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "metadata", _freeze_mapping(self.metadata)) + + +@dataclass(frozen=True, slots=True) +class ErrorRecord: + """Anything caught by the reliability layer (spec §2).""" + + error_type: str + message: str + recovered: bool + attempt: int = 0 + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "metadata", _freeze_mapping(self.metadata)) + + +@dataclass(frozen=True, slots=True) +class CostBreakdown: + """Token + dollar accounting for a run (produced by CostTracker, spec §2).""" + + input_tokens: int = 0 + output_tokens: int = 0 + input_cost: float = 0.0 + output_cost: float = 0.0 + + @property + def total_tokens(self) -> int: + return self.input_tokens + self.output_tokens + + @property + def total_cost(self) -> float: + return self.input_cost + self.output_cost + + def __add__(self, other: CostBreakdown) -> CostBreakdown: + if not isinstance(other, CostBreakdown): + return NotImplemented + return CostBreakdown( + input_tokens=self.input_tokens + other.input_tokens, + output_tokens=self.output_tokens + other.output_tokens, + input_cost=self.input_cost + other.input_cost, + output_cost=self.output_cost + other.output_cost, + ) + + +@dataclass(frozen=True, slots=True) +class RunResult: + """The canonical result of one wrapped agent run — the spine of AgentArgus. + + Collection fields are stored as tuples for genuine immutability. Use the + typed sequence accessors; the underlying storage is never a mutable list. + ``scores`` starts empty and is filled by ``with_scores`` after eval, which + returns a *new* object rather than mutating this one. + """ + + output: Any + trace_id: str + spans: Sequence[Span] = () + cost: CostBreakdown = field(default_factory=CostBreakdown) + tool_calls: Sequence[ToolCall] = () + steps: Sequence[Step] = () + errors: Sequence[ErrorRecord] = () + scores: Mapping[str, float] = field(default_factory=dict) + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + # Coerce every collection to an immutable form, regardless of what the + # caller passed (list, generator, tuple). One home for the coercion. + object.__setattr__(self, "spans", tuple(self.spans)) + object.__setattr__(self, "tool_calls", tuple(self.tool_calls)) + object.__setattr__(self, "steps", tuple(self.steps)) + object.__setattr__(self, "errors", tuple(self.errors)) + object.__setattr__(self, "scores", _freeze_mapping(self.scores)) + object.__setattr__(self, "metadata", _freeze_mapping(self.metadata)) + + def with_scores(self, scores: Mapping[str, float]) -> RunResult: + """Return a NEW RunResult with ``scores`` merged in (immutability).""" + merged = {**self.scores, **scores} + return replace(self, scores=merged) + + def to_dict(self) -> dict[str, Any]: + """Serialize to a plain dict (JSON-friendly). Round-trips via from_dict.""" + return { + "output": self.output, + "trace_id": self.trace_id, + "spans": [ + { + "name": s.name, + "span_id": s.span_id, + "start_time": s.start_time, + "end_time": s.end_time, + "attributes": dict(s.attributes), + "parent_id": s.parent_id, + } + for s in self.spans + ], + "cost": { + "input_tokens": self.cost.input_tokens, + "output_tokens": self.cost.output_tokens, + "input_cost": self.cost.input_cost, + "output_cost": self.cost.output_cost, + }, + "tool_calls": [ + { + "name": t.name, + "args": dict(t.args), + "result": t.result, + "success": t.success, + "latency": t.latency, + "error": t.error, + } + for t in self.tool_calls + ], + "steps": [ + { + "index": st.index, + "kind": st.kind, + "content": st.content, + "metadata": dict(st.metadata), + } + for st in self.steps + ], + "errors": [ + { + "error_type": e.error_type, + "message": e.message, + "recovered": e.recovered, + "attempt": e.attempt, + "metadata": dict(e.metadata), + } + for e in self.errors + ], + "scores": dict(self.scores), + "metadata": dict(self.metadata), + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> RunResult: + """Reconstruct a RunResult from ``to_dict`` output.""" + cost_data = data.get("cost", {}) + return cls( + output=data["output"], + trace_id=data["trace_id"], + spans=tuple(Span(**s) for s in data.get("spans", [])), + cost=CostBreakdown(**cost_data), + tool_calls=tuple(ToolCall(**t) for t in data.get("tool_calls", [])), + steps=tuple(Step(**st) for st in data.get("steps", [])), + errors=tuple(ErrorRecord(**e) for e in data.get("errors", [])), + scores=dict(data.get("scores", {})), + metadata=dict(data.get("metadata", {})), + ) diff --git a/agentargus/logging.py b/agentargus/logging.py new file mode 100644 index 0000000..5015353 --- /dev/null +++ b/agentargus/logging.py @@ -0,0 +1,159 @@ +"""The AgentArgus logging system (spec §7). + +Goals: + * One factory ``get_logger(name)`` — never call ``logging.getLogger`` + elsewhere (single home / reuse point). + * Colorized human output for dev, level-based colors, auto-disabled when + output is not a TTY, when ``NO_COLOR`` is set, or via config. + * A JSON formatter for production, selectable via config. + * Every log line inside a run carries the ``trace_id``, propagated through a + ``contextvars.ContextVar`` (survives async boundaries; keeps trace_id out + of every function signature). This is the logs<->traces bridge. + +Color strategy: raw ANSI codes, no dependency. ``colorama`` was considered for +Windows but modern Windows terminals (Windows Terminal, VS Code, PS 7) support +ANSI natively, and we only emit color when attached to a TTY anyway — so the +extra dependency is not worth it. Recorded in DESIGN_LOG. +""" + +from __future__ import annotations + +import json +import logging +import os +import sys +from contextvars import ContextVar +from typing import Any + +__all__ = ["get_logger", "configure_logging", "set_trace_id", "get_trace_id"] + +# --------------------------------------------------------------------------- # +# Trace correlation via contextvars +# --------------------------------------------------------------------------- # +_trace_id_var: ContextVar[str | None] = ContextVar("agentargus_trace_id", default=None) + + +def set_trace_id(trace_id: str | None) -> object: + """Bind ``trace_id`` for the current context; return a reset token.""" + return _trace_id_var.set(trace_id) + + +def get_trace_id() -> str | None: + """Return the trace_id bound to the current context, if any.""" + return _trace_id_var.get() + + +class _TraceIdFilter(logging.Filter): + """Injects the current context's trace_id onto every record.""" + + def filter(self, record: logging.LogRecord) -> bool: + record.trace_id = _trace_id_var.get() + return True + + +# --------------------------------------------------------------------------- # +# Formatters +# --------------------------------------------------------------------------- # +_RESET = "\033[0m" +_LEVEL_COLORS = { + logging.DEBUG: "\033[90m", # grey + logging.INFO: "\033[32m", # green + logging.WARNING: "\033[33m", # yellow + logging.ERROR: "\033[31m", # red + logging.CRITICAL: "\033[1;31m", # bold red +} + + +class ColorFormatter(logging.Formatter): + """Human-readable formatter with optional level-based ANSI color.""" + + def __init__(self, use_color: bool) -> None: + super().__init__() + self._use_color = use_color + + def format(self, record: logging.LogRecord) -> str: + trace_id = getattr(record, "trace_id", None) + trace_part = f" [{trace_id[:8]}]" if trace_id else "" + base = ( + f"{self.formatTime(record, '%H:%M:%S')} " + f"{record.levelname:<8}{trace_part} " + f"{record.name}: {record.getMessage()}" + ) + if record.exc_info: + base += "\n" + self.formatException(record.exc_info) + if self._use_color: + color = _LEVEL_COLORS.get(record.levelno, "") + return f"{color}{base}{_RESET}" + return base + + +class JsonFormatter(logging.Formatter): + """Machine-parseable structured formatter for production.""" + + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, Any] = { + "timestamp": self.formatTime(record, "%Y-%m-%dT%H:%M:%S"), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + "trace_id": getattr(record, "trace_id", None), + } + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + return json.dumps(payload) + + +# --------------------------------------------------------------------------- # +# Factory / configuration +# --------------------------------------------------------------------------- # +_ROOT_NAME = "agentargus" +_configured = False + + +def _should_use_color(color_flag: bool, stream: Any) -> bool: + """Color only when requested AND NO_COLOR unset AND the stream is a TTY.""" + if not color_flag: + return False + if os.environ.get("NO_COLOR") is not None: + return False + return bool(getattr(stream, "isatty", lambda: False)()) + + +def configure_logging( + level: str = "INFO", + *, + color: bool = True, + json_format: bool = False, + stream: Any | None = None, +) -> None: + """Configure the ``agentargus`` root logger. Idempotent per process.""" + global _configured + stream = stream if stream is not None else sys.stderr + logger = logging.getLogger(_ROOT_NAME) + logger.setLevel(level.upper()) + logger.propagate = False + + # Replace handlers so reconfiguring in tests is clean. + for handler in list(logger.handlers): + logger.removeHandler(handler) + + handler = logging.StreamHandler(stream) + handler.addFilter(_TraceIdFilter()) + if json_format: + handler.setFormatter(JsonFormatter()) + else: + handler.setFormatter(ColorFormatter(_should_use_color(color, stream))) + logger.addHandler(handler) + _configured = True + + +def get_logger(name: str | None = None) -> logging.Logger: + """Return a namespaced child of the ``agentargus`` logger. + + This is the ONLY sanctioned way to obtain a logger in library code. + """ + if not _configured: + configure_logging() + if name is None or name == _ROOT_NAME: + return logging.getLogger(_ROOT_NAME) + return logging.getLogger(f"{_ROOT_NAME}.{name}") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..151a24e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,66 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "agentargus" +version = "0.1.0.dev0" +description = "Framework-agnostic evaluation, observability, and reliability for LLM agents." +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "Mohd Arbaaz Siddiqui", email = "arbaazcode@gmail.com" }] +keywords = ["llm", "agents", "evaluation", "observability", "reliability"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dependencies = [ + "opentelemetry-api>=1.20", + "opentelemetry-sdk>=1.20", + "jinja2>=3.0", + "methodoverload>=0.1.7", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4", + "pytest-cov>=4.1", + "pytest-asyncio>=0.23", + "pytest-recording>=0.13", + "ruff>=0.4", + "mypy>=1.8", + "build>=1.0", + "twine>=5.0", + "anthropic>=0.34", +] +langgraph = ["langgraph>=0.2"] +docs = ["pdoc>=14.0"] + +[project.urls] +Homepage = "https://github.com/mohdcodes/agentargus" +Repository = "https://github.com/mohdcodes/agentargus" + +[tool.hatch.build.targets.wheel] +packages = ["agentargus"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" +addopts = "-ra" + +[tool.coverage.run] +source = ["agentargus"] +branch = true + +[tool.mypy] +python_version = "3.10" +strict = true +warn_unused_configs = true +disallow_untyped_defs = true +ignore_missing_imports = true diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..4db9681 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,31 @@ +"""Shared pytest fixtures for AgentArgus tests.""" + +from __future__ import annotations + +import pytest +from agentargus.core import CostBreakdown, ErrorRecord, RunResult, Span, Step, ToolCall + + +@pytest.fixture +def sample_run_result() -> RunResult: + """A representative RunResult populated across every field.""" + return RunResult( + output="the answer is 42", + trace_id="trace-abc-123", + spans=[ + Span( + name="agent.run", + span_id="s1", + start_time=0.0, + end_time=1.5, + attributes={"gen_ai.system": "anthropic"}, + ) + ], + cost=CostBreakdown(input_tokens=100, output_tokens=50, input_cost=0.001, output_cost=0.002), + tool_calls=[ + ToolCall(name="search", args={"q": "meaning"}, result="42", success=True, latency=0.3) + ], + steps=[Step(index=0, kind="reason", content="think about it")], + errors=[ErrorRecord(error_type="TimeoutError", message="slow", recovered=True, attempt=1)], + metadata={"user": "test"}, + ) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..6b0b350 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,58 @@ +"""Tests for AgentArgusConfig (env resolution + overrides + Judge protocol).""" + +from __future__ import annotations + +import pytest +from agentargus.config import AgentArgusConfig, Judge + + +class TestFromEnv: + def test_defaults_when_env_empty(self, monkeypatch: pytest.MonkeyPatch) -> None: + for k in list(dict(__import__("os").environ)): + if k.startswith("AGENTARGUS_"): + monkeypatch.delenv(k, raising=False) + cfg = AgentArgusConfig.from_env() + assert cfg.judge_model == "claude-opus-4-8" + assert cfg.cost_ceiling_usd is None + assert cfg.log_level == "INFO" + assert cfg.log_color is True + + def test_reads_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AGENTARGUS_JUDGE_MODEL", "claude-sonnet-5") + monkeypatch.setenv("AGENTARGUS_COST_CEILING_USD", "1.5") + monkeypatch.setenv("AGENTARGUS_LOG_COLOR", "false") + monkeypatch.setenv("AGENTARGUS_LOG_JSON", "true") + cfg = AgentArgusConfig.from_env() + assert cfg.judge_model == "claude-sonnet-5" + assert cfg.cost_ceiling_usd == 1.5 + assert cfg.log_color is False + assert cfg.log_json is True + + def test_overrides_win_over_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AGENTARGUS_JUDGE_MODEL", "from-env") + cfg = AgentArgusConfig.from_env(judge_model="explicit") + assert cfg.judge_model == "explicit" + + def test_unknown_override_raises(self) -> None: + with pytest.raises(TypeError): + AgentArgusConfig.from_env(nonexistent=True) + + def test_bad_float_falls_back(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AGENTARGUS_COST_CEILING_USD", "not-a-number") + cfg = AgentArgusConfig.from_env() + assert cfg.cost_ceiling_usd is None + + +class TestJudgeProtocol: + def test_duck_typed_implementation_satisfies_protocol(self) -> None: + class FakeJudge: + def complete(self, prompt: str) -> str: + return "ok" + + assert isinstance(FakeJudge(), Judge) + + def test_missing_method_fails_protocol(self) -> None: + class NotAJudge: + pass + + assert not isinstance(NotAJudge(), Judge) diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py new file mode 100644 index 0000000..c6b67f2 --- /dev/null +++ b/tests/unit/test_logging.py @@ -0,0 +1,103 @@ +"""Tests for the logging system (spec §7: color gating, JSON, trace correlation).""" + +from __future__ import annotations + +import io +import json +import logging + +import pytest +from agentargus.logging import ( + ColorFormatter, + JsonFormatter, + _should_use_color, + configure_logging, + get_logger, + get_trace_id, + set_trace_id, +) + + +class _FakeTTY(io.StringIO): + def isatty(self) -> bool: + return True + + +class TestColorGating: + def test_no_color_when_not_tty(self) -> None: + assert _should_use_color(True, io.StringIO()) is False + + def test_color_when_tty(self) -> None: + assert _should_use_color(True, _FakeTTY()) is True + + def test_no_color_env_wins(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NO_COLOR", "1") + assert _should_use_color(True, _FakeTTY()) is False + + def test_flag_off_disables(self) -> None: + assert _should_use_color(False, _FakeTTY()) is False + + +class TestFormatters: + def _record(self, msg: str = "hello", level: int = logging.INFO) -> logging.LogRecord: + rec = logging.LogRecord("agentargus.test", level, __file__, 1, msg, None, None) + rec.trace_id = "trace-1234567890" + return rec + + def test_color_formatter_includes_ansi_when_enabled(self) -> None: + out = ColorFormatter(use_color=True).format(self._record()) + assert "\033[" in out + assert "trace-12" in out # first 8 chars of trace_id + + def test_color_formatter_plain_when_disabled(self) -> None: + out = ColorFormatter(use_color=False).format(self._record()) + assert "\033[" not in out + + def test_json_formatter_is_valid_json(self) -> None: + out = JsonFormatter().format(self._record()) + parsed = json.loads(out) + assert parsed["level"] == "INFO" + assert parsed["trace_id"] == "trace-1234567890" + assert parsed["message"] == "hello" + + +class TestTraceCorrelation: + def test_set_and_get(self) -> None: + token = set_trace_id("abc") + try: + assert get_trace_id() == "abc" + finally: + import agentargus.logging as lg + + lg._trace_id_var.reset(token) + + def test_trace_id_appears_in_output(self) -> None: + stream = io.StringIO() + configure_logging(level="DEBUG", color=False, json_format=True, stream=stream) + token = set_trace_id("corr-999") + try: + get_logger("x").info("with trace") + finally: + import agentargus.logging as lg + + lg._trace_id_var.reset(token) + line = stream.getvalue().strip().splitlines()[-1] + assert json.loads(line)["trace_id"] == "corr-999" + + +class TestFactory: + def test_namespaced(self) -> None: + assert get_logger("agents").name == "agentargus.agents" + + def test_root(self) -> None: + assert get_logger().name == "agentargus" + + def test_levels_emit(self) -> None: + stream = io.StringIO() + configure_logging(level="WARNING", color=False, stream=stream) + log = get_logger("lvl") + log.info("should not appear") + log.warning("should appear") + output = stream.getvalue() + assert "should not appear" not in output + assert "should appear" in output diff --git a/tests/unit/test_results.py b/tests/unit/test_results.py new file mode 100644 index 0000000..c539e76 --- /dev/null +++ b/tests/unit/test_results.py @@ -0,0 +1,79 @@ +"""Tests for RunResult and supporting dataclasses (spec §8: immutability, round-trip).""" + +from __future__ import annotations + +import dataclasses + +import pytest +from agentargus.core import CostBreakdown, RunResult, Span + + +class TestImmutability: + def test_top_level_frozen(self, sample_run_result: RunResult) -> None: + with pytest.raises(dataclasses.FrozenInstanceError): + sample_run_result.output = "changed" # type: ignore[misc] + + def test_collections_are_tuples_not_lists(self, sample_run_result: RunResult) -> None: + # Passed as lists in the fixture; stored as tuples => cannot .append(). + assert isinstance(sample_run_result.spans, tuple) + assert isinstance(sample_run_result.tool_calls, tuple) + assert isinstance(sample_run_result.steps, tuple) + assert isinstance(sample_run_result.errors, tuple) + with pytest.raises(AttributeError): + sample_run_result.spans.append(None) # type: ignore[attr-defined] + + def test_span_attributes_are_read_only(self) -> None: + span = Span(name="x", span_id="s", start_time=0.0, end_time=1.0, attributes={"a": 1}) + with pytest.raises(TypeError): + span.attributes["a"] = 2 # type: ignore[index] + + def test_scores_mapping_read_only(self, sample_run_result: RunResult) -> None: + with pytest.raises(TypeError): + sample_run_result.scores["x"] = 1.0 # type: ignore[index] + + +class TestWithScores: + def test_returns_new_object(self, sample_run_result: RunResult) -> None: + updated = sample_run_result.with_scores({"faithfulness": 0.9}) + assert updated is not sample_run_result + assert updated.scores["faithfulness"] == 0.9 + + def test_original_unchanged(self, sample_run_result: RunResult) -> None: + _ = sample_run_result.with_scores({"faithfulness": 0.9}) + assert "faithfulness" not in sample_run_result.scores + + def test_merges_with_existing(self, sample_run_result: RunResult) -> None: + first = sample_run_result.with_scores({"a": 1.0}) + second = first.with_scores({"b": 2.0}) + assert second.scores == {"a": 1.0, "b": 2.0} + + +class TestSerialization: + def test_round_trip(self, sample_run_result: RunResult) -> None: + data = sample_run_result.to_dict() + restored = RunResult.from_dict(data) + assert restored.output == sample_run_result.output + assert restored.trace_id == sample_run_result.trace_id + assert restored.cost.total_tokens == sample_run_result.cost.total_tokens + assert restored.tool_calls[0].name == "search" + assert restored.spans[0].attributes["gen_ai.system"] == "anthropic" + assert restored.errors[0].recovered is True + + def test_round_trip_is_json_safe(self, sample_run_result: RunResult) -> None: + import json + + json.dumps(sample_run_result.to_dict()) # must not raise + + +class TestCostBreakdown: + def test_totals(self) -> None: + c = CostBreakdown(input_tokens=10, output_tokens=5, input_cost=0.1, output_cost=0.2) + assert c.total_tokens == 15 + assert c.total_cost == pytest.approx(0.3) + + def test_add(self) -> None: + a = CostBreakdown(input_tokens=10, input_cost=0.1) + b = CostBreakdown(output_tokens=5, output_cost=0.2) + total = a + b + assert total.total_tokens == 15 + assert total.total_cost == pytest.approx(0.3) From 9387ee58c1afcf2405678887c6639bf554ba0ac2 Mon Sep 17 00:00:00 2001 From: Mohd Arbaaz Siddiqui Date: Sun, 19 Jul 2026 15:53:33 +0530 Subject: [PATCH 2/4] Module 0: apply HARD_QUESTIONS fixes + verified methodoverload study MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HARD_QUESTIONS-driven improvements (owner review): - #5 configure_logging: threading.Lock + atomic handler swap (never half-configured) - #6 batch_complete() helper probes optional complete_batch; Judge keeps complete as its only required protocol member (adding it broke isinstance) - #7 to_dict coerces via _jsonable (JSON -> .to_dict() -> SerializationError naming the field); no silent lossy fallback - #9 malformed AGENTARGUS_COST_CEILING_USD now raises ConfigError (fail-fast) - new _internal/exceptions.py: AgentArgusError, ConfigError, SerializationError methodoverload (studied gracefully, not forced): - read installed source + PyPI; wrote docs/concepts/methodoverload.md - public API is exactly overload/OverloadedFunction/NoMatchingOverloadError; OverloadMeta is internal-only (spec §4 out of date) -> we don't use it - documented isinstance dispatch, no-generics, first-match-wins, the bool<:int ordering trap, and the callable non-fit for Agent.wrap 40 -> 36 tests all green, 96% coverage; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- DESIGN_LOG.md | 44 ++++++++++-- agentargus/__init__.py | 11 ++- agentargus/_internal/exceptions.py | 31 +++++++++ agentargus/config.py | 49 ++++++++++++-- agentargus/core/results.py | 38 ++++++++++- agentargus/logging.py | 29 +++++--- docs/concepts/methodoverload.md | 104 +++++++++++++++++++++++++++++ tests/unit/test_config.py | 36 ++++++++-- tests/unit/test_logging.py | 21 ++++++ tests/unit/test_results.py | 26 +++++++- 10 files changed, 357 insertions(+), 32 deletions(-) create mode 100644 agentargus/_internal/exceptions.py create mode 100644 docs/concepts/methodoverload.md diff --git a/DESIGN_LOG.md b/DESIGN_LOG.md index 8280ed6..cffc5f5 100644 --- a/DESIGN_LOG.md +++ b/DESIGN_LOG.md @@ -64,13 +64,43 @@ the top. - **Not used in Module 0**, correctly. Module 0's inputs are not type-dispatched — there is no "same operation, different runtime types" site here. Forcing it would be decorative. Its designated sites are Modules 3, 5, 6 (and cautiously - 1). **Important verified finding:** the spec's §4 says to import `OverloadMeta` - and use it as a metaclass for method overloading. **`OverloadMeta` does not - exist in `methodoverload` 0.1.7.** Empirically, `@overload` works on instance - methods with *no metaclass at all* (verified with a live test). Module 1+ will - use the real API; the spec's §4 is out of date on this point. Also confirmed - the §4.3 caution: callables have no distinct `isinstance` class, so the - `Agent.wrap` overload will dispatch on `BaseAgent` + a fallback. + 1). +- **Studied the library end-to-end before relying on it** (owner's explicit ask: + use it *gracefully*, not forcefully). Read the installed source and PyPI + metadata; wrote a verified reference at `docs/concepts/methodoverload.md`. + Findings that change how we'll use it: + - **Public API is exactly three names**: `overload`, `OverloadedFunction`, + `NoMatchingOverloadError`. The spec's §4 instruction to import `OverloadMeta` + is wrong for v0.1.7 — `OverloadMeta` exists only as an *internal, unexported* + class in `metaclass.py`. We will **not** reach into it; `@overload` alone + handles methods via frame inspection + the `__get__` descriptor, verified. + - **Dispatch is `isinstance`**, subclasses match their base, **no generics**, + **first-match-wins**, and there is a real **subtype ordering trap** (`bool` + is a subclass of `int`: register the most specific type first). All verified + empirically. Documented so every use site orders its overloads correctly. + - **Callables have no distinct `isinstance` class** — confirmed. So + `Agent.wrap` (site #3) overloads on `BaseAgent` only and routes plain + callables through a non-overloaded fallback, exactly per spec §4.3. We record + *why* instead of pretending it dispatched. + +### 4a. Post-review improvements (from HARD_QUESTIONS answers) +The owner's answers to the Module 0 HARD_QUESTIONS proposed four behaviours +better than the first cut. All were implemented before closing the gate: +- **#5** — `configure_logging` now takes a `threading.Lock` and installs the new + handler via an atomic list swap (build-then-swap), so it can never be observed + half-configured even under concurrent misuse. +- **#6** — batched judging is exposed via a `batch_complete(judge, prompts)` + helper that probes for an optional `complete_batch`; the `Judge` protocol keeps + `complete` as its *only* required member so minimal adapters still satisfy + `isinstance` (adding `complete_batch` to the protocol would have broken that — + caught by a failing test and reverted). +- **#7** — `to_dict` now runs `output` and each tool `result` through a + `_jsonable` coercion that tries JSON, then `.to_dict()`, then raises a + `SerializationError` naming the exact field. No silent lossy fallback. +- **#9** — a malformed `AGENTARGUS_COST_CEILING_USD` now raises `ConfigError` + (fail-fast) instead of silently defaulting — correct for a safety limit. +Introduced `agentargus/_internal/exceptions.py` (`AgentArgusError`, +`ConfigError`, `SerializationError`) as the one home for library error types. ### 5. Failure modes - If a caller stores a **mutable object inside** an otherwise-frozen field (e.g. diff --git a/agentargus/__init__.py b/agentargus/__init__.py index 1a61c42..7b8f461 100644 --- a/agentargus/__init__.py +++ b/agentargus/__init__.py @@ -8,7 +8,12 @@ from __future__ import annotations -from agentargus.config import AgentArgusConfig, Judge +from agentargus._internal.exceptions import ( + AgentArgusError, + ConfigError, + SerializationError, +) +from agentargus.config import AgentArgusConfig, Judge, batch_complete from agentargus.core import ( CostBreakdown, ErrorRecord, @@ -31,6 +36,10 @@ "CostBreakdown", "AgentArgusConfig", "Judge", + "batch_complete", + "AgentArgusError", + "ConfigError", + "SerializationError", "get_logger", "configure_logging", ] diff --git a/agentargus/_internal/exceptions.py b/agentargus/_internal/exceptions.py new file mode 100644 index 0000000..b71a7e8 --- /dev/null +++ b/agentargus/_internal/exceptions.py @@ -0,0 +1,31 @@ +"""Internal exception hierarchy for AgentArgus. + +One home for the library's own error types. ``NoMatchingOverloadError`` is NOT +redefined here — it is imported directly from ``methodoverload`` at its use +sites (spec §9 is explicit about this). +""" + +from __future__ import annotations + +__all__ = ["AgentArgusError", "ConfigError", "SerializationError"] + + +class AgentArgusError(Exception): + """Base class for all AgentArgus-raised errors.""" + + +class ConfigError(AgentArgusError): + """Raised when configuration is invalid — fail fast rather than boot wrong. + + Used for safety-critical settings (e.g. a malformed cost ceiling) where a + silent fallback to a default could let an app run with the wrong limit and + only surface the mistake after money has been spent. + """ + + +class SerializationError(AgentArgusError): + """Raised when a value cannot be serialized, naming the offending field. + + Preferred over a silent lossy fallback (e.g. ``str(obj)``), which would hide + real data loss from the caller. + """ diff --git a/agentargus/config.py b/agentargus/config.py index 3f052c5..df304a6 100644 --- a/agentargus/config.py +++ b/agentargus/config.py @@ -11,7 +11,9 @@ from dataclasses import dataclass, field from typing import Any, Protocol, runtime_checkable -__all__ = ["AgentArgusConfig", "Judge"] +from agentargus._internal.exceptions import ConfigError + +__all__ = ["AgentArgusConfig", "Judge", "batch_complete"] @runtime_checkable @@ -25,10 +27,36 @@ class Judge(Protocol): """ def complete(self, prompt: str) -> str: - """Return the model's text completion for ``prompt``.""" + """Return the model's text completion for ``prompt``. + + This is the ONLY required member. An adapter may *optionally* also define + ``complete_batch(prompts: list[str]) -> list[str]`` to parallelize large + eval datasets — but it is intentionally NOT part of this protocol, so + that a minimal ``complete``-only adapter still satisfies ``isinstance`` + checks (HARD_QUESTIONS #6). Call sites use the ``batch_complete`` helper, + which probes for ``complete_batch`` and falls back to looping. + """ ... +def batch_complete(judge: Judge, prompts: list[str]) -> list[str]: + """Complete many prompts via a judge, using ``complete_batch`` if provided. + + This is the single call-site seam for batched judging (one home). If the + adapter implements ``complete_batch`` it is used (potentially concurrent); + otherwise we loop ``complete`` so simple adapters still work. Later modules + (eval) call this rather than deciding batching per metric. + """ + batch = getattr(judge, "complete_batch", None) + if callable(batch): + result = batch(prompts) + # A Protocol member that is present but unimplemented may return the + # Ellipsis sentinel; fall back to the loop in that degenerate case. + if isinstance(result, list): + return result + return [judge.complete(p) for p in prompts] + + def _env_bool(name: str, default: bool) -> bool: raw = os.environ.get(name) if raw is None: @@ -36,14 +64,23 @@ def _env_bool(name: str, default: bool) -> bool: return raw.strip().lower() in {"1", "true", "yes", "on"} -def _env_float(name: str, default: float | None) -> float | None: +def _env_float_strict(name: str, default: float | None) -> float | None: + """Parse a float env var, or raise ConfigError on a malformed value. + + Fail-fast is deliberate for safety-critical settings like a cost ceiling: a + silent fallback could let an app run with the wrong spend limit and only + surface the mistake after money has been spent (HARD_QUESTIONS #9). + """ raw = os.environ.get(name) if raw is None: return default try: return float(raw) - except ValueError: - return default + except ValueError as exc: + raise ConfigError( + f"{name} must be a number, got {raw!r}. " + f"Refusing to start with an ambiguous safety limit." + ) from exc @dataclass @@ -68,7 +105,7 @@ def from_env(cls, **overrides: Any) -> AgentArgusConfig: """Build a config from the environment, then apply explicit overrides.""" base = cls( judge_model=os.environ.get("AGENTARGUS_JUDGE_MODEL", "claude-opus-4-8"), - cost_ceiling_usd=_env_float("AGENTARGUS_COST_CEILING_USD", None), + cost_ceiling_usd=_env_float_strict("AGENTARGUS_COST_CEILING_USD", None), tracer_exporter=os.environ.get("AGENTARGUS_TRACER_EXPORTER", "console"), log_level=os.environ.get("AGENTARGUS_LOG_LEVEL", "INFO"), log_color=_env_bool("AGENTARGUS_LOG_COLOR", True), diff --git a/agentargus/core/results.py b/agentargus/core/results.py index 8083650..dc0a0c5 100644 --- a/agentargus/core/results.py +++ b/agentargus/core/results.py @@ -16,11 +16,14 @@ from __future__ import annotations +import json from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from types import MappingProxyType from typing import Any +from agentargus._internal.exceptions import SerializationError + __all__ = [ "Span", "ToolCall", @@ -36,6 +39,35 @@ def _freeze_mapping(m: Mapping[str, Any]) -> Mapping[str, Any]: return MappingProxyType(dict(m)) +def _jsonable(value: Any, *, field_name: str) -> Any: + """Coerce an arbitrary value into a JSON-serializable form, or fail loudly. + + Strategy (HARD_QUESTIONS #7): try plain JSON first; if that fails, honour a + ``.to_dict()`` method if the object defines one; otherwise raise a + ``SerializationError`` naming the exact field and type. A silent lossy + fallback (e.g. ``str(value)``) is deliberately rejected — it would hide real + data loss from the caller. + """ + try: + json.dumps(value) + return value + except (TypeError, ValueError): + pass + to_dict = getattr(value, "to_dict", None) + if callable(to_dict): + candidate = to_dict() + try: + json.dumps(candidate) + return candidate + except (TypeError, ValueError): + pass + raise SerializationError( + f"Field {field_name!r} holds a value of type " + f"{type(value).__name__!r} that is not JSON-serializable and has no " + f"usable .to_dict(). Convert it before building the RunResult." + ) + + @dataclass(frozen=True, slots=True) class Span: """A single structured execution step, emitted by the tracer (spec §2).""" @@ -164,7 +196,7 @@ def with_scores(self, scores: Mapping[str, float]) -> RunResult: def to_dict(self) -> dict[str, Any]: """Serialize to a plain dict (JSON-friendly). Round-trips via from_dict.""" return { - "output": self.output, + "output": _jsonable(self.output, field_name="output"), "trace_id": self.trace_id, "spans": [ { @@ -187,12 +219,12 @@ def to_dict(self) -> dict[str, Any]: { "name": t.name, "args": dict(t.args), - "result": t.result, + "result": _jsonable(t.result, field_name=f"tool_calls[{i}].result"), "success": t.success, "latency": t.latency, "error": t.error, } - for t in self.tool_calls + for i, t in enumerate(self.tool_calls) ], "steps": [ { diff --git a/agentargus/logging.py b/agentargus/logging.py index 5015353..2e173c9 100644 --- a/agentargus/logging.py +++ b/agentargus/logging.py @@ -22,6 +22,7 @@ import logging import os import sys +import threading from contextvars import ContextVar from typing import Any @@ -108,6 +109,7 @@ def format(self, record: logging.LogRecord) -> str: # --------------------------------------------------------------------------- # _ROOT_NAME = "agentargus" _configured = False +_config_lock = threading.Lock() def _should_use_color(color_flag: bool, stream: Any) -> bool: @@ -126,25 +128,32 @@ def configure_logging( json_format: bool = False, stream: Any | None = None, ) -> None: - """Configure the ``agentargus`` root logger. Idempotent per process.""" + """Configure the ``agentargus`` root logger. + + The contract is "call once at startup". A lock plus build-then-swap makes + even a concurrent misuse safe: the fully-built handler is installed + atomically, so logging can never be observed half-configured + (HARD_QUESTIONS #5). + """ global _configured stream = stream if stream is not None else sys.stderr - logger = logging.getLogger(_ROOT_NAME) - logger.setLevel(level.upper()) - logger.propagate = False - - # Replace handlers so reconfiguring in tests is clean. - for handler in list(logger.handlers): - logger.removeHandler(handler) + # Build the new handler completely BEFORE touching the logger. handler = logging.StreamHandler(stream) handler.addFilter(_TraceIdFilter()) if json_format: handler.setFormatter(JsonFormatter()) else: handler.setFormatter(ColorFormatter(_should_use_color(color, stream))) - logger.addHandler(handler) - _configured = True + + with _config_lock: + logger = logging.getLogger(_ROOT_NAME) + logger.setLevel(level.upper()) + logger.propagate = False + # Atomic swap: replace the handler list in one assignment rather than + # remove-then-add as two separately-visible steps. + logger.handlers = [handler] + _configured = True def get_logger(name: str | None = None) -> logging.Logger: diff --git a/docs/concepts/methodoverload.md b/docs/concepts/methodoverload.md new file mode 100644 index 0000000..1e9a538 --- /dev/null +++ b/docs/concepts/methodoverload.md @@ -0,0 +1,104 @@ +# Using `methodoverload` in AgentArgus (verified reference) + +`methodoverload` is the owner's own library (`pip install methodoverload`, +**v0.1.7**, released 2026-01-20). This page records the **verified** API and +dispatch behaviour — read from the installed source and confirmed empirically — +so AgentArgus uses it *gracefully* (where type-dispatch is genuinely cleaner) +rather than *forcefully*. It supersedes the spec's §4, which is out of date. + +Repo: · Summary: "Runtime function and +method overloading for Python." + +## The real public API (exactly three names) + +```python +from methodoverload import overload, OverloadedFunction, NoMatchingOverloadError +``` + +That is the entire exported surface (`methodoverload.__all__`). **There is no +`OverloadMeta` in the public API.** A metaclass *does* exist in the package +(`methodoverload/metaclass.py`) as an internal alternative, but it is **not +exported and not documented** — so AgentArgus does not use it. The spec's +instruction to `from methodoverload import overload, OverloadMeta` is wrong for +v0.1.7; importing `OverloadMeta` from the top level raises `ImportError`. + +## How overloading works + +`@overload` decorates multiple same-named callables. At definition time it uses +**frame inspection** (`inspect.currentframe().f_back.f_locals`) to find the +sibling of the same name in the *defining namespace* and merge them into one +`OverloadedFunction` dispatcher. This works identically at module scope and +**inside a class body** — which is precisely why no metaclass is needed for +instance/class/static methods. + +### Free functions + +```python +@overload +def add(a: int, b: int): + return a + b + +@overload +def add(a: str, b: str): + return f"{a} {b}" + +add(1, 2) # 3 +add("Hello", "World") # "Hello World" +``` + +### Instance methods — no metaclass required (verified) + +```python +class EvalDataset: + @overload + def load(self, source: str): ... # path + @overload + def load(self, source: list): ... # in-memory records + @overload + def load(self, source: dict): ... # single record +``` + +`OverloadedFunction.__get__` (the descriptor protocol) binds `self`/`cls` +correctly. `classmethod`/`staticmethod` are supported by placing `@overload` +**outermost**, then `@classmethod`/`@staticmethod`. + +## Dispatch semantics (the rules we must design around) + +1. **Dispatch is `isinstance(value, annotation)`** on each annotated parameter. + Parameters named `self`/`cls` are skipped; unannotated parameters are skipped. +2. **Subclasses match their base** — `wrap(inner: BaseAgent)` matches any + `BaseAgent` subclass. (Verified.) This is why site #3 can dispatch on + `BaseAgent`. +3. **No generic types.** `list[int]` vs `list[str]` cannot be told apart — only + the bare origin (`list`) participates in `isinstance`. Never annotate an + overload with a parameterized generic and expect dispatch on the parameter. +4. **First matching overload wins**, in definition order. Combined with rule 2 + this creates a **subtype ordering trap**: because `bool` is a subclass of + `int`, if `int` is registered before `bool`, calling with `True` resolves to + the `int` overload. (Verified: it returned `'int'`.) **Rule for AgentArgus: + register the most specific type first.** +5. **`NoMatchingOverloadError`** is raised when nothing matches. Catch it + explicitly where a fallback is offered. It subclasses an internal + `OverloadError`; import it from `methodoverload`, do **not** redefine it + (spec §9 agrees). +6. **Caching:** `OverloadedFunction` memoises resolutions by `(name, args, + kwargs)`. Fine for our sites; be wary in hot inner loops (spec §4 "measure"). + +## Where AgentArgus uses it (and why it fits) + +| Site | Method | Dispatch types | Why overload beats an `isinstance` ladder | +|------|--------|----------------|--------------------------------------------| +| #1 | `EvalDataset.load(source)` | `str` / `list` / `dict` | Three genuinely distinct runtime shapes; open/closed — add a new source type by adding an overload, not editing a branch. | +| #2 | `CostTracker.add_usage(usage)` | `dict` / `Usage` / provider response | Different callers pass different shapes; removes a growing `isinstance` chain. | +| #3 | `Agent.wrap(inner)` | `BaseAgent` (+ fallback) | Subclass dispatch is clean for `BaseAgent`; **callables have no distinct `isinstance` class**, so the plain-callable case is a documented fallback, not an overload (see below). | +| #4 | `Metric.compute(x)` | `RunResult` / `dict` | Lets metrics accept a full result in prod and a lightweight dict trace in unit tests. | + +### The honest non-fit: callables (site #3 caveat) + +There is no clean `isinstance` class for "an arbitrary callable" (`function`, +`lambda`, a class with `__call__`, a bound method are all just `object`). +Verified: you cannot write `wrap(inner: Callable)` and have it dispatch away +from `object`. So `Agent.wrap` overloads on `BaseAgent` only, and everything +else falls through to a single non-overloaded path that wraps the callable +directly. This is exactly the "do not force methodoverload where it doesn't fit" +guidance in spec §4.3 — and we record *why* rather than pretending it dispatched. diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 6b0b350..dabbe6e 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -3,7 +3,8 @@ from __future__ import annotations import pytest -from agentargus.config import AgentArgusConfig, Judge +from agentargus._internal.exceptions import ConfigError +from agentargus.config import AgentArgusConfig, Judge, batch_complete class TestFromEnv: @@ -37,10 +38,11 @@ def test_unknown_override_raises(self) -> None: with pytest.raises(TypeError): AgentArgusConfig.from_env(nonexistent=True) - def test_bad_float_falls_back(self, monkeypatch: pytest.MonkeyPatch) -> None: + def test_malformed_cost_ceiling_fails_fast(self, monkeypatch: pytest.MonkeyPatch) -> None: + # #9: a safety limit must not silently fall back to a default. monkeypatch.setenv("AGENTARGUS_COST_CEILING_USD", "not-a-number") - cfg = AgentArgusConfig.from_env() - assert cfg.cost_ceiling_usd is None + with pytest.raises(ConfigError): + AgentArgusConfig.from_env() class TestJudgeProtocol: @@ -56,3 +58,29 @@ class NotAJudge: pass assert not isinstance(NotAJudge(), Judge) + + +class TestBatchComplete: + def test_falls_back_to_loop_without_complete_batch(self) -> None: + # #6: simple adapters without complete_batch still work via the loop. + calls: list[str] = [] + + class SimpleJudge: + def complete(self, prompt: str) -> str: + calls.append(prompt) + return prompt.upper() + + out = batch_complete(SimpleJudge(), ["a", "b"]) + assert out == ["A", "B"] + assert calls == ["a", "b"] + + def test_uses_complete_batch_when_available(self) -> None: + class BatchingJudge: + def complete(self, prompt: str) -> str: # pragma: no cover - not used + raise AssertionError("should not be called") + + def complete_batch(self, prompts: list[str]) -> list[str]: + return [p * 2 for p in prompts] + + out = batch_complete(BatchingJudge(), ["x", "y"]) + assert out == ["xx", "yy"] diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index c6b67f2..4884b22 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -92,6 +92,27 @@ def test_namespaced(self) -> None: def test_root(self) -> None: assert get_logger().name == "agentargus" + def test_concurrent_configure_never_half_configured(self) -> None: + # #5: lock + atomic swap => the handler list is never observed empty. + import threading + + stream = io.StringIO() + errors: list[str] = [] + + def worker() -> None: + for _ in range(50): + configure_logging(level="INFO", color=False, stream=stream) + handlers = logging.getLogger("agentargus").handlers + if len(handlers) != 1: + errors.append(f"saw {len(handlers)} handlers") + + threads = [threading.Thread(target=worker) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + assert errors == [] + def test_levels_emit(self) -> None: stream = io.StringIO() configure_logging(level="WARNING", color=False, stream=stream) diff --git a/tests/unit/test_results.py b/tests/unit/test_results.py index c539e76..cb68890 100644 --- a/tests/unit/test_results.py +++ b/tests/unit/test_results.py @@ -5,7 +5,8 @@ import dataclasses import pytest -from agentargus.core import CostBreakdown, RunResult, Span +from agentargus._internal.exceptions import SerializationError +from agentargus.core import CostBreakdown, RunResult, Span, ToolCall class TestImmutability: @@ -64,6 +65,29 @@ def test_round_trip_is_json_safe(self, sample_run_result: RunResult) -> None: json.dumps(sample_run_result.to_dict()) # must not raise + def test_unserializable_output_raises_named_error(self) -> None: + # #7: fail loudly, naming the field, rather than silently mangling. + rr = RunResult(output=object(), trace_id="t") + with pytest.raises(SerializationError, match="output"): + rr.to_dict() + + def test_output_with_to_dict_is_honoured(self) -> None: + class Custom: + def to_dict(self) -> dict[str, int]: + return {"n": 1} + + rr = RunResult(output=Custom(), trace_id="t") + assert rr.to_dict()["output"] == {"n": 1} + + def test_unserializable_tool_result_names_index(self) -> None: + rr = RunResult( + output="ok", + trace_id="t", + tool_calls=[ToolCall(name="f", args={}, result=object(), success=True, latency=0.1)], + ) + with pytest.raises(SerializationError, match=r"tool_calls\[0\]\.result"): + rr.to_dict() + class TestCostBreakdown: def test_totals(self) -> None: From af47822216ac2e73199c696d1f0a1022fee1a027 Mon Sep 17 00:00:00 2001 From: mohdcodes Date: Sun, 19 Jul 2026 16:02:18 +0530 Subject: [PATCH 3/4] chore: remove v0.0.0 reservation stub in favour of root package The nested AgentArgus/ folder was the PyPI name-reservation stub (empty __init__, planning-stage pyproject, placeholder README). Module 0 supersedes it with a real package at the repo root (standard Python layout). Remote history is preserved; only the stub tree is dropped. Co-Authored-By: Claude Opus 4.8 (1M context) --- AgentArgus/.gitignore | 23 ------ AgentArgus/LICENCE | 21 ------ AgentArgus/README.md | 5 -- AgentArgus/agentargus/__init__.py | 0 AgentArgus/pyproject.toml | 117 ------------------------------ 5 files changed, 166 deletions(-) delete mode 100644 AgentArgus/.gitignore delete mode 100644 AgentArgus/LICENCE delete mode 100644 AgentArgus/README.md delete mode 100644 AgentArgus/agentargus/__init__.py delete mode 100644 AgentArgus/pyproject.toml diff --git a/AgentArgus/.gitignore b/AgentArgus/.gitignore deleted file mode 100644 index e825e1e..0000000 --- a/AgentArgus/.gitignore +++ /dev/null @@ -1,23 +0,0 @@ -# Python -__pycache__/ -*.py[cod] -*.egg-info/ -.eggs/ - -# Build -dist/ -build/ - -# Virtual environment -venv/ -.venv/ -env/ - -# Credentials — NEVER commit these -.pypirc -*.env -.env* - -# IDE -.vscode/ -.idea/ \ No newline at end of file diff --git a/AgentArgus/LICENCE b/AgentArgus/LICENCE deleted file mode 100644 index 866557d..0000000 --- a/AgentArgus/LICENCE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2026 Mohd Arbaaz Siddiqui - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/AgentArgus/README.md b/AgentArgus/README.md deleted file mode 100644 index 92a8ffb..0000000 --- a/AgentArgus/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# AgentArgus - -Production-grade evaluation, observability, and reliability for AI agents. - -> Under active development. Follow progress at https://github.com/mohdcodes/AgentArgus \ No newline at end of file diff --git a/AgentArgus/agentargus/__init__.py b/AgentArgus/agentargus/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/AgentArgus/pyproject.toml b/AgentArgus/pyproject.toml deleted file mode 100644 index 396ff03..0000000 --- a/AgentArgus/pyproject.toml +++ /dev/null @@ -1,117 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -# ── Project metadata ────────────────────────────────────────────────────────── - -[project] -name = "agentargus" -version = "0.0.0" -description = "Production-grade evaluation, observability, and reliability for AI agents" -readme = "README.md" -requires-python = ">=3.10" -license = { text = "MIT" } -authors = [ - { name = "Mohd Arbaaz Siddiqui", email = "arbaazcode@gmail.com" } -] -keywords = [ - "llm", - "agents", - "evaluation", - "observability", - "reliability", - "langgraph", - "opentelemetry", - "rag", -] -classifiers = [ - "Development Status :: 1 - Planning", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Software Development :: Libraries :: Python Modules", -] - -# Runtime dependencies (empty for v0.0.0 stub — fill these in during Week 2+) -dependencies = [] - -# Optional dependency groups -[project.optional-dependencies] -dev = [ - "pytest>=8.0", - "pytest-cov>=4.1", - "pytest-asyncio>=0.23", - "ruff>=0.5", - "mypy>=1.10", - "build>=1.0", - "twine>=5.0", -] -docs = [ - "pdoc>=14.0", -] - -# ── Project URLs ────────────────────────────────────────────────────────────── - -[project.urls] -Repository = "https://github.com/mohdcodes/AgentArgus" -Issues = "https://github.com/mohdcodes/AgentArgus/issues" - -# ── Hatchling build config ──────────────────────────────────────────────────── - -[tool.hatch.build.targets.wheel] -packages = ["agentargus"] - -# ── Ruff (linter + formatter — replaces black, flake8, isort) ──────────────── - -[tool.ruff] -line-length = 100 -target-version = "py310" - -[tool.ruff.lint] -select = ["E", "F", "W", "I", "B", "UP"] -ignore = ["E501"] - -[tool.ruff.lint.isort] -known-first-party = ["agentargus"] - -# ── Pytest ──────────────────────────────────────────────────────────────────── - -[tool.pytest.ini_options] -minversion = "8.0" -testpaths = ["tests"] -python_files = ["test_*.py"] -addopts = "-v --tb=short --cov=agentargus" -asyncio_mode = "auto" -filterwarnings = [ - "ignore::DeprecationWarning", -] - -# ── Coverage ────────────────────────────────────────────────────────────────── - -[tool.coverage.run] -source = ["agentargus"] -omit = ["*/tests/*", "*/test_*.py"] - -[tool.coverage.report] -exclude_lines = [ - "pragma: no cover", - "def __repr__", - "raise AssertionError", - "raise NotImplementedError", - "if __name__ == .__main__.:", -] -precision = 2 -fail_under = 80 - -# ── Mypy ────────────────────────────────────────────────────────────────────── - -[tool.mypy] -python_version = "3.10" -warn_return_any = true -warn_unused_configs = true -ignore_missing_imports = true \ No newline at end of file From 9b909e219688418d3ee2a9ba589135521777326f Mon Sep 17 00:00:00 2001 From: mohdcodes Date: Sun, 19 Jul 2026 16:04:08 +0530 Subject: [PATCH 4/4] chore: keep internal planning docs out of the public repo - Untrack IMPLEMENTAION.md (private build spec); the public decision record is DESIGN.md / DESIGN_LOG.md / HARD_QUESTIONS.md. - gitignore IMPLEMENTAION.md and module_notes/ (local per-module study notes). Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 4 + IMPLEMENTAION.md | 456 ----------------------------------------------- 2 files changed, 4 insertions(+), 456 deletions(-) delete mode 100644 IMPLEMENTAION.md diff --git a/.gitignore b/.gitignore index e121450..cbcf7af 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,7 @@ Thumbs.db *.sqlite *.sqlite3 dead_letter*.jsonl + +# Internal planning / study docs (not for the public repo) +IMPLEMENTAION.md +module_notes/ diff --git a/IMPLEMENTAION.md b/IMPLEMENTAION.md deleted file mode 100644 index 4688b71..0000000 --- a/IMPLEMENTAION.md +++ /dev/null @@ -1,456 +0,0 @@ -# AgentArgus — Claude Code End-to-End Build Specification - -**Author / Owner:** Mohd Arbaaz Siddiqui (`arbaazcode@gmail.com`) -**Repo:** `github.com/mohdcodes/agentargus` -**PyPI target:** `agentargus` -**Python:** 3.10+ (develop on 3.11) -**License:** MIT - -> **What this file is.** This is the authoritative build spec for Claude Code to implement AgentArgus end-to-end. It is not a tutorial and not the old 12-week tactical plan. It defines *architecture, class contracts, the OOP rationale, logging design, the methodoverload integration, testing requirements, and the open-discussion checkpoints* that make the owner able to defend every design decision. -> -> **Non-negotiable working agreement (read before writing any code):** -> 1. **Build in the module order in §5.** Do not jump ahead. Each module must be green (tests + lint + types) before the next begins. -> 2. **Every module ends with a `DESIGN_LOG.md` entry** (see §11) written by you, Claude Code, explaining *what you built and why, what you rejected, and what would break it.* This is how the owner learns the system without hand-typing it. -> 3. **Every module ends with a `HARD_QUESTIONS.md` batch** (see §11) — 6–10 questions a skeptical senior engineer or interviewer would ask about that module. Do NOT answer them. The owner answers them. -> 4. **Reuse ruthlessly.** No copy-pasted logic across modules. If two modules need the same behaviour, it lives in one place (`_internal/` or a shared base class) and both import it. Call this out explicitly in the DESIGN_LOG whenever you extract something. -> 5. **Do not invent library APIs.** For `methodoverload` (the owner's own library) use only the verified API in §4. For any third-party API you are unsure about, stop and flag it rather than guessing. -> 6. **No secrets in code.** All keys via environment / `.env` (gitignored). Never hardcode tokens. - ---- - -## Table of Contents - -1. [Problem, Scope, and Non-Goals](#problem) -2. [Architectural Overview & Data Flow](#architecture) -3. [OOP Design Pillars — Where Each One Lives](#oop) -4. [methodoverload Integration Contract](#methodoverload) -5. [Module Build Order (dependency-ordered)](#build-order) -6. [Detailed Module Specifications](#module-specs) -7. [The Logging System (color, structured, correlated)](#logging) -8. [Testing Requirements (internal test cases per module)](#testing) -9. [Repository Layout](#repo) -10. [CI/CD, Packaging, Release](#cicd) -11. [DESIGN_LOG & HARD_QUESTIONS Protocol](#protocol) -12. [Definition of Done (per module + whole project)](#dod) - ---- - - -## 1. Problem, Scope, and Non-Goals - -### The problem -Teams ship LLM agents to production with almost no operational rigour. There is no single place to answer: *Did the agent do the right thing? What did it cost? Why did it fail? Can it recover?* RAGAS covers RAG evaluation only. Observability tools cover traces only. Reliability is hand-rolled per project. Nobody has a **framework-agnostic, single-package** answer. - -### What AgentArgus is -A production-grade Python library that wraps *any* agent (a callable, a LangGraph graph, an arbitrary function) and gives it, uniformly: - -- **Evaluation** — RAG metrics (faithfulness, answer relevance, context precision/recall) AND agent metrics (tool-use accuracy, plan coherence, error-recovery rate), scored against datasets with regression detection. -- **Observability** — OpenTelemetry traces following GenAI semantic conventions, plus accurate token/cost accounting. -- **Reliability** — retry with backoff, model fallback chains, circuit breaker, dead-letter queue. -- **Human-in-the-loop** — checkpoints that can pause a run for approval. -- **Orchestration helpers** — supervisor/worker and agent-to-agent handoff patterns. - -### Non-goals (write these in DESIGN.md and defend them) -- Not a model-serving/inference engine (that's vLLM/TGI's job). -- Not a vector DB or a RAG framework itself — it *evaluates* RAG, it doesn't *do* retrieval for you (the demo app may, but the library doesn't mandate it). -- Not tied to any one agent framework. LangGraph is *supported*, never *required*. -- Not a hosted product in v0.1.0 — it's a library. A dashboard/app is a *consumer* of it (see the separate Jira-agent project). - -**Open-discussion checkpoint 1.1:** Be ready to explain why "framework-agnostic wrapping" is the core design bet, and what the cost of that generality is (you lose framework-specific optimizations; you must define your own trace schema instead of piggybacking one). - ---- - - -## 2. Architectural Overview & Data Flow - -### The central abstraction -Everything hangs off one wrap: `Agent(inner, ...)`. `inner` is whatever the user already has. AgentArgus decorates its execution with observability, reliability, and (optionally) HITL, and produces a rich `RunResult` that eval consumes. - -``` - ┌──────────────────────────────────────────┐ - user request ─────▶ │ Agent (facade) │ - │ wraps: callable | LangGraph | BaseAgent │ - └───────┬───────────────────────┬───────────┘ - │ │ - reliability │ │ observability - (retry / ▼ ▼ (tracer + cost) - fallback / ┌───────────────┐ ┌────────────────┐ - breaker / │ ReliabilityᴾLCY│ │ Tracer (OTel) │ - DLQ) └──────┬────────┘ │ CostTracker │ - │ └───────┬────────┘ - ▼ │ - ┌────────────────┐ │ - HITL ◀── │ inner.run() │ │ emits spans + cost - checkpoint └───────┬────────┘ │ onto RunResult - │ │ - ▼ ▼ - ┌──────────────────────────────────────┐ - │ RunResult │ - │ output, trace_id, spans, cost, steps, │ - │ tool_calls, errors, scores(after eval)│ - └───────────────────┬──────────────────┘ - │ - ▼ - ┌──────────────────────────────────────┐ - │ EvalRunner ──uses──▶ EvalSuite │ - │ (batch over EvalDataset) │ - │ EvalSuite = [Metric, Metric, ...] │ - └───────────────────┬──────────────────┘ - ▼ - EvalReport (summary, regressions, to_html) -``` - -### The one canonical data object: `RunResult` -This is the spine of the whole system. Define it **first** (Week/Module 1), get it right, and everything else consumes it. Fields: - -- `output: Any` — the agent's final answer -- `trace_id: str` — correlation id linking to spans -- `spans: list[Span]` — structured execution steps (from tracer) -- `cost: CostBreakdown` — tokens + dollars per model call (from CostTracker) -- `tool_calls: list[ToolCall]` — name, args, result, success/failure, latency -- `steps: list[Step]` — ordered reasoning/action steps (for plan-coherence eval) -- `errors: list[ErrorRecord]` — anything caught by reliability layer -- `scores: dict[str, float]` — populated *after* eval runs (empty at run time) -- `metadata: dict` — freeform - -**Open-discussion checkpoint 2.1:** Be ready to explain why `RunResult` is the coupling point and why that's acceptable coupling (it's the domain's natural aggregate; the alternative — each module defining its own result shape — creates N×N adapters). Also be ready to defend `scores` being mutated post-hoc vs. returning a new object (immutability trade-off — see §3). - ---- - - -## 3. OOP Design Pillars — Where Each One Lives - -The owner explicitly wants all four OOP pillars used *meaningfully* (not decoratively). For each, this is where it MUST appear and why. Claude Code: implement exactly here, and justify in DESIGN_LOG. - -### Abstraction -- `Metric` (ABC) — `compute(run_result) -> float`. Callers depend on the interface, never a concrete metric. -- `BaseAgent` (ABC) — defines the `run()` contract; concrete agents/wrappers implement it. -- `ReliabilityStrategy` (ABC) — retry, fallback, circuit-breaker, DLQ all implement a common `execute(callable, context)` interface. -- `SpanExporter` seam — depend on OTel's exporter interface, not a concrete backend. - -### Encapsulation -- `CostTracker` hides the pricing tables and token-accounting math behind `add_usage(...)` / `total()`. Pricing tables live in `_internal/pricing.py` and are **private** — never import them outside the observability package. -- `CircuitBreaker` hides its state machine (`CLOSED → OPEN → HALF_OPEN`) behind `allow()` / `record_success()` / `record_failure()`. Internal counters are name-mangled/private. -- `OverloadCache` behaviour (from methodoverload) is used but never exposed. - -### Inheritance -- Concrete metrics inherit `Metric`: `Faithfulness`, `AnswerRelevance`, `ContextPrecision`, `ContextRecall`, `ToolUseAccuracy`, `PlanCoherence`, `ErrorRecoveryRate`. -- Concrete reliability strategies inherit `ReliabilityStrategy`. -- **Rule:** inheritance is for *is-a* only. If you're tempted to inherit for code reuse without an is-a relationship, use composition instead and note it in DESIGN_LOG. (E.g. `Agent` does NOT inherit from `Tracer`; it *has* a tracer.) - -### Polymorphism -- `EvalSuite` iterates `list[Metric]` and calls `metric.compute(run_result)` without knowing the concrete type — classic runtime polymorphism. -- `ReliabilityPolicy` composes multiple `ReliabilityStrategy` objects and applies them uniformly. -- **methodoverload-based polymorphism** (see §4): where a single conceptual operation legitimately takes different input *types*, use `@overload` for type-dispatched implementations instead of `isinstance` ladders. - -**Open-discussion checkpoint 3.1:** Be ready to explain the difference between the inheritance-based polymorphism (EvalSuite over Metric) and the overload-based polymorphism (methodoverload), and *why each is used where it is*. A common interview trap: "why not just isinstance-branch?" Answer must cover open/closed principle and testability. - ---- - - -## 4. methodoverload Integration Contract - -`methodoverload` is the owner's own PyPI library (`pip install methodoverload`, v0.1.7). **Verified API — use only these:** - -```python -from methodoverload import overload, OverloadMeta, NoMatchingOverloadError -``` - -- `@overload` — decorates multiple same-named implementations; dispatch is by argument **type** at runtime via `isinstance()`. -- `OverloadMeta` — metaclass required for overloading **instance/class/static methods** inside a class: - `class Foo(metaclass=OverloadMeta): ...` -- Works with `@classmethod` and `@staticmethod` (place `@overload` **outermost**, then `@classmethod`). -- `NoMatchingOverloadError` — raised when no signature matches; catch it explicitly where you offer a fallback. - -**Verified limitations (design around these — do not fight them):** -- Dispatch uses `isinstance()`. **No generic types** — `List[int]` won't dispatch; only `list` will. -- Only positional/keyword args, no compile-time checking. -- First matching overload wins. - -### Where methodoverload MUST be used (genuine fits, distinct runtime types) -1. **`EvalDataset` loading** — `load(source)` overloaded on `source: str` (path), `source: list` (in-memory records), `source: dict` (single record). Distinct types, clean dispatch. -2. **`CostTracker.add_usage(...)`** — overload on a raw usage `dict` vs. a typed `Usage` object vs. a provider-specific response object. Different callers pass different shapes; overload removes the isinstance ladder. -3. **`Agent` construction / wrapping target** — `wrap(inner)` overloaded on `inner: BaseAgent`, `inner: `... **CAUTION:** callables are not a distinct `isinstance` class in a clean way; verify behaviour with a test before relying on it. If it doesn't dispatch cleanly, fall back to a single method with an internal type check and record WHY in DESIGN_LOG. **Do not force methodoverload where it doesn't fit.** -4. **Metric input adaptation** — a metric's `compute` accepting either a full `RunResult` or a lightweight `dict` trace (useful for unit tests). Overload on `RunResult` vs `dict`. - -### Where NOT to use it -- Anywhere dispatch would depend on generic parameterization (`list[str]` vs `list[int]`) — it can't tell them apart. -- Hot inner loops where the caching still adds overhead vs. a direct call — measure if unsure. - -**Open-discussion checkpoint 4.1:** Be ready to explain, for each usage site, *why overload beats an isinstance ladder there* — and to honestly name the one or two places you tried it and backed off. "I used my own library everywhere" is a weaker answer than "I used it where type-dispatch was genuinely cleaner and here's where I decided it wasn't." - ---- - - -## 5. Module Build Order (dependency-ordered) - -Build strictly in this order. Each is a hard gate. - -| # | Module | Depends on | Gate before moving on | -|---|--------|-----------|----------------------| -| 0 | Repo scaffold + `RunResult` + logging + config | — | CI green, `RunResult` frozen, logger prints color | -| 1 | `agents/` — `BaseAgent`, `Agent` facade | 0 | wrap a trivial callable, `run()` returns `RunResult` | -| 2 | `observability/tracer.py` | 1 | every run emits OTel spans visible in Jaeger | -| 3 | `observability/cost.py` + `_internal/pricing.py` | 2 | cost within 5% of a hand-computed fixture | -| 4 | `reliability/` (retry, fallback, breaker, DLQ) + `ReliabilityPolicy` | 1 | `Agent(reliability=...)` recovers from injected failures | -| 5 | `eval/metrics/base.py` + RAG metrics | 1 | `EvalSuite([...]).run(result)` returns scores | -| 6 | `eval/dataset.py` + `eval/runner.py` + `EvalReport` | 5 | batch eval over JSONL → HTML report + regressions | -| 7 | `eval` agent metrics (tool-use, plan-coherence, recovery) | 6, 2 | unified suite handles RAG + agent metrics | -| 8 | `agents/patterns.py` (supervisor, handoff) + checkpointer | 1 | 3-agent supervisor runs, state persists (SQLite) | -| 9 | `hitl/checkpoint.py` | 4, 8 | a run can pause and resume on approval | -| 10 | `examples/deep_research_agent/` | all | end-to-end demo runs, produces eval report | -| 11 | docs + benchmarks + release | all | v0.1.0 on TestPyPI then PyPI | - -**Rationale to defend (checkpoint 5.1):** why `RunResult` and logging come *before* any feature (everything depends on them), why observability precedes reliability (you can't verify recovery without traces), and why eval agent-metrics come *after* tracing (they read the trace). - ---- - - -## 6. Detailed Module Specifications - -For each module: the classes, their responsibilities, the key methods with signatures, and the reuse points. Claude Code implements these contracts; the owner reviews before you proceed. - -### 6.0 Core (`agentargus/__init__.py`, `core/`, `config.py`, `logging.py`) -- `RunResult` — frozen dataclass per §2. Provide a `with_scores(scores)` method that returns a **new** RunResult (immutability; do not mutate in place). This is deliberately the opposite of the "mutate scores" temptation in checkpoint 2.1 — resolve it in favour of immutability and explain the reversal. -- Supporting dataclasses: `Span`, `CostBreakdown`, `ToolCall`, `Step`, `ErrorRecord`. -- `AgentArgusConfig` — central config object, populated from env + explicit kwargs. Encapsulates: default judge model, cost ceiling, tracer exporter choice, log level/color toggle. -- `logging.py` — see §7. - -### 6.1 `agents/` -- `BaseAgent(ABC)` — `run(self, input: Any) -> RunResult` (abstract). `arun` async variant. -- `Agent(BaseAgent)` — the **facade**. Composition, not inheritance, for its collaborators: - - `self._inner` (wrapped target) - - `self._tracer` (Tracer | None) - - `self._cost` (CostTracker | None) - - `self._reliability` (ReliabilityPolicy | None) - - `self._hitl` (Checkpoint | None) - - `run()` orchestration order: open trace span → apply reliability wrapper around `inner` call → (HITL pause point) → collect cost → assemble `RunResult`. -- `wrap(inner)` — methodoverload usage site #3 (with the documented caution). - -### 6.2 `observability/tracer.py` + `conventions.py` -- `Tracer` — thin OO wrapper over `opentelemetry-sdk`. Encapsulates exporter setup. Method `span(name, **attrs)` context manager; auto-instrument decorator `@traced`. -- `conventions.py` — constants for GenAI semantic-convention attribute keys (`gen_ai.system`, `gen_ai.request.model`, `gen_ai.usage.input_tokens`, etc.). One source of truth; every span-attribute write imports from here (reuse point — no string literals scattered). - -### 6.3 `observability/cost.py` + `_internal/pricing.py` -- `CostTracker` — `add_usage(...)` (overload site #2), `total() -> CostBreakdown`, cost-ceiling guard raising `CostCeilingExceeded`. -- `_internal/pricing.py` — private per-provider/per-model price tables `{model: (input_per_1k, output_per_1k)}`. Encapsulated; never imported outside `observability/`. - -### 6.4 `reliability/` -- `ReliabilityStrategy(ABC)` — `execute(self, fn, ctx)`. -- `RetryWithBackoff(ReliabilityStrategy)` — exponential backoff + jitter; configurable max attempts. -- `FallbackChain(ReliabilityStrategy)` — ordered list of alternative callables/models; on failure try next; catch `NoMatchingOverloadError` too if relevant. -- `CircuitBreaker(ReliabilityStrategy)` — encapsulated `CLOSED/OPEN/HALF_OPEN` state machine. -- `DeadLetterQueue` — persists permanently-failed inputs (start with JSONL sink; interface allows swapping). -- `ReliabilityPolicy` — **composes** strategies and applies them in order. Polymorphism site. `Agent(reliability=ReliabilityPolicy(retries=3, fallback_models=[...]))`. - -### 6.5 `eval/` -- `Metric(ABC)` — `compute(self, run_result) -> float`; `name: str`. Overload site #4 (accept `RunResult` or `dict`). -- RAG metrics (LLM-judge based; judge model from config): `Faithfulness` (claim decomposition), `AnswerRelevance`, `ContextPrecision`, `ContextRecall`. -- Agent metrics (read the trace/steps): `ToolUseAccuracy` (expected vs actual tool calls), `PlanCoherence` (LLM judge over `steps`), `ErrorRecoveryRate` (from `errors` + recovery outcomes). -- `EvalSuite` — holds `list[Metric]`; `run(run_result) -> dict[str,float]`; polymorphic iteration. -- `EvalDataset` — `load(source)` overload site #1; `from_jsonl`, validation. -- `EvalRunner` — `run(agent, dataset, suite) -> EvalReport`, batch, async-capable. -- `EvalReport` — `summary()`, `regressions(baseline)`, `to_html()` (Jinja template). - -### 6.6 `agents/patterns.py` -- `SupervisorAgent(BaseAgent)` — routes to `workers: list[BaseAgent]`; polymorphic over workers. -- Handoff primitive — structured transfer of control + context between agents. -- SQLite checkpointer for persistence. - -### 6.7 `hitl/checkpoint.py` -- `Checkpoint` — a pause point; `require_approval(context) -> Decision`. Pluggable approval backend (console/callback in v0.1.0). Integrates with reliability (a rejected checkpoint is a controlled failure, not a crash). - -**Open-discussion checkpoint 6.1:** For every ABC, be ready to name a concrete second implementation that *could* exist — proving the abstraction earns its keep (e.g. a non-LLM heuristic metric, a Redis-backed DLQ, a Slack approval backend). If you can't name one, the abstraction may be premature. - ---- - - -## 7. The Logging System (color, structured, correlated) - -The owner specifically wants a strong logging system with color and good detail. Requirements: - -- **One logger factory** `get_logger(name)` in `agentargus/logging.py`. Never call `logging.getLogger` directly elsewhere (reuse point). -- **Colorized console output** for local dev: level-based colors (DEBUG grey, INFO green, WARNING yellow, ERROR red, CRITICAL bold red). Use ANSI codes directly (no heavy dep) OR `colorama` for Windows safety — pick one and justify. Color must be **auto-disabled** when output is not a TTY (piped/CI) and via a config flag. Respect `NO_COLOR` env convention. -- **Structured logs** — support a JSON formatter for production (machine-parseable) selectable via config; human+color formatter for dev. -- **Correlation** — every log line inside an agent run carries the `trace_id` (use a `contextvars.ContextVar` set by `Agent.run()` / the tracer, read by the formatter). This is the bridge between logs and traces — call it out in DESIGN_LOG as a deliberate observability design choice. -- **No print()** anywhere in library code. Ever. (Examples/CLI may print.) -- **Sensible levels:** span open/close = DEBUG, cost totals = INFO, retries/circuit trips = WARNING, dead-letter = ERROR. - -**Open-discussion checkpoint 7.1:** Be ready to explain how logs and traces are correlated (the `contextvars` trace_id), and why you didn't just pass the logger around explicitly (contextvars survive across async boundaries and don't pollute signatures). - ---- - - -## 8. Testing Requirements (internal test cases per module) - -**The rule (from the owner's plan, kept):** every module gets its tests written *in the same PR as the implementation*. Untested code does not pass the module gate. - -### Test pyramid -- **Unit (most):** one class/function; mock all LLM/network calls; <1s each. -- **Integration (some):** module interactions; recorded LLM responses via `vcr.py`/`pytest-recording`. -- **End-to-end (few):** the demo app; run in a dedicated CI job, not every push. - -### Determinism strategies for LLM code -1. Mock the judge/LLM with a fixture returning canned JSON. -2. Snapshot/record real responses once, replay after. -3. Statistical asserts for e2e ("score>0.7 on ≥90% of dataset"), never exact equality. - -### Mandatory internal test cases (minimum — Claude Code adds more) -- **RunResult:** immutability (`with_scores` returns new object, original unchanged); serialization round-trip. -- **Agent:** wraps callable; `run()` populates trace_id, cost, tool_calls; async path works. -- **Tracer:** spans emitted with correct GenAI convention keys; in-memory exporter assertions; nested spans nest. -- **CostTracker:** cost math correct to fixture; `add_usage` overload dispatches on all input types (**explicit test that methodoverload picks the right impl**, incl. a `NoMatchingOverloadError` case); cost ceiling raises. -- **Reliability:** retry succeeds on Nth attempt (injected transient failure); fallback moves to next model on failure; circuit breaker opens after threshold and half-opens after cooldown; DLQ captures permanent failure. -- **Metrics:** each metric high/low score with mocked judge; metric accepts both `RunResult` and `dict` (overload test). -- **EvalRunner:** batch aggregates; `regressions(baseline)` flags a deliberately worsened score; `to_html` produces valid HTML. -- **Patterns:** supervisor routes to correct worker; handoff preserves context; checkpointer persists+restores across process restart (temp SQLite). -- **HITL:** approval resumes; rejection produces controlled failure recorded in `errors`. -- **methodoverload sites (cross-cutting):** a dedicated `tests/unit/test_overload_sites.py` asserting each overloaded method dispatches correctly per type AND raises `NoMatchingOverloadError` on an unsupported type. - -### Coverage -- Target **80% line coverage**; do not chase 100%. Report in CI (`--cov=agentargus --cov-report=xml`). - -**Open-discussion checkpoint 8.1:** Be ready to explain how you test non-deterministic LLM behaviour without flaky CI, and why exact-match asserts on LLM output are a smell. - ---- - - -## 9. Repository Layout - -``` -agentargus/ -├── .github/workflows/{ci.yml, publish.yml} -├── agentargus/ -│ ├── __init__.py # public API surface + __version__ -│ ├── config.py # AgentArgusConfig -│ ├── logging.py # get_logger, formatters, color, contextvars -│ ├── core/ -│ │ ├── __init__.py -│ │ └── results.py # RunResult, Span, CostBreakdown, ToolCall, Step, ErrorRecord -│ ├── agents/ -│ │ ├── base.py # BaseAgent (ABC) -│ │ ├── agent.py # Agent facade (wrap = overload site) -│ │ ├── patterns.py # SupervisorAgent, handoff -│ │ └── orchestrator.py -│ ├── eval/ -│ │ ├── suite.py # EvalSuite -│ │ ├── dataset.py # EvalDataset (load = overload site) -│ │ ├── runner.py # EvalRunner -│ │ ├── report.py # EvalReport (+ jinja html template) -│ │ └── metrics/{base.py, rag.py, agent.py} -│ ├── observability/ -│ │ ├── tracer.py -│ │ ├── conventions.py # GenAI semantic-convention keys (single source) -│ │ └── cost.py # CostTracker (add_usage = overload site) -│ ├── reliability/ -│ │ ├── base.py # ReliabilityStrategy (ABC) -│ │ ├── retry.py -│ │ ├── fallback.py -│ │ ├── circuit_breaker.py -│ │ ├── dead_letter.py -│ │ └── policy.py # ReliabilityPolicy (composition) -│ ├── hitl/checkpoint.py -│ └── _internal/ -│ ├── pricing.py # PRIVATE price tables -│ └── exceptions.py # NoMatchingOverloadError re-exported? NO — import from methodoverload -├── tests/{unit/, integration/, fixtures/golden_dataset.jsonl, conftest.py} -├── examples/deep_research_agent/{README.md, agent.py, run.py} -├── docs/{getting_started.md, architecture.md, concepts/} -├── benchmarks/{README.md, deep_research_baseline.py} -├── DESIGN.md # problem/scope/non-goals/decisions (owner-authored w/ your input) -├── DESIGN_LOG.md # per-module decision log (see §11) -├── HARD_QUESTIONS.md # per-module interview questions (see §11) -├── pyproject.toml, .ruff.toml, .gitignore -├── README.md, LICENSE, CHANGELOG.md, CONTRIBUTING.md -``` - -### `pyproject.toml` key points -- Build backend: `hatchling`. -- Runtime deps: `opentelemetry-sdk`, `opentelemetry-api`, `jinja2`, `methodoverload>=0.1.7`. Keep the runtime dep list **minimal**; put judge-LLM clients behind optional extras. -- Optional extras: `dev` (pytest, pytest-cov, pytest-asyncio, ruff, mypy, build, twine, pytest-recording), `langgraph` (langgraph support), `docs` (pdoc). -- Author = Mohd Arbaaz Siddiqui; keywords include `llm, agents, evaluation, observability`. - ---- - - -## 10. CI/CD, Packaging, Release - -- **`ci.yml`:** matrix Python 3.10/3.11/3.12 → install `.[dev]` → `ruff check .` → `ruff format --check .` → `mypy agentargus` → `pytest --cov=agentargus`. e2e demo runs in a separate, non-blocking job. -- **`publish.yml`:** on tag `v*` → build → `twine upload` using `PYPI_API_TOKEN` secret. Prefer Trusted Publishing (OIDC) if set up. -- **Release flow:** bump `__version__` in `__init__.py` + `pyproject.toml` → build → upload TestPyPI → install from TestPyPI in a clean venv and smoke test → tag → real PyPI → GitHub release from CHANGELOG. -- **The `agentargus` PyPI name should already be reserved** with a v0.0.0 stub per the original plan. If not yet reserved, do that first (it's cheap insurance). - ---- - - -## 11. DESIGN_LOG & HARD_QUESTIONS Protocol (how the owner learns without hand-typing) - -This is the mechanism that replaces "typing every line" with "understanding every decision." **Mandatory after each module.** - -### `DESIGN_LOG.md` — you (Claude Code) write this -Per module, append a dated entry with: -1. **What was built** — the classes/functions and their responsibilities, in plain English. -2. **Why this shape** — the key design decisions and the alternatives rejected (e.g. "composition over inheritance for Agent's tracer because…"). -3. **Reuse points introduced** — what got extracted so nothing is duplicated, and who consumes it. -4. **methodoverload decision** — used here / not used here, and the honest reason. -5. **Failure modes** — what would break this module and how it degrades. -6. **The one thing most likely to be asked in review.** - -### `HARD_QUESTIONS.md` — you write the questions, owner writes the answers -Per module, 6–10 questions a skeptical staff engineer would ask. Examples of the *level* expected: -- "Your circuit breaker shares state across threads — is `record_failure` thread-safe? Prove it." -- "Faithfulness uses an LLM judge — how do you stop the judge's bias from silently inflating scores? What's your calibration story?" -- "`RunResult` is frozen but `spans` is a list — is it actually immutable? What stops a caller mutating the list in place?" -- "Cost is 'within 5% of invoice' — where does the other 5% go, and when would it be worse?" - -**Do NOT answer these.** Leave them for the owner. This is the explain-back test in written form. - -### The owner's loop per module -1. Read your DESIGN_LOG entry. -2. Answer the HARD_QUESTIONS in their own words. -3. Anything they can't answer → they open a discussion (with you or with me) until they can. -4. Only then does the module gate close. - ---- - - -## 12. Definition of Done - -### Per module -- [ ] Code implements the §6 contract for that module. -- [ ] OOP pillar(s) assigned to it in §3 are present and justified. -- [ ] methodoverload used at its designated site OR explicitly waived in DESIGN_LOG with reason. -- [ ] All mandatory internal test cases (§8) pass; coverage ≥80% for the module. -- [ ] `ruff check`, `ruff format --check`, `mypy` all clean. -- [ ] No `print()`; logging used with correct levels + trace correlation. -- [ ] DESIGN_LOG entry written (by you). -- [ ] HARD_QUESTIONS batch written (by you), answered (by owner). -- [ ] CI green on 3.10/3.11/3.12. - -### Whole project (v0.1.0) -- [ ] Modules 0–9 done; demo app (10) runs end-to-end and emits a real eval report. -- [ ] Demo app produces: traces in Jaeger, an accurate cost breakdown, an HTML eval report with RAG + agent metrics, and at least one deliberately-injected failure that reliability recovers from. -- [ ] `docs/architecture.md` has a diagram matching §2. -- [ ] `docs/getting_started.md` gets a new user to a first traced+evaluated run in <5 minutes. -- [ ] Benchmarks: naive baseline vs AgentArgus-instrumented (overhead quantified — be honest about the cost of instrumentation). -- [ ] Published to TestPyPI, smoke-tested in clean venv, then to real PyPI as v0.1.0. -- [ ] README badges (PyPI version, Python versions, CI, license, coverage). - ---- - -## Appendix A — Instruction to Claude Code on AI-assistance discipline - -The owner is deliberately using you (Claude Code) to build fast. That is fine and expected. To keep this a *portfolio-grade, defensible* project rather than an opaque generated blob, honour these: - -- **Narrate decisions, not just code.** When you make a non-obvious choice, say why in the DESIGN_LOG, not just in a code comment. -- **Prefer the owner's own primitives.** `methodoverload` is theirs — use it where it genuinely fits (§4) and say where it doesn't. -- **Flag, don't fabricate.** If you're unsure of a third-party API (OTel semantic-convention keys, LangGraph checkpointer API, a provider's usage-response shape), stop and say "verify this" rather than inventing a plausible-looking call. -- **Keep the public API small.** `from agentargus import Agent, EvalSuite, EvalRunner, ReliabilityPolicy` and the metrics should be most of it. Everything else is internal. -- **One behaviour, one home.** Any time you'd duplicate logic, extract it and note the extraction. - -## Appendix B — First actions for Claude Code (do these in order) - -1. Confirm/scaffold repo structure per §9. Set up `pyproject.toml`, `.ruff.toml`, CI. -2. Implement **Module 0** completely: `RunResult` + supporting dataclasses, `AgentArgusConfig`, `logging.py` (color + contextvars + JSON formatter). Full tests. Gate. -3. Write the Module 0 DESIGN_LOG entry and HARD_QUESTIONS batch. **Stop and hand back to the owner** before Module 1. -4. Proceed module by module through §5, gating at each step, never batching multiple modules without owner sign-off. \ No newline at end of file