test_golden_qa.py

"""Parametrised evaluation tests from the golden QA dataset.

Marked with ``@pytest.mark.eval`` so the default ``just check`` /
``pytest tests/`` invocations skip them. Run explicitly with::

    uv run pytest eval/

Wire a real LLM-backed ``answer_fn`` by editing ``_answer_fn`` below or
constructing ``EvalRunner`` with your own callable. The default echoes the
question verbatim, which makes the trivial example case (``echo hello``
expects ``hello``) pass without any LLM credentials.
"""

from __future__ import annotations

import pytest

from src.eval.models import EvalCase
from src.eval.runner import EvalRunner, load_golden_dataset

# Load cases without initialising a runner (no side effects at import time)
golden = load_golden_dataset()


def _answer_fn(question: str) -> str:
    """Placeholder agent. Strips a leading ``echo `` so the example case
    resolves to the expected answer; in real use this calls the agent
    loop / LLM client wired by the project."""
    return question.removeprefix("echo ").strip()


@pytest.fixture(scope="module")
def runner() -> EvalRunner:
    """Single runner shared across cases in this module."""
    return EvalRunner(answer_fn=_answer_fn)


@pytest.mark.eval
@pytest.mark.parametrize("case", golden, ids=lambda c: c.id)
def test_golden_qa(case: EvalCase, runner: EvalRunner) -> None:
    """Evaluate a golden QA test case against the configured agent."""
    result = runner.evaluate(case)
    assert result.pass_result, (
        f"[{case.id}] {case.category}/{case.difficulty}\n"
        f"Q: {case.question}\n"
        f"Expected: {case.expected_answer}\n"
        f"Got: {result.actual_answer}\n"
        f"Reason: {result.failure_reason}"
    )