fix(cli): run trace-only evaluations through the judge pipeline

CHANGELOG.md

@@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- `judge-traces` now runs the configured judge against imported OTLP conversations and
+  writes standard run/viewer artifacts without invoking the target. It requires an
+  existing taxonomy and judge credentials; use `--parse-only` for the previous
+  conversion-only behavior without model calls. Incomplete evaluations exit nonzero.
+
 ### Fixed
 
 ## [0.3.0] - 2026-09-04

assert_ai/cli.py

@@ -1909,8 +1909,23 @@ def analysis_test_set_metrics(
 )
 @click.option("--group-by", default="session.id", show_default=True, help="OTel attribute to group spans by")
 @click.option("--output", default=None, type=click.Path(path_type=Path), help="Output directory for scores")
-def judge_traces(traces: Path, config_path: Path, group_by: str, output: Path | None):
+@click.option("--parse-only", is_flag=True, help="Convert traces without calling a judge (legacy behavior).")
+def judge_traces(traces: Path, config_path: Path, group_by: str, output: Path | None, parse_only: bool):
     """Judge pre-collected OTel traces without running inference."""
+    if not parse_only:
+        from assert_ai.config import ConfigError
+        from assert_ai.trace_judging import judge_trace_file
+
+        try:
+            code, run_root, counts = judge_trace_file(
+                traces=traces, config=config_path, group_by=group_by, output=output,
+            )
+        except (ConfigError, OSError, ValueError) as exc:
+            raise click.ClickException(str(exc)) from exc
+        click.echo(f"Trace evaluation: {json.dumps(counts, sort_keys=True)}")
+        click.echo(f"Run dir: {run_root}")
+        raise SystemExit(code)
+
     from assert_ai.core.otel import parse_otel_traces
 
     click.echo(f"Parsing OTel traces from {traces}...")
@@ -1935,11 +1950,9 @@ def judge_traces(traces: Path, config_path: Path, group_by: str, output: Path |
             f.write(json.dumps(row) + "\n")
     click.echo(f"Wrote {len(inference_rows)} inference rows to {inference_set_path}")
 
-    click.echo(f"Judging {len(inference_rows)} conversations...")
-    # Full judge execution requires LLM access; the inference rows are ready
-    # for the judge stage to consume.
+    click.echo("Parse only: no judge or target was called.")
     click.echo(f"Inference set written to {inference_set_path}")
-    click.echo("Run the full pipeline with --force-stage judge to score these inference rows.")
+    click.echo("Use judge-traces without --parse-only to create a scored run.")
 
 
 @cli.group(

assert_ai/core/judge.py

@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 import asyncio
+import hashlib
 import json
 import logging
 import re
@@ -43,6 +44,7 @@
     "get_verdict_dimension",
     "has_successful_judge_verdict",
     "infer_judge_status",
+    "inference_row_sha256",
     "is_not_applicable_dimension",
     "is_valid_confidence_label",
     "is_valid_event_flag",
@@ -214,6 +216,17 @@ def infer_judge_status(record: Dict[str, Any]) -> str:
     return "ok" if success else "judge_failed"
 
 
+def inference_row_sha256(row: Dict[str, Any]) -> str:
+    """Fingerprint the exact inference content a score row judges."""
+    payload = json.dumps(
+        row,
+        ensure_ascii=False,
+        separators=(",", ":"),
+        sort_keys=True,
+    )
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
 def has_successful_judge_verdict(
     verdict: Optional[Dict[str, Any]],
     required_dimension_names: list[str] | None = None,