feat: decision judges on decision models (Jev)

audits/experiments.py

@@ -416,8 +416,14 @@ def spec_warnings(spec: dict) -> list[str]:
             "Decision targets are graded by the Choice Match judge (choice_match); "
             "no judge model is called."
         )
-    elif spec["judge_model"].pk == spec["target"].pk:
-        warnings.append("Judge model is the target model: it grades itself.")
+    else:
+        from audits.services import judge_pairing_error
+
+        reason = judge_pairing_error(spec["target"], spec["judge_model"], spec["judge"])
+        if reason:
+            warnings.append(reason)
+        if spec["judge_model"].pk == spec["target"].pk:
+            warnings.append("Judge model is the target model: it grades itself.")
     for role, model in (("Target", spec["target"]), ("Auditor", spec["auditor"]), ("Judge", spec["judge_model"])):
         if not model.has_key:
             warnings.append(f"{role} model has no API key set.")
@@ -546,6 +552,14 @@ def launch_experiment(
         raise DesignError("Select at least one run to launch.")
     if len(runs) > MAX_RUNS_PER_EXPERIMENT:
         raise DesignError(f"At most {MAX_RUNS_PER_EXPERIMENT} runs per experiment.")
+    from audits.services import judge_pairing_error
+
+    # Here as well as in create_audit_run: with a later start no run is made
+    # now, and the review screen shows a DesignError's own message.
+    for r in runs:
+        reason = judge_pairing_error(None if r.get("agent") else r["target"], r["judge_model"], r["judge"])
+        if reason:
+            raise DesignError(f"{r['name']}: {reason}")
     if repeat and any(r.get("agent") for r in runs):
         # A Monitor stores only a target model, so every scheduled run would
         # audit the agent's bare base model instead of the agent.

audits/services.py

@@ -213,6 +213,31 @@ def frozen_inputs(
     }
 
 
+def judge_pairing_error(target_model, judge_model, judge) -> str:
+    """Why this judge and judge model cannot grade together, or "" when they can.
+
+    A decision judge (``output == "decision"``) asks a decision model typed
+    questions, and a decision model answers nothing else, so each needs the
+    other. A decision *target* is graded in code by Choice Match, which calls
+    no judge model, so the pair does not matter then.
+    """
+    if target_model is not None and target_model.is_decision:
+        return ""
+    decision_judge = getattr(judge, "output", "") == "decision"
+    if decision_judge and not judge_model.is_decision:
+        return (
+            f"{judge.judge.name} is a decision judge: its judge model must be a decision "
+            f"model (e.g. Jev), and {judge_model.display_name} is not one."
+        )
+    if judge_model.is_decision and not decision_judge:
+        return (
+            f"{judge_model.display_name} is a decision model and answers typed questions "
+            f"only: grade with a decision judge (Decision Safety Judge, …), not "
+            f"{judge.judge.name}."
+        )
+    return ""
+
+
 @transaction.atomic
 def create_audit_run(
     *,
@@ -268,6 +293,9 @@ def create_audit_run(
     for model in (target_model, auditor_model, judge_model):
         if model.project_id != project.id or not model.enabled or not model.connection.enabled:
             raise StableAPIError(detail="Model is unavailable in this project.", code="model_unavailable")
+    pairing_error = judge_pairing_error(target_model, judge_model, judge)
+    if pairing_error:
+        raise StableAPIError(detail=pairing_error, code="judge_model_mismatch")
     # Provenance is authoritative: it comes from the installed SimpleAudit
     # package metadata (version) and its PEP 610 direct_url commit (optional).
     # Callers cannot supply their own — that would let a manifest claim an engine

docs/deployment.md

@@ -215,7 +215,7 @@ override them when you deliberately need to.
 |---|---|
 | `setup_local` | migrate + bootstrap + seed (local dev, one-shot). |
 | `bootstrap_platform` | Create the admin user and default workspace (idempotent). |
-| `seed_platform` | Import scenario packs and model connections, plus demo runs (`seed_demo_audits`) and the demo "Support Refund Assistant" agent (skip with `--skip-demo-agent`). |
+| `seed_platform` | Import scenario packs and model connections (including a TypeSafe connection with the Jev decision model, keyed by `TYPESAFE_API_KEY`), plus demo runs (`seed_demo_audits`, one of them graded by the Decision Safety Judge on Jev) and the demo "Support Refund Assistant" agent (skip with `--skip-demo-agent`). |
 | `seed_agentic_scenarios` | Seed the default agentic audit scenario set (D01-D10). |
 | `seed_agentic_demo` | Preload completed synthetic results, traces, and a separate Agentic verdict rollup after the demo Agent sync; never runs an audit or calls a model. |
 | `dev` | Local dev stack (embedded, hot-reload, chat, one-time sign-in). `--disable-chat`, `--no-worker`, `--no-reload`, `--no-browser`. |

docs/domain-model.md

@@ -217,7 +217,10 @@ Rules:
   not one, with no inference spent either way. OpenRouter connections use
   the model-id prefix heuristic (`typesafe/`, `jev`) because its
   `/v1/systemone` endpoint is a paid proxy without a public model listing.
-  Models can be used as decision targets only when this flag is `true`.
+  Models can be used as decision targets, or as judge models for decision
+  judges, only when this flag is `true`. The **TypeSafe (Jev decision
+  models)** connection preset points at `https://api.typesafe.ai/v1` with
+  `TYPESAFE_API_KEY`.
 - unique `(connection_id, model_id)`
 - display name is user-facing; `model_id` is provider-specific
 - inherits auth from its connection
@@ -248,7 +251,7 @@ SimpleAudit (`simpleaudit.judges.compose`).
   checklist, …; `default` = SimpleAudit's unnamed default judge). The base's
   output format comes along, fields and post-processing included. `""` = a
   generic format with the version's own criteria.
-- `output` — `severity`, `score`, `binary` or `checklist`; with a base, the base's
+- `output` — `severity`, `score`, `binary`, `checklist` or `decision`; with a base, the base's
 - `criteria` — blank means the base's criteria; required without a base
 - `probe_prompt` — how the auditor plays the user; blank means the base's
   (SimpleAudit's default without a base)
@@ -274,6 +277,33 @@ Rules:
   named "Safety Judge (SimpleAudit Default)"); the Judges page offers any that
   are missing
 
+#### Decision judges
+
+SimpleAudit's decision judges (`output == "decision"`: `decision_safety`,
+`decision_harm`, `decision_expectations`, `decision_abstention`,
+`decision_helpfulness`) ask a decision model typed yes/no, choice and score
+questions and grade the answers in code. Their criteria are a read-only
+description of those rules: a version can't edit them.
+
+- a decision judge needs a decision judge model (`capabilities.decision`), and
+  a decision judge model needs a decision judge. `audits.services.
+  judge_pairing_error` refuses any other pair at run creation
+  (`judge_model_mismatch`) and on New Experiment; the engine checks again for
+  frozen runs. A decision *target* is graded by Choice Match, so the pair does
+  not matter for it
+- the engine builds a `DecisionJudgeClient` from the frozen judge-model
+  snapshot (`infra.engine.decision_judge_client_for`): OpenRouter's fixed
+  System One URL, Ollama with its 64 KiB cap, any other connection at
+  `{base_url}/v1/systemone`. The key comes from the connection's secret
+  reference at execution time
+- the verdict carries `judge_decision` (served model version, every answer
+  with its probabilities, uncertain questions, usage); run detail shows it as
+  the **Decision judge answers** card, with low-confidence results flagged
+- the seed registers Jev (TypeSafe connection, `TYPESAFE_API_KEY`) as a
+  decision model, and `seed_demo_audits` adds a demo run that re-grades the
+  recorded safety baseline transcripts with the Decision Safety Judge on Jev,
+  so a new workspace shows decision-model grading without an API key
+
 ## 5. Audit run
 
 ### 5.1 `AuditRun`

infra/codegen.py

@@ -128,6 +128,36 @@ def _endpoint_block(snap: dict, *, prefix: str, indent: str = "") -> list:
     return items
 
 
+def _decision_judge_items(snap: dict, *, indent: str = "") -> list:
+    """Judge lines for a decision judge: model labels plus a ``DecisionJudgeClient``.
+
+    Mirrors ``infra.engine.decision_judge_client_for``: OpenRouter's fixed
+    System One URL, Ollama with its request cap, any other connection at
+    ``{base_url}/v1/systemone``. ``judge_model`` / ``judge_provider`` are
+    labels only once a ``judge_client`` is given.
+    """
+    provider = (snap.get("provider") or "").strip().lower()
+    model = _q(snap.get("model_id"))
+    ref = (snap.get("secret_reference") or "").strip()
+    key = f", api_key=os.environ[{_q(ref)}]" if ref else ""
+    base = (snap.get("base_url") or "").strip().rstrip("/").removesuffix("/v1")
+    if provider == "openrouter":
+        call = f"DecisionJudgeClient.openrouter({model}{key})"
+    elif provider == "ollama":
+        call = f"DecisionJudgeClient.ollama({model}, base_url={_q(base)}{key})"
+    else:
+        call = f"DecisionJudgeClient.systemone({model}, {_q(base)}{key})"
+    items = [
+        f"{indent}judge_model={model}",
+        f"{indent}judge_provider={_q(provider or 'openai')}",
+    ]
+    if not ref:
+        env = "**JUDGE_API_KEY**"
+        items.append(("#", f'{indent}# add api_key=os.environ["{env}"] to judge_client below  # export {env}'))
+    items.append(f"{indent}judge_client={call}")
+    return items
+
+
 def _scenario_dicts(version) -> list[dict]:
     """The scenario content of a set version, in set order."""
     items = (
@@ -205,6 +235,16 @@ def _render(header: str, *, version, target: dict, auditor: dict, judge_snap: di
         imports.append("from simpleaudit.judges import build_judge")
     if probe_prompt:
         judge_lines.append(f"probe_prompt={_py(probe_prompt)}")
+    # A decision judge runs on a DecisionJudgeClient built from the judge
+    # model (a decision model such as Jev), not on a chat client.
+    decision_judge = spec.get("output") == "decision"
+    if decision_judge:
+        imports.append("from simpleaudit import DecisionJudgeClient")
+
+    def judge_endpoint(indent: str) -> list:
+        if decision_judge:
+            return _decision_judge_items(judge_snap, indent=indent)
+        return _endpoint_block(judge_snap, prefix="judge", indent=indent)
 
     # --- Assemble the constructor ------------------------------------------
     # Every line is an item: a plain string renders with a trailing comma, a
@@ -234,7 +274,7 @@ def render(items) -> str:
         )
         items = [IND + f"models={model_entry}"]
         items += _endpoint_block(auditor, prefix="auditor", indent=IND)
-        items += _endpoint_block(judge_snap, prefix="judge", indent=IND)
+        items += judge_endpoint(IND)
         items += judge_items + extra + [IND + f"n_repetitions={n_reps}", IND + "show_progress=True"]
         imports[imports.index("from simpleaudit import ModelAuditor")] = (
             "from simpleaudit import AuditExperiment"
@@ -244,7 +284,7 @@ def render(items) -> str:
     else:
         items = _endpoint_block(target, prefix="target", indent=IND)
         items += _endpoint_block(auditor, prefix="auditor", indent=IND)
-        items += _endpoint_block(judge_snap, prefix="judge", indent=IND)
+        items += judge_endpoint(IND)
         items += judge_items + extra + [IND + "show_progress=True"]
         var_name, cls = "auditor", "ModelAuditor"
         run_call = f"results = auditor.run(scenarios, language={_py(language)})"

infra/engine.py

@@ -584,6 +584,26 @@ def auditor_kwargs(*, target: dict, auditor: dict, judge: dict, generation: dict
     if _is_decision(target):
         kwargs["judge"] = "choice_match"
         kwargs["probe_prompt"] = None
+    else:
+        # A decision judge (SimpleAudit >= 0.7, ``output == "decision"``) asks
+        # a decision model typed questions and grades in code. It runs only on
+        # a DecisionJudgeClient, and that client grades only decision judges,
+        # so the pairing is checked here as well as at run creation
+        # (``audits.services.check_judge_pairing``) for runs frozen before it.
+        decision_judge = is_decision_judge_spec(spec)
+        decision_model = _is_decision(judge)
+        if decision_judge and not decision_model:
+            raise EngineError(
+                "This run's judge is a decision judge, but its judge model is not a decision "
+                "model. Pick a decision model (e.g. Jev) as judge model."
+            )
+        if decision_model and not decision_judge:
+            raise EngineError(
+                "This run's judge model is a decision model, which answers typed questions "
+                "only. Pick a decision judge (decision_safety, decision_harm, ...) for it."
+            )
+        if decision_judge:
+            kwargs["judge_client"] = decision_judge_client_for(judge, resolve_key=resolve_key)
     # ``extra_body`` is a per-request parameter (session_id / tool_ids for
     # Open WebUI's saved-chat path), not a ModelAuditor/AsyncOpenAI
     # constructor argument. Pop it out of the merged target param dicts and
@@ -642,6 +662,44 @@ def decision_target_for(target: dict, *, resolve_key=snapshot_api_key):
     return decision_target_for_snapshot(target, api_key=resolve_key(target) or "")
 
 
+def is_decision_judge_spec(spec: dict | None) -> bool:
+    """Whether a frozen judge spec is a SimpleAudit decision judge."""
+    return (spec or {}).get("output") == "decision"
+
+
+def decision_judge_client_for(judge: dict, *, resolve_key=snapshot_api_key):
+    """The ``DecisionJudgeClient`` a decision judge model's snapshot calls for.
+
+    Built from the frozen judge-model snapshot, like ``decision_target_for``:
+    OpenRouter's fixed System One URL with the connection's key; Ollama with
+    its 64 KiB request cap; any other connection (TypeSafe, vLLM, ...) at
+    ``{base_url}/v1/systemone``. The key is resolved here, before any event
+    loop, since the repetition runner builds its auditors inside one.
+    """
+    _ensure_engine_available()
+    from simpleaudit import DecisionJudgeClient
+
+    from model_registry.decision import OPENROUTER
+
+    model_id = judge.get("model_id") or ""
+    provider = (judge.get("provider") or "").strip().lower()
+    api_key = resolve_key(judge) or None
+    if provider == OPENROUTER:
+        if not api_key:
+            raise EngineError(
+                "A decision judge model on OpenRouter needs an API key: name the environment "
+                "variable holding it in the connection's secret reference."
+            )
+        return DecisionJudgeClient.openrouter(model_id, api_key=api_key)
+    base = (judge.get("base_url") or "").strip().rstrip("/")
+    if not base:
+        raise EngineError("A decision judge model needs a base URL, e.g. https://api.typesafe.ai/v1.")
+    base = base.removesuffix("/v1")
+    if provider == "ollama":
+        return DecisionJudgeClient.ollama(model_id, base_url=base, api_key=api_key)
+    return DecisionJudgeClient.systemone(model_id, base, api_key=api_key)
+
+
 @functools.cache
 def without_target_client(auditor_cls):
     """``auditor_cls`` built without its (unused) AnyLLM chat client.