fix: reject non-finite accuracy scores in the single-turn gate

src/submission_checker/data/messages.yaml

@@ -96,6 +96,9 @@ accuracy-gate:
     incomplete-datasets:
       text: 'Cannot compute aggregate accuracy: incomplete score (complete: false) for dataset(s): {datasets}'
       fix: 'Rescore each listed dataset until every sample is graded and the client reports complete: true.'
+    non-finite-scores:
+      text: 'Cannot compute aggregate accuracy: non-finite score for {scores}'
+      fix: Rescore each listed dataset; a NaN or infinite score means scoring failed.
     warn-2: results.json exposes only an unnamed scalar accuracy score; gating it as '{only_metric}'.
       Secondary metrics (if any) are not present in results.json and are not checked.
     fail: '{matched_key} = {score:.4f} < min {lower:.4f}'

src/submission_checker/models/aggregate/context.py

@@ -5,6 +5,7 @@
 
 from __future__ import annotations
 
+import math
 from pathlib import Path
 
 __all__ = ["ModelContext", "accuracy_bands", "match_accuracy_bands"]
@@ -1101,6 +1102,20 @@ def _gate_accuracy(self, accuracy_result: AccuracyResult) -> None:
         if unscored or incomplete:
             return  # A mean over only the scored subsets is not the aggregate accuracy.
         per_ds = accuracy_result.metric_scores()  # {ds: {metric: float}}
+        # A JSON NaN (or ROUGE's "nan" string) compares false with every bound, so it
+        # would pass; +inf would pass a lower bound.
+        non_finite = [
+            f"{ds_name} ({metric}={value})"
+            for ds_name, scores in per_ds.items()
+            if ds_name in datasets
+            for metric, value in scores.items()
+            if not math.isfinite(value)
+        ]
+        if non_finite:
+            self._check_results.append(
+                err("accuracy-gate", "non-finite-scores", json_path, scores=", ".join(non_finite))
+            )
+            return
         weighted: dict[str, list[tuple[float, float | None]]] = {}
         for ds_name, scores in per_ds.items():
             if ds_name not in datasets:

tests/submission_checker/test_checks_agentic_accuracy.py

@@ -194,6 +194,14 @@ def test_value_outside_unit_interval_errors(self, tmp_path: Path) -> None:
         errors = _errors(ctx, "agentic-accuracy-inline")
         assert errors and "not a fraction" in errors[0].message
 
+    @pytest.mark.parametrize("value", [float("nan"), float("inf")])
+    def test_non_finite_value_errors(self, tmp_path: Path, value: float) -> None:
+        """NaN fails the [0, 1] range check, so it is reported, never passed."""
+        ctx = _ctx(tmp_path, accuracy_by_point={16: _accuracy(inline=value, raw=True)})
+        errors = _errors(ctx, "agentic-accuracy-inline")
+        assert errors and "not a fraction" in errors[0].message
+        assert not [h for h in _hits(ctx, "agentic-accuracy-inline") if h.key == "pass"]
+
     def test_qwen_uses_its_own_threshold(self, tmp_path: Path) -> None:
         """56.0 clears Qwen's 55.86 and fails Kimi's 58.32."""
         for model, fails in (("qwen3_6-35b-a3b", False), ("kimi-k3", True)):

tests/submission_checker/test_checks_aggregate.py

@@ -4,6 +4,8 @@
 
 from __future__ import annotations
 
+import json
+
 import pytest
 
 from submission_checker.models import (
@@ -1053,6 +1055,60 @@ def test_inline_only_point_is_not_gated_as_accuracy(self, tmp_path):
             r.rule in ("accuracy-gate", "accuracy-sample-count") for r in ctx._check_results
         )
 
+    @pytest.mark.parametrize("score", [float("nan"), "nan", float("inf"), float("-inf")])
+    def test_non_finite_score_rejects_aggregate(self, tmp_path, score):
+        entries = {"deepseek_r1_accuracy": {"num_samples": 4388, "score": score}}
+        ctx = _model_ctx(
+            tmp_path,
+            accuracy_result=self._report(entries, True),
+            model_name="deepseek_r1-torch-fp4",
+        )
+        gates = [r for r in ctx._check_results if r.rule == "accuracy-gate"]
+        assert [r.severity for r in gates] == [Severity.ERROR]
+        assert "non-finite" in gates[0].message and "deepseek_r1_accuracy" in gates[0].message
+
+    def test_nan_as_the_client_writes_it_rejects_aggregate(self, tmp_path):
+        # The client writes accuracy_results.json with json.dumps, which emits a float
+        # NaN as the bare literal `NaN`; json.loads reads it back as float("nan").
+        text = (
+            '{"accuracy_scores": [{"dataset_name": "deepseek_r1_accuracy", "score": NaN,'
+            ' "unit_samples": 4388, "num_repeats": 1, "total_samples": 4388,'
+            ' "complete": true, "dataset_type": "accuracy"}]}'
+        )
+        ctx = _model_ctx(
+            tmp_path,
+            accuracy_result=AccuracyResult.model_validate(json.loads(text)),
+            model_name="deepseek_r1-torch-fp4",
+        )
+        gates = [r for r in ctx._check_results if r.rule == "accuracy-gate"]
+        assert any(r.severity == Severity.ERROR and "non-finite" in r.message for r in gates)
+        assert not any(r.severity == Severity.INFO for r in gates)
+
+    def test_rouge_nan_string_rejects_aggregate(self, tmp_path):
+        # RougeScorer formats each metric as a string; an empty set gives "nan".
+        ar = AccuracyResult(
+            {
+                "cnn_dailymail::llama3_8b": {
+                    "num_samples": 13368,
+                    "score": {"rouge1": "nan", "rouge2": "15.9", "rougeL": "nan"},
+                }
+            }
+        )
+        ctx = _model_ctx(tmp_path, accuracy_result=ar, model_name="Llama-3_1-8B-Instruct")
+        gates = [r for r in ctx._check_results if r.rule == "accuracy-gate"]
+        assert [r.severity for r in gates] == [Severity.ERROR]
+        assert "rouge1=nan" in gates[0].message and "rougeL=nan" in gates[0].message
+        assert "rouge2" not in gates[0].message
+
+    def test_non_finite_inline_score_does_not_block_accuracy_gate(self, tmp_path):
+        entries = self._gptoss_with_inline(accuracy_score=0.85, inline_score=float("nan"))
+        ctx = _model_ctx(
+            tmp_path, accuracy_result=self._report(entries, True), model_name="gpt-oss-120b"
+        )
+        gates = [r for r in ctx._check_results if r.rule == "accuracy-gate"]
+        assert not any(r.severity == Severity.ERROR for r in gates)
+        assert any(r.severity == Severity.INFO and "= 85.0000" in r.message for r in gates)
+
     def test_multidataset_sample_count_is_summed(self, tmp_path):
         ctx = _model_ctx(
             tmp_path, accuracy_result=AccuracyResult(self._GPTOSS), model_name="gpt-oss-120b"