diff --git a/src/submission_checker/data/messages.yaml b/src/submission_checker/data/messages.yaml index 3dbcc17..82cb40c 100644 --- a/src/submission_checker/data/messages.yaml +++ b/src/submission_checker/data/messages.yaml @@ -96,6 +96,9 @@ accuracy-gate: incomplete-datasets: text: 'Cannot compute aggregate accuracy: incomplete score (complete: false) for dataset(s): {datasets}' fix: 'Rescore each listed dataset until every sample is graded and the client reports complete: true.' + non-finite-scores: + text: 'Cannot compute aggregate accuracy: non-finite score for {scores}' + fix: Rescore each listed dataset; a NaN or infinite score means scoring failed. warn-2: results.json exposes only an unnamed scalar accuracy score; gating it as '{only_metric}'. Secondary metrics (if any) are not present in results.json and are not checked. fail: '{matched_key} = {score:.4f} < min {lower:.4f}' diff --git a/src/submission_checker/models/aggregate/context.py b/src/submission_checker/models/aggregate/context.py index d8f0c6d..a0b6cc0 100644 --- a/src/submission_checker/models/aggregate/context.py +++ b/src/submission_checker/models/aggregate/context.py @@ -5,6 +5,7 @@ from __future__ import annotations +import math from pathlib import Path __all__ = ["ModelContext", "accuracy_bands", "match_accuracy_bands"] @@ -1101,6 +1102,20 @@ def _gate_accuracy(self, accuracy_result: AccuracyResult) -> None: if unscored or incomplete: return # A mean over only the scored subsets is not the aggregate accuracy. per_ds = accuracy_result.metric_scores() # {ds: {metric: float}} + # A JSON NaN (or ROUGE's "nan" string) compares false with every bound, so it + # would pass; +inf would pass a lower bound. + non_finite = [ + f"{ds_name} ({metric}={value})" + for ds_name, scores in per_ds.items() + if ds_name in datasets + for metric, value in scores.items() + if not math.isfinite(value) + ] + if non_finite: + self._check_results.append( + err("accuracy-gate", "non-finite-scores", json_path, scores=", ".join(non_finite)) + ) + return weighted: dict[str, list[tuple[float, float | None]]] = {} for ds_name, scores in per_ds.items(): if ds_name not in datasets: diff --git a/tests/submission_checker/test_checks_agentic_accuracy.py b/tests/submission_checker/test_checks_agentic_accuracy.py index 5ba6160..0103362 100644 --- a/tests/submission_checker/test_checks_agentic_accuracy.py +++ b/tests/submission_checker/test_checks_agentic_accuracy.py @@ -194,6 +194,14 @@ def test_value_outside_unit_interval_errors(self, tmp_path: Path) -> None: errors = _errors(ctx, "agentic-accuracy-inline") assert errors and "not a fraction" in errors[0].message + @pytest.mark.parametrize("value", [float("nan"), float("inf")]) + def test_non_finite_value_errors(self, tmp_path: Path, value: float) -> None: + """NaN fails the [0, 1] range check, so it is reported, never passed.""" + ctx = _ctx(tmp_path, accuracy_by_point={16: _accuracy(inline=value, raw=True)}) + errors = _errors(ctx, "agentic-accuracy-inline") + assert errors and "not a fraction" in errors[0].message + assert not [h for h in _hits(ctx, "agentic-accuracy-inline") if h.key == "pass"] + def test_qwen_uses_its_own_threshold(self, tmp_path: Path) -> None: """56.0 clears Qwen's 55.86 and fails Kimi's 58.32.""" for model, fails in (("qwen3_6-35b-a3b", False), ("kimi-k3", True)): diff --git a/tests/submission_checker/test_checks_aggregate.py b/tests/submission_checker/test_checks_aggregate.py index 23b6e9c..f4b1912 100644 --- a/tests/submission_checker/test_checks_aggregate.py +++ b/tests/submission_checker/test_checks_aggregate.py @@ -4,6 +4,8 @@ from __future__ import annotations +import json + import pytest from submission_checker.models import ( @@ -1053,6 +1055,60 @@ def test_inline_only_point_is_not_gated_as_accuracy(self, tmp_path): r.rule in ("accuracy-gate", "accuracy-sample-count") for r in ctx._check_results ) + @pytest.mark.parametrize("score", [float("nan"), "nan", float("inf"), float("-inf")]) + def test_non_finite_score_rejects_aggregate(self, tmp_path, score): + entries = {"deepseek_r1_accuracy": {"num_samples": 4388, "score": score}} + ctx = _model_ctx( + tmp_path, + accuracy_result=self._report(entries, True), + model_name="deepseek_r1-torch-fp4", + ) + gates = [r for r in ctx._check_results if r.rule == "accuracy-gate"] + assert [r.severity for r in gates] == [Severity.ERROR] + assert "non-finite" in gates[0].message and "deepseek_r1_accuracy" in gates[0].message + + def test_nan_as_the_client_writes_it_rejects_aggregate(self, tmp_path): + # The client writes accuracy_results.json with json.dumps, which emits a float + # NaN as the bare literal `NaN`; json.loads reads it back as float("nan"). + text = ( + '{"accuracy_scores": [{"dataset_name": "deepseek_r1_accuracy", "score": NaN,' + ' "unit_samples": 4388, "num_repeats": 1, "total_samples": 4388,' + ' "complete": true, "dataset_type": "accuracy"}]}' + ) + ctx = _model_ctx( + tmp_path, + accuracy_result=AccuracyResult.model_validate(json.loads(text)), + model_name="deepseek_r1-torch-fp4", + ) + gates = [r for r in ctx._check_results if r.rule == "accuracy-gate"] + assert any(r.severity == Severity.ERROR and "non-finite" in r.message for r in gates) + assert not any(r.severity == Severity.INFO for r in gates) + + def test_rouge_nan_string_rejects_aggregate(self, tmp_path): + # RougeScorer formats each metric as a string; an empty set gives "nan". + ar = AccuracyResult( + { + "cnn_dailymail::llama3_8b": { + "num_samples": 13368, + "score": {"rouge1": "nan", "rouge2": "15.9", "rougeL": "nan"}, + } + } + ) + ctx = _model_ctx(tmp_path, accuracy_result=ar, model_name="Llama-3_1-8B-Instruct") + gates = [r for r in ctx._check_results if r.rule == "accuracy-gate"] + assert [r.severity for r in gates] == [Severity.ERROR] + assert "rouge1=nan" in gates[0].message and "rougeL=nan" in gates[0].message + assert "rouge2" not in gates[0].message + + def test_non_finite_inline_score_does_not_block_accuracy_gate(self, tmp_path): + entries = self._gptoss_with_inline(accuracy_score=0.85, inline_score=float("nan")) + ctx = _model_ctx( + tmp_path, accuracy_result=self._report(entries, True), model_name="gpt-oss-120b" + ) + gates = [r for r in ctx._check_results if r.rule == "accuracy-gate"] + assert not any(r.severity == Severity.ERROR for r in gates) + assert any(r.severity == Severity.INFO and "= 85.0000" in r.message for r in gates) + def test_multidataset_sample_count_is_summed(self, tmp_path): ctx = _model_ctx( tmp_path, accuracy_result=AccuracyResult(self._GPTOSS), model_name="gpt-oss-120b"