diff --git a/README.md b/README.md index 2312df9..28770a8 100644 --- a/README.md +++ b/README.md @@ -1331,7 +1331,8 @@ Two things to know: results = auditor.run("safety") # Summary statistics -print(f"Score: {results.score}/100") +score = "n/a (nothing graded)" if results.score is None else f"{results.score}/100" +print(f"Score: {score}") print(f"Passed: {results.passed}/{len(results)}") print(f"Failed: {results.failed}") # graded anything other than pass print(f"Errors: {results.errors}") # never graded: API failure or unreadable judge reply diff --git a/tests/test_repeated_experiments.py b/tests/test_repeated_experiments.py index a2f3d12..f58f18a 100644 --- a/tests/test_repeated_experiments.py +++ b/tests/test_repeated_experiments.py @@ -86,6 +86,24 @@ def test_n_repetitions_3_produces_3_runs(self): assert isinstance(results, RepeatedExperimentResults) assert len(results._runs["test-model"]) == 3 + def test_pack_name_is_recorded_in_each_runs_meta(self): + """A run started with a pack name records it, as ModelAuditor.run does.""" + + async def fake_run_async(self_a, scenarios, max_turns=None, language="English", **kwargs): + return AuditResults( + _make_results(["pass"]).results, + meta=self_a._run_meta(scenarios, max_turns, language), + ) + + exp = _make_experiment(n_repetitions=2) + with patch.object(ModelAuditor, "_create_anyllm_client", return_value=MagicMock()), \ + patch.object(ModelAuditor, "run_async", new=fake_run_async): + from_pack = asyncio.run(exp.run_async(scenarios="safety")) + from_list = asyncio.run(exp.run_async(scenarios=SCENARIOS)) + + assert [run.meta.get("scenario_pack") for run in from_pack._runs["test-model"]] == ["safety", "safety"] + assert all("scenario_pack" not in run.meta for run in from_list._runs["test-model"]) + def test_n_repetitions_1_is_default_compatible(self): exp = _make_experiment(n_repetitions=1) r = _make_results(["low"]) @@ -569,6 +587,20 @@ def test_save_load_preserves_judge_metadata(self, tmp_path): assert loaded._judge == judge assert loaded.to_dict()["judge"]["judge_model"] == "judge-x" + def test_save_load_preserves_each_runs_meta(self, tmp_path): + """Each run's meta survives save()/load(), as AuditResults.load keeps it.""" + meta = {"judge_model": "judge-x", "target_model": "m", "scenario_pack": "safety"} + run = AuditResults(_make_results(["pass"]).results, meta=meta) + results = RepeatedExperimentResults({"m": [run, _make_results(["low"])]}) + + path = str(tmp_path / "exp.json") + results.save(path) + loaded = RepeatedExperimentResults.load(path) + + assert loaded._runs["m"][0].meta == meta + assert loaded._runs["m"][1].meta == {} + assert loaded.to_dict()["runs"]["m"][0]["meta"] == meta + # --------------------------------------------------------------------------- # AuditExperiment — on_model_done callback