From ddc1f3d96b4fd55953c01eb4c93ab659745fcca6 Mon Sep 17 00:00:00 2001 From: Eirik Botten Nicolaysen Date: Tue, 3 Nov 2026 17:50:23 +0100 Subject: [PATCH 1/3] Keep each run's meta when loading repeated experiment results RepeatedExperimentResults.save() writes every run's meta, but load() rebuilt each run as AuditResults(results), so the judge, target and auditor settings were gone after a round trip. AuditResults.load() already passes meta=data.get("meta"); the repeated loader now does the same. --- simpleaudit/repeated_results.py | 2 +- tests/test_repeated_experiments.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/simpleaudit/repeated_results.py b/simpleaudit/repeated_results.py index de0e948..3d2cb09 100644 --- a/simpleaudit/repeated_results.py +++ b/simpleaudit/repeated_results.py @@ -675,7 +675,7 @@ def load(cls, filepath: str) -> "RepeatedExperimentResults": reconstructed = [] for run_data in run_list: results = [AuditResult(**r) for r in run_data["results"]] - instance = AuditResults(results) + instance = AuditResults(results, meta=run_data.get("meta")) instance.timestamp = run_data.get("timestamp", instance.timestamp) reconstructed.append(instance) runs_by_model[label] = reconstructed diff --git a/tests/test_repeated_experiments.py b/tests/test_repeated_experiments.py index 3ef9d0f..3263bfa 100644 --- a/tests/test_repeated_experiments.py +++ b/tests/test_repeated_experiments.py @@ -569,6 +569,20 @@ def test_save_load_preserves_judge_metadata(self, tmp_path): assert loaded._judge == judge assert loaded.to_dict()["judge"]["judge_model"] == "judge-x" + def test_save_load_preserves_each_runs_meta(self, tmp_path): + """Each run's meta survives save()/load(), as AuditResults.load keeps it.""" + meta = {"judge_model": "judge-x", "target_model": "m", "scenario_pack": "safety"} + run = AuditResults(_make_results(["pass"]).results, meta=meta) + results = RepeatedExperimentResults({"m": [run, _make_results(["low"])]}) + + path = str(tmp_path / "exp.json") + results.save(path) + loaded = RepeatedExperimentResults.load(path) + + assert loaded._runs["m"][0].meta == meta + assert loaded._runs["m"][1].meta == {} + assert loaded.to_dict()["runs"]["m"][0]["meta"] == meta + # --------------------------------------------------------------------------- # AuditExperiment — on_model_done callback From c15a07882d8c5a1ece20bd4d8976475e3f1de01a Mon Sep 17 00:00:00 2001 From: Eirik Botten Nicolaysen Date: Tue, 3 Nov 2026 17:51:28 +0100 Subject: [PATCH 2/3] Record the scenario pack in AuditExperiment runs' meta AuditExperiment expands a pack name to its scenario list before calling ModelAuditor.run_async, and _run_meta only records scenario_pack when it gets a string. Runs started with AuditExperiment.run("safety") therefore had no scenario_pack, while ModelAuditor.run("safety") had one. The experiment now records the pack name itself, with the same isinstance check; a list of scenarios still records none. --- simpleaudit/experiment.py | 4 ++++ tests/test_repeated_experiments.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/simpleaudit/experiment.py b/simpleaudit/experiment.py index 6bfb70d..40d4b7f 100644 --- a/simpleaudit/experiment.py +++ b/simpleaudit/experiment.py @@ -365,6 +365,10 @@ async def _run_single_rep( trace_correlation=corr, evidence_resolver=evidence_resolver, ) + # The auditor only sees the expanded list, so it cannot record the + # pack name itself; record it here as ModelAuditor._run_meta does. + if isinstance(scenarios, str): + partial.meta["scenario_pack"] = scenarios if result is None or not by_name: result = partial else: diff --git a/tests/test_repeated_experiments.py b/tests/test_repeated_experiments.py index 3263bfa..9c12348 100644 --- a/tests/test_repeated_experiments.py +++ b/tests/test_repeated_experiments.py @@ -86,6 +86,24 @@ def test_n_repetitions_3_produces_3_runs(self): assert isinstance(results, RepeatedExperimentResults) assert len(results._runs["test-model"]) == 3 + def test_pack_name_is_recorded_in_each_runs_meta(self): + """A run started with a pack name records it, as ModelAuditor.run does.""" + + async def fake_run_async(self_a, scenarios, max_turns=None, language="English", **kwargs): + return AuditResults( + _make_results(["pass"]).results, + meta=self_a._run_meta(scenarios, max_turns, language), + ) + + exp = _make_experiment(n_repetitions=2) + with patch.object(ModelAuditor, "_create_anyllm_client", return_value=MagicMock()), \ + patch.object(ModelAuditor, "run_async", new=fake_run_async): + from_pack = asyncio.run(exp.run_async(scenarios="safety")) + from_list = asyncio.run(exp.run_async(scenarios=SCENARIOS)) + + assert [run.meta.get("scenario_pack") for run in from_pack._runs["test-model"]] == ["safety", "safety"] + assert all("scenario_pack" not in run.meta for run in from_list._runs["test-model"]) + def test_n_repetitions_1_is_default_compatible(self): exp = _make_experiment(n_repetitions=1) r = _make_results(["low"]) From 0972e7062372f7dbb31ec8a26ba39602b578cfaf Mon Sep 17 00:00:00 2001 From: Eirik Botten Nicolaysen Date: Tue, 3 Nov 2026 17:51:50 +0100 Subject: [PATCH 3/3] Show a missing score in the README results example AuditResults.score is None when nothing was graded, so the example printed "Score: None/100". It now prints the same "n/a (nothing graded)" that summary() prints in that case. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0fbe3c5..135d004 100644 --- a/README.md +++ b/README.md @@ -1277,7 +1277,8 @@ Two things to know: results = auditor.run("safety") # Summary statistics -print(f"Score: {results.score}/100") +score = "n/a (nothing graded)" if results.score is None else f"{results.score}/100" +print(f"Score: {score}") print(f"Passed: {results.passed}/{len(results)}") print(f"Failed: {results.failed}") # graded anything other than pass print(f"Errors: {results.errors}") # never graded: API failure or unreadable judge reply