Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1277,7 +1277,8 @@ Two things to know:
results = auditor.run("safety")

# Summary statistics
print(f"Score: {results.score}/100")
score = "n/a (nothing graded)" if results.score is None else f"{results.score}/100"
print(f"Score: {score}")
print(f"Passed: {results.passed}/{len(results)}")
print(f"Failed: {results.failed}") # graded anything other than pass
print(f"Errors: {results.errors}") # never graded: API failure or unreadable judge reply
Expand Down
4 changes: 4 additions & 0 deletions simpleaudit/experiment.py
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,10 @@ async def _run_single_rep(
trace_correlation=corr,
evidence_resolver=evidence_resolver,
)
# The auditor only sees the expanded list, so it cannot record the
# pack name itself; record it here as ModelAuditor._run_meta does.
if isinstance(scenarios, str):
partial.meta["scenario_pack"] = scenarios
if result is None or not by_name:
result = partial
else:
Expand Down
2 changes: 1 addition & 1 deletion simpleaudit/repeated_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -675,7 +675,7 @@ def load(cls, filepath: str) -> "RepeatedExperimentResults":
reconstructed = []
for run_data in run_list:
results = [AuditResult(**r) for r in run_data["results"]]
instance = AuditResults(results)
instance = AuditResults(results, meta=run_data.get("meta"))
instance.timestamp = run_data.get("timestamp", instance.timestamp)
reconstructed.append(instance)
runs_by_model[label] = reconstructed
Expand Down
32 changes: 32 additions & 0 deletions tests/test_repeated_experiments.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,24 @@ def test_n_repetitions_3_produces_3_runs(self):
assert isinstance(results, RepeatedExperimentResults)
assert len(results._runs["test-model"]) == 3

def test_pack_name_is_recorded_in_each_runs_meta(self):
"""A run started with a pack name records it, as ModelAuditor.run does."""

async def fake_run_async(self_a, scenarios, max_turns=None, language="English", **kwargs):
return AuditResults(
_make_results(["pass"]).results,
meta=self_a._run_meta(scenarios, max_turns, language),
)

exp = _make_experiment(n_repetitions=2)
with patch.object(ModelAuditor, "_create_anyllm_client", return_value=MagicMock()), \
patch.object(ModelAuditor, "run_async", new=fake_run_async):
from_pack = asyncio.run(exp.run_async(scenarios="safety"))
from_list = asyncio.run(exp.run_async(scenarios=SCENARIOS))

assert [run.meta.get("scenario_pack") for run in from_pack._runs["test-model"]] == ["safety", "safety"]
assert all("scenario_pack" not in run.meta for run in from_list._runs["test-model"])

def test_n_repetitions_1_is_default_compatible(self):
exp = _make_experiment(n_repetitions=1)
r = _make_results(["low"])
Expand Down Expand Up @@ -569,6 +587,20 @@ def test_save_load_preserves_judge_metadata(self, tmp_path):
assert loaded._judge == judge
assert loaded.to_dict()["judge"]["judge_model"] == "judge-x"

def test_save_load_preserves_each_runs_meta(self, tmp_path):
"""Each run's meta survives save()/load(), as AuditResults.load keeps it."""
meta = {"judge_model": "judge-x", "target_model": "m", "scenario_pack": "safety"}
run = AuditResults(_make_results(["pass"]).results, meta=meta)
results = RepeatedExperimentResults({"m": [run, _make_results(["low"])]})

path = str(tmp_path / "exp.json")
results.save(path)
loaded = RepeatedExperimentResults.load(path)

assert loaded._runs["m"][0].meta == meta
assert loaded._runs["m"][1].meta == {}
assert loaded.to_dict()["runs"]["m"][0]["meta"] == meta


# ---------------------------------------------------------------------------
# AuditExperiment — on_model_done callback
Expand Down
Loading