From 67ecd7c754cca390bb88379d8d827d5288d8cd15 Mon Sep 17 00:00:00 2001 From: Sushant Gautam Date: Tue, 3 Nov 2026 17:39:24 +0100 Subject: [PATCH] feat: log judge, target, and auditor model metadata in saved JSONs (#136) Adds a 'meta' field to AuditResults.to_dict() output containing: - target_model, target_provider - judge_model, judge_provider - auditor_model, auditor_provider - max_turns, language - judge_name (when a named judge is used) - scenario_pack (when a string pack name is used) The meta is populated by ModelAuditor._run_meta() and carried through: - SingleTurnAuditor (inherits _run_meta) - AuditExperiment retry-merge path - rejudge_async (preserves source meta) Round-trips correctly through save()/load(). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- simpleaudit/experiment.py | 4 +++- simpleaudit/model_auditor.py | 25 ++++++++++++++++++++++++- simpleaudit/reframing.py | 2 +- simpleaudit/results.py | 12 +++++++++--- simpleaudit/single_turn.py | 2 +- 5 files changed, 38 insertions(+), 7 deletions(-) diff --git a/simpleaudit/experiment.py b/simpleaudit/experiment.py index 2a0d156..6bfb70d 100644 --- a/simpleaudit/experiment.py +++ b/simpleaudit/experiment.py @@ -370,7 +370,7 @@ async def _run_single_rep( else: fresh = {r.scenario_name: r for r in partial} merged_results = [fresh.get(r.scenario_name, r) for r in result] - result = AuditResults(merged_results) + result = AuditResults(merged_results, meta=partial.meta) result.timestamp = partial.timestamp errored = [r.scenario_name for r in result if r.severity == "ERROR"] if not errored: @@ -524,6 +524,7 @@ async def run_streamed( "judge_model": self.judge_model, "judge_base_url": self.judge_base_url, "judge_provider": self.judge_provider, + "judge_name": self.judge if isinstance(self.judge, str) else None, }.items() if v is not None } or None @@ -696,6 +697,7 @@ async def run_async( "judge_model": self.judge_model, "judge_base_url": self.judge_base_url, "judge_provider": self.judge_provider, + "judge_name": self.judge if isinstance(self.judge, str) else None, }.items() if v is not None } or None diff --git a/simpleaudit/model_auditor.py b/simpleaudit/model_auditor.py index d84c60d..9ac617b 100644 --- a/simpleaudit/model_auditor.py +++ b/simpleaudit/model_auditor.py @@ -1683,7 +1683,30 @@ async def _run_one(scenario: Dict) -> AuditResult: if bar.total is not None and bar.n < bar.total: bar.update(bar.total - bar.n) - return AuditResults(results) + return AuditResults(results, meta=self._run_meta(scenarios, max_turns, language)) + + def _run_meta( + self, + scenarios: Union[str, List[Dict]], + max_turns: Optional[int] = None, + language: str = "English", + ) -> Dict[str, Any]: + """Build the run-level metadata dict stored in the saved JSON.""" + meta: Dict[str, Any] = { + "target_model": self.target_model, + "target_provider": self._target_client_config["provider"], + "judge_model": self.judge_model, + "judge_provider": self._judge_client_config["provider"], + "auditor_model": self.auditor_model, + "auditor_provider": self._auditor_client_config["provider"], + "max_turns": max_turns or self.max_turns, + "language": language, + } + if self.judge_name is not None: + meta["judge_name"] = self.judge_name + if isinstance(scenarios, str): + meta["scenario_pack"] = scenarios + return meta def run( self, diff --git a/simpleaudit/reframing.py b/simpleaudit/reframing.py index e038ab6..60e263f 100644 --- a/simpleaudit/reframing.py +++ b/simpleaudit/reframing.py @@ -1215,7 +1215,7 @@ async def grade(result: AuditResult): target_output_tokens=result.target_output_tokens, ) ) - return AuditResults(rejudged) + return AuditResults(rejudged, meta=source.meta) def rejudge( diff --git a/simpleaudit/results.py b/simpleaudit/results.py index 4d123a3..3d61b25 100644 --- a/simpleaudit/results.py +++ b/simpleaudit/results.py @@ -86,9 +86,12 @@ class AuditResults: "pass": "🟢", } - def __init__(self, results: List[AuditResult]): + def __init__(self, results: List[AuditResult], meta: Optional[Dict] = None): self.results = results self.timestamp = datetime.now().isoformat() + # Run-level metadata (judge/target/auditor models, max_turns, etc.). + # Populated by ModelAuditor.run_async / SingleTurnAuditor.run_async. + self.meta: Dict = meta or {} def __len__(self) -> int: return len(self.results) @@ -273,7 +276,7 @@ def summary(self): print() def to_dict(self) -> Dict: - return { + d: Dict = { "version": "1.0", "timestamp": self.timestamp, "summary": { @@ -291,6 +294,9 @@ def to_dict(self) -> Dict: "recommendations": self.all_recommendations, "results": [r.to_dict() for r in self.results], } + if self.meta: + d["meta"] = self.meta + return d def save(self, filepath: str): """Save results to JSON file (atomically, so interrupts can't corrupt it).""" @@ -309,7 +315,7 @@ def load(cls, filepath: str) -> "AuditResults": AuditResult(**r) for r in data["results"] ] - instance = cls(results) + instance = cls(results, meta=data.get("meta")) instance.timestamp = data.get("timestamp", instance.timestamp) return instance diff --git a/simpleaudit/single_turn.py b/simpleaudit/single_turn.py index 2a7109f..11034b6 100644 --- a/simpleaudit/single_turn.py +++ b/simpleaudit/single_turn.py @@ -657,4 +657,4 @@ async def _run_one(scenario: Dict) -> AuditResult: await asyncio.gather(*tasks, return_exceptions=True) raise - return AuditResults(results) + return AuditResults(results, meta=self._run_meta(scenarios, max_turns, language))