Skip to content

feat: Phase D5 scorecard uncertainty intervals, risk vs coverage, and manifest-backed subgroup analysis #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
txmed82 merged 1 commit into from
Sep 9, 2026
Merged

feat: Phase D5 scorecard uncertainty intervals, risk vs coverage, and manifest-backed subgroup analysis #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/NEXT_STATUS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ language — this table is the current status source of truth.
| A | partial | Brief + runnable lumen task v1 + strict verifier (#33-35), reference-policy A3 comparison (#44) | A3 trained-checkpoint comparison; A5 external reproduction (blocked: partner) |
| B | partial | Threat model + env scrubbing (#36), container backend (#38), bounded transfer (#39), remainder + attestation + CI image (#40), probes (#41) | B5 cloud mint/storage (blocked: cloud owner); registry image publication (commercial decision) |
| C | partial | Dossier + cited gate (#37), orphan-label refusal + provenance + fault robustness (#42) | C4 world-native distributions (need SOFA/GPU world revisions); C5 phantom (blocked: partner) |
| D | partial | Paired comparison + CIs (#43), compare CLI (#45), split manifests with patient/site grouping | D5 uncertainty views in scorecards (no dataset carries the metadata yet) |
| D | partial | Paired comparison + CIs (#43), compare CLI (#45), split manifests with patient/site grouping (#60, #61, #62), scorecard uncertainty views | D5 benchmark-level reporting with external study data |
| E | partial | Obs/action contract + A3 (#44), interactive streams (#46), LeRobot reader + MONAI (#51), honest video adapter (#54) | E4 media alignment (no test media; mp4 banned from VC); semantic output schemas (needs E contract design) |
| F | partial | Prefix-replay branching proof + seed caveat (#47), trajectory-backed forecast task + null baseline (#56), planning utility measurement and recipe (#58) | F1 trajectory benchmark imports; F4 closed-loop simulator branching |
| G | partial | SB3 PPO recipe + measurement (#48), runnable fixes (#50, #52), kernel-observed episode divergence (#57, #59) | Verifier-derived training rewards (projection withdrawn); Prime interop |
Expand Down
8 changes: 8 additions & 0 deletions src/or_audit/eval/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@
ToolEvent,
TraceStep,
)
from or_audit.eval.uncertainty import (
bootstrap_mean_ci,
clustered_bootstrap_mean_ci,
wilson_score_interval,
)
from or_audit.eval.vector import TrialVector, project
from or_audit.eval.worlds import (
WORLD_KIND_ENTRY_POINT_GROUP,
Expand Down Expand Up @@ -166,9 +171,11 @@
"WorldSpec",
"assemble_job_result",
"assert_bind",
"bootstrap_mean_ci",
"builtin_random_agent",
"clear_adapter_registry",
"clear_simulation_registry",
"clustered_bootstrap_mean_ci",
"determinism_at_least",
"discover_world_adapters",
"export_rl",
Expand Down Expand Up @@ -203,6 +210,7 @@
"run_cartesian_job",
"run_job",
"stream_adapters",
"wilson_score_interval",
"world_adapter_discovery",
"world_kind_key",
"world_kind_spec",
Expand Down
172 changes: 166 additions & 6 deletions src/or_audit/eval/scorecard.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,15 @@

import html
import json
from collections import defaultdict
from pathlib import Path
from statistics import fmean
from typing import Any

from or_audit.eval.job import JobResult
from or_audit.eval.contracts import MetricKind
from or_audit.eval.job import JobResult, TrialRecord
from or_audit.eval.sim.base import BACKEND_SYNTHETIC_STUB, BACKEND_UNKNOWN
from or_audit.eval.uncertainty import bootstrap_mean_ci, wilson_score_interval

STUB_HEADLINE = "NOT PHYSICAL EVIDENCE - SYNTHETIC STAND-IN"
METRICS_ONLY_HEADLINE = "METRICS-ONLY - NOT SAFETY-ATTESTED"
Expand All @@ -30,6 +33,27 @@ def _metrics_only(result: JobResult, world_engine: dict[str, Any] | None) -> boo
return bool((world_engine or {}).get("metrics_only"))


def _is_abstained(trial: TrialRecord) -> bool:
if any(gate.abstained for gate in trial.vector.gates):
return True
ab_metric = trial.vector.metric("abstained")
return bool(ab_metric is not None and ab_metric.value is True)


def _trial_subgroup(trial: TrialRecord) -> str:
"""Extract declared subgroup from trial trajectory/metadata; never invent seeds."""
for step in trial.trajectory:
sc = step.get("scenario") if isinstance(step, dict) else getattr(step, "scenario", None)
if sc is not None:
if isinstance(sc, dict):
subgroup = sc.get("subgroup") or sc.get("id")
else:
subgroup = getattr(sc, "subgroup", None) or getattr(sc, "id", None)
if subgroup:
return str(subgroup)
return ""


def scorecard_data(
result: JobResult,
*,
Expand Down Expand Up @@ -66,27 +90,105 @@ def scorecard_data(
"unassessable": len(values) - len(assessed),
}
if row["kind"] == "boolean":
true_count = assessed.count(True)
rate = true_count / len(assessed) if assessed else None
ci_95 = list(wilson_score_interval(true_count, len(assessed))) if assessed else None
row.update(
{
"true": assessed.count(True),
"true": true_count,
"false": assessed.count(False),
"rate": assessed.count(True) / len(assessed) if assessed else None,
"rate": rate,
"ci_95": ci_95,
}
)
elif row["kind"] == "continuous":
numeric = [float(value) for value in assessed]
ci_95 = list(bootstrap_mean_ci(numeric)) if numeric else None
row.update(
{
"mean": fmean(numeric) if numeric else None,
"min": min(numeric) if numeric else None,
"max": max(numeric) if numeric else None,
"ci_95": ci_95,
}
)
else:
row["counts"] = {
category: assessed.count(category) for category in sorted(set(assessed))
}
metrics.append(row)
headline_outcome = next((m for m in result.trials[0].vector.metrics if m.headline), None)
headline_kind = headline_outcome.kind if headline_outcome else MetricKind.BOOLEAN

# Coverage & Risk analysis for abstaining models (Phase D5)
abstained_count = sum(1 for trial in result.trials if _is_abstained(trial))
coverage = (result.n - abstained_count) / result.n if result.n > 0 else 0.0
covered_trials = [t for t in result.trials if not _is_abstained(t)]
risk_at_coverage: float | None = None
if covered_trials:
if headline_kind is MetricKind.BOOLEAN:
failed_count = sum(
1
for t in covered_trials
if t.vector.any_gate_failed or t.vector.headline.value is False
)
else:
failed_count = sum(1 for t in covered_trials if t.vector.any_gate_failed)
risk_at_coverage = round(failed_count / len(covered_trials), 4)
coverage_report = {
"abstained": abstained_count,
"coverage": round(coverage, 4),
"risk_at_coverage": risk_at_coverage,
}

# Subgroups & Worst-Case Scenario Analysis (Phase D5)
# Subgroups must come from declared manifest metadata only; absent metadata reports unavailable.
scenario_trials: dict[str, list[TrialRecord]] = defaultdict(list)
for trial in result.trials:
sg = _trial_subgroup(trial)
if sg:
scenario_trials[sg].append(trial)

subgroups = []
worst_case = None
if len(scenario_trials) > 1:
worst_rate = 1.1

@factory-droid factory-droid Bot Sep 9, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Worst-case subgroup selection can be None or inverted for continuous metrics

worst_rate is initialized to 1.1 and the selector always treats smaller s_rate as worse; for continuous headline means that can exceed 1.1, worst_case can incorrectly remain None, and for continuous metrics with direction == "minimize" the worst subgroup should be the maximum mean, not the minimum. Initialize with ±inf and compare based on headline_outcome.direction for the continuous branch.

for sc_id, s_trials in sorted(scenario_trials.items()):
s_count = len(s_trials)
if headline_kind is MetricKind.BOOLEAN:
s_pass = sum(
1
for t in s_trials
if not t.vector.any_gate_failed and t.vector.headline.value is True
)
s_rate = s_pass / s_count if s_count > 0 else 0.0
ci = list(wilson_score_interval(s_pass, s_count)) if s_count > 0 else [0.0, 1.0]
elif headline_kind is MetricKind.CONTINUOUS:
s_vals = [
float(t.vector.headline.value)
for t in s_trials
if t.vector.headline.value is not None
]
s_rate = fmean(s_vals) if s_vals else 0.0

@factory-droid factory-droid Bot Sep 9, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Continuous subgroup stats report 0.0 with CI [0, 0] when nothing was assessed

In the continuous subgroup branch, an empty s_vals currently yields s_rate = 0.0 and ci = [0.0, 0.0], which fabricates a measured zero and a zero-width interval. This is inconsistent with the top-level continuous metric row (which uses None when unassessable) and it can wrongly dominate worst_case. Consider emitting rate=None/ci_95=None (and rendering n/a), and skip such subgroups when computing worst_case.

ci = list(bootstrap_mean_ci(s_vals)) if s_vals else [0.0, 0.0]
s_pass = sum(1 for t in s_trials if not t.vector.any_gate_failed)
else:
s_pass = sum(1 for t in s_trials if not t.vector.any_gate_failed)
s_rate = s_pass / s_count if s_count > 0 else 0.0
ci = list(wilson_score_interval(s_pass, s_count)) if s_count > 0 else [0.0, 1.0]

entry = {
"subgroup": sc_id,

@factory-droid factory-droid Bot Sep 9, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] [security] Subgroup labels are emitted verbatim into scorecards

_trial_subgroup() returns scenario.subgroup/scenario.id verbatim and the value is written into scorecard.json and rendered into scorecard.md; if scenario IDs encode patient/site identifiers (a plausible footgun in this domain) or contain special characters, this can leak or inject confusing content into shared artifacts. Consider hashing/allowlisting subgroup identifiers (or explicitly documenting they must be non-sensitive and safe-to-render).

"count": s_count,
"pass": s_pass,
"rate": round(s_rate, 4),
"ci_95": ci,
"is_underpowered": s_count < 10,
}
subgroups.append(entry)
if s_rate < worst_rate:
worst_rate = s_rate
worst_case = entry
return {
"task_id": result.task_id,
"task_version": result.task_version,
Expand All @@ -106,6 +208,11 @@ def scorecard_data(
"claim_footer": result.claim_footer,
"metrics_only": _metrics_only(result, world_engine),
"head": result.head,
"independent_cases": result.independent_cases,

@factory-droid factory-droid Bot Sep 9, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P0] Preserve head-covered world_engine provenance in scorecards

scorecard_data() currently sets world_engine only from the optional world_engine argument, so render_markdown()/render_html() can show backend/engine as unknown and skip the synthetic-stub banner even when JobResult.world_engine was recorded at run time and hashed into the artifact head. Prefer result.world_engine.model_dump(mode="json") when present, and fall back to the world_engine parameter only for legacy callers.

"split_manifest_digest": result.split_manifest_digest,
"coverage": coverage_report,
"subgroups": subgroups,
"worst_case": worst_case,
}


Expand Down Expand Up @@ -154,6 +261,14 @@ def render_markdown(
f"- Task digest: `{data['task_digest']}`",
f"- Agent digest: `{data['agent_digest']}`",
f"- Artifact head: `{data['head']}`",
]
)
if data.get("independent_cases") is not None:
lines.append(f"- Independent cases: `{data['independent_cases']}`")
if data.get("split_manifest_digest"):
lines.append(f"- Split manifest digest: `{data['split_manifest_digest']}`")
lines.extend(
[
"",
"## Safety gates",
"",
Expand All @@ -171,8 +286,8 @@ def render_markdown(
"",
"## Metrics",
"",
"| Metric | Headline | Result | Assessed | Unassessable |",
"|---|:---:|---:|---:|---:|",
"| Metric | Headline | Result | 95% CI | Assessed | Unassessable |",
"|---|:---:|---:|:---:|---:|---:|",
]
)
for metric in data["metrics"]:
Expand All @@ -185,10 +300,55 @@ def render_markdown(
", ".join(f"{category}: {count}" for category, count in metric["counts"].items())
or "n/a"
)
ci_str = (
f"[{metric['ci_95'][0]:.4f}, {metric['ci_95'][1]:.4f}]"
if metric.get("ci_95")
else "n/a"
)
lines.append(
f"| {metric['id']} | {'yes' if metric['headline'] else 'no'} | {value} | "
f"{metric['assessed']} | {metric['unassessable']} |"
f"{ci_str} | {metric['assessed']} | {metric['unassessable']} |"
)
if data["coverage"]["abstained"] > 0:
cov = data["coverage"]
lines.extend(
[
"",
"## Risk vs coverage",
"",
(
f"- Model coverage: `{cov['coverage'] * 100:.1f}%` "
f"({data['n'] - cov['abstained']}/{data['n']} non-abstained)"
),
(
f"- Risk at coverage: `{cov['risk_at_coverage'] * 100:.1f}%`"
if cov["risk_at_coverage"] is not None
else "- Risk at coverage: n/a"
),
]
)
if len(data["subgroups"]) > 1:
lines.extend(
[
"",
"## Subgroups and worst-case analysis",
"",
"| Subgroup | Count | Pass rate | 95% CI | Power |",
"|---|---:|---:|:---:|:---:|",
]
)
for sg in data["subgroups"]:
ci_str = f"[{sg['ci_95'][0]:.4f}, {sg['ci_95'][1]:.4f}]"
pwr = "underpowered (<10)" if sg["is_underpowered"] else "adequate"
lines.append(
f"| {sg['subgroup']} | {sg['count']} | {sg['rate']:.4f} | {ci_str} | {pwr} |"
)
if data["worst_case"]:
wc = data["worst_case"]
lines.append(
f"\n> **Worst-case subgroup:** `{wc['subgroup']}` "
f"with pass rate `{wc['rate']:.4f}`."
)
if data["claim_footer"]:
lines.extend(["", "## Claim boundary", "", data["claim_footer"]])
lines.extend(
Expand Down
Loading
Loading