diff --git a/scripts/freeze_adds0928.py b/scripts/freeze_adds0928.py index ac60b024..3a5828ab 100644 --- a/scripts/freeze_adds0928.py +++ b/scripts/freeze_adds0928.py @@ -265,9 +265,9 @@ def main(argv: list[str] | None = None) -> None: manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") # Read-only input paths are all in this checkout. Override defaults that - # captured the old audit directory when freeze_snapshot was imported. + # captured the old audit directory when freeze_snapshot was imported. The + # freezer dates the response window from the staged predictions. freezer.SNAPSHOT_DATE = date - freezer.MODEL_RESPONSE_DATE = f"2026-06-12 to {date}" freezer.SOURCE_RUN = source_run freezer.SOURCE_US = source_us freezer.SOURCE_ANNOTATIONS = staged_annotations diff --git a/scripts/freeze_gpt61sol.py b/scripts/freeze_gpt61sol.py index c2297ea4..f4b27540 100644 --- a/scripts/freeze_gpt61sol.py +++ b/scripts/freeze_gpt61sol.py @@ -555,6 +555,14 @@ def main(argv: list[str] | None = None) -> None: ) evidence_paths[model] = str(state_path) del staged_rows + # The freezer dates the response window from these rows: from the run + # label's start to the last answer, not to the release date. Refuse here, + # before the pointer is written, rather than partway through the freeze. + window = freezer.model_response_window( + source_us / "predictions.csv", + freezer.MODEL_RESPONSE_START, + freezer.MODEL_RESPONSE_DATE, + ) adjudications = freezer.load_adjudications(staged_annotations / ADJUDICATIONS) from policybench.adjudications import verify_adjudications_applied @@ -593,14 +601,13 @@ def main(argv: list[str] | None = None) -> None: print( f"Validated local release inputs: {args.tag}, {BOARD_MODELS} models, " f"{payload_hash}; references unchanged; {added} adjudications added; " - f"{len(amendments)} wording amendments" + f"{len(amendments)} wording amendments; model responses {window}" ) return # Read-only input paths are all in this checkout. Override defaults that # captured the old audit directory when freeze_snapshot was imported. freezer.SNAPSHOT_DATE = date - freezer.MODEL_RESPONSE_DATE = f"2026-06-12 to {date}" freezer.SOURCE_RUN = source_run freezer.SOURCE_US = source_us freezer.SOURCE_ANNOTATIONS = staged_annotations diff --git a/scripts/freeze_snapshot.py b/scripts/freeze_snapshot.py index ccea1910..0692f2db 100644 --- a/scripts/freeze_snapshot.py +++ b/scripts/freeze_snapshot.py @@ -48,7 +48,7 @@ import re import shutil import subprocess -from datetime import date +from datetime import date, timedelta from pathlib import Path import numpy as np @@ -85,7 +85,14 @@ SNAPSHOT_DIR_NAME = "20260501" # Stable id; reused across refreshes. SNAPSHOT_DATE = "2026-09-22" -MODEL_RESPONSE_DATE = "2026-06-12 to 2026-09-22" +# The response window runs from the first wave to the last answer. The first +# waves' rows carry no request timestamps, so the start is the run label's date +# (us_full_run_20260612); the end is derived from the predictions +# (model_response_window), never taken from the release date. +MODEL_RESPONSE_START = "2026-06-12" +# A driver may name the window it expects; the freeze refuses unless the +# predictions give the same one. None names no window. +MODEL_RESPONSE_DATE: str | None = None RUN_LABEL = "us_full_run_20260612_policyengine_4_16_1_populace" # Completed runs live under the main clone's gitignored results/local. A @@ -443,6 +450,67 @@ def developer_adjudications_block(cases_dir: Path = AUDIT_CASES_DIR) -> dict: } +REQUEST_TIMESTAMP_COLUMNS = ("request_started_at", "request_completed_at") + + +def _utc_date(seconds: float) -> date: + """The UTC calendar date of a Unix time, on any machine's clock. + + Floor division, not ``datetime.fromtimestamp``, whose microsecond rounding + can carry a time just before midnight into the next day. + """ + return date(1970, 1, 1) + timedelta(days=int(seconds // 86_400)) + + +def model_response_window( + predictions: Path, start: str, expected: str | None = None +) -> str: + """The ``' to '`` window the predictions' answers fall in. + + ``end`` is the UTC date of the latest ``request_completed_at`` (Unix + seconds) the predictions record, whatever the release is called and + wherever it is frozen. Rows without timestamps (the first waves, which + predate the columns, and the batch adapter's) cannot move it. ``start`` is + configured, since those first waves recorded no time. Every timestamp the + predictions do record must fall inside the window, by UTC date. Refuses + predictions that time no answer, and an ``expected`` window that differs. + """ + columns = list(REQUEST_TIMESTAMP_COLUMNS) + missing = set(columns) - set(pd.read_csv(predictions, nrows=0).columns) + if missing: + raise SystemExit( + f"{predictions} has no {', '.join(sorted(missing))} column; " + "cannot date the model response window" + ) + frame = pd.read_csv(predictions, usecols=columns, dtype="float64") + completed = frame["request_completed_at"].dropna() + if completed.empty: + raise SystemExit( + f"{predictions} times no answer; cannot date the model response window" + ) + first, end = date.fromisoformat(start), _utc_date(completed.max()) + timed = frame.to_numpy() + earliest, latest = _utc_date(np.nanmin(timed)), _utc_date(np.nanmax(timed)) + if earliest < first: + raise SystemExit( + f"{predictions} records a request on {earliest}, before the " + f"configured model response start {start}" + ) + if latest > end: + raise SystemExit( + f"{predictions} records a request on {latest}, after its last " + f"answer ({end})" + ) + window = f"{first.isoformat()} to {end.isoformat()}" + if expected is not None and expected != window: + raise SystemExit( + f"The configured model response window ({expected}) is not the " + f"predictions' ({window}): the window ends on the last answer's " + "UTC date, not the release date" + ) + return window + + def _response_window_phrase(window: str) -> str: """'2026-06-12 to 2026-09-22' -> 'June 12 and September 22, 2026'.""" start, end = (date.fromisoformat(part.strip()) for part in window.split(" to ")) @@ -1477,6 +1545,7 @@ def build_manifest( run_files: dict[str, str], committed: dict[str, str], annotation_files: dict[str, str], + response_window: str, ) -> dict: reference_refresh = read_reference_refresh() engine_setup = read_reference_engine_setup() @@ -1532,7 +1601,7 @@ def build_manifest( "artifacts copied under " f"paper/snapshot/{SNAPSHOT_DIR_NAME}/runs/.", "Model responses were collected in waves between " - f"{_response_window_phrase(MODEL_RESPONSE_DATE)}, as models were " + f"{_response_window_phrase(response_window)}, as models were " "added to the board; each model's full 100-household run is a " "single consistent wave. PolicyBench computes each scored " "reference output with policyengine_us.Simulation from " @@ -1567,7 +1636,7 @@ def build_manifest( "run metadata (scenarios.csv.meta.json) records the " "populace_us_2024 build actually loaded.", "Model APIs and upstream model aliases may change after the " - f"recorded {MODEL_RESPONSE_DATE} response window, so exact " + f"recorded {response_window} response window, so exact " "reruns can diverge even with the committed household inputs, " "reference outputs, parsed dashboard export, and analysis " "summaries.", @@ -1583,7 +1652,7 @@ def build_manifest( "subsets per model as recorded in model_serving_config.json." ), }, - "model_response_date": MODEL_RESPONSE_DATE, + "model_response_date": response_window, "reference_output_refresh": reference_refresh, "household_dataset": household_dataset, "files": [ @@ -1764,13 +1833,17 @@ def main() -> None: raise SystemExit(f"Published dashboard not found: {PUBLISHED_DASHBOARD_SOURCE}") if not REFERENCE_META_SOURCE.exists(): raise SystemExit(f"Reference metadata not found: {REFERENCE_META_SOURCE}") + # Dated from the rows freeze_run gzips into the snapshot, before any write. + response_window = model_response_window( + SOURCE_US / "predictions.csv", MODEL_RESPONSE_START, MODEL_RESPONSE_DATE + ) remove_stale_artifacts() run_files = freeze_run() committed = freeze_committed_artifacts() annotation_files = freeze_annotations() - manifest = build_manifest(run_files, committed, annotation_files) + manifest = build_manifest(run_files, committed, annotation_files, response_window) # Pin the rendered-paper block to the served files. Falls back to the block # already recorded in the manifest when the paper has not been rendered yet. try: diff --git a/tests/test_finish_adds0928.py b/tests/test_finish_adds0928.py index 3aa8d033..2ff39bd5 100644 --- a/tests/test_finish_adds0928.py +++ b/tests/test_finish_adds0928.py @@ -1085,7 +1085,6 @@ def test_a_re_export_after_the_freeze_gates_the_committed_exclusions(monkeypatch # pass. Registering each with monkeypatch restores it after the test. FREEZER_ASSIGNED = ( "SNAPSHOT_DATE", - "MODEL_RESPONSE_DATE", "SOURCE_RUN", "SOURCE_US", "SOURCE_ANNOTATIONS", diff --git a/tests/test_model_response_window.py b/tests/test_model_response_window.py new file mode 100644 index 00000000..a6e65aea --- /dev/null +++ b/tests/test_model_response_window.py @@ -0,0 +1,297 @@ +"""The snapshot's model response window ends on the last answer it records. + +Release dashboard-data-20260930 dated the window's end by its tag: the drivers +set ``2026-06-12 to ``. GPT-6.1 Sol, the last model on that +board, finished answering at 2026-09-29 22:36 UTC and no model answered on +September 30. ``freeze_snapshot.model_response_window`` now dates the end from +the predictions' latest ``request_completed_at``, and the freeze refuses a +configured window that disagrees. +""" + +from __future__ import annotations + +import os +import sys +import tempfile +import time +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import freeze_snapshot as freezer # noqa: E402 + +START = "2026-06-12" + + +def utc(*parts: int) -> float: + return datetime(*parts, tzinfo=timezone.utc).timestamp() + + +# Release 20260930's last answer (GPT-6.1 Sol) and release 20260929's (Grok +# 4.7), as the frozen predictions record them. +LAST_ANSWER_20260930 = utc(2026, 9, 29, 22, 36, 32, 205154) +LAST_ANSWER_20260929 = utc(2026, 9, 29, 1, 26, 43, 931609) + + +def write_predictions(path: Path, rows: list[tuple]) -> Path: + """A predictions file whose rows are (request_started_at, + request_completed_at) pairs; None is a row without timestamps, as the + first waves' and the batch adapter's rows are.""" + frame = pd.DataFrame( + rows, columns=["request_started_at", "request_completed_at"], dtype="float64" + ) + frame.insert(0, "model", "model") + frame.to_csv(path, index=False) + return path + + +def test_the_window_ends_on_the_last_answer_not_the_release_date(tmp_path): + """The mismatch release 20260930 published: its freeze configured + "2026-06-12 to 2026-09-30" from the tag, while the last answer is + September 29's. The derived window ends on the 29th, and the configured + one is refused.""" + predictions = write_predictions( + tmp_path / "predictions.csv", + [(None, None), (LAST_ANSWER_20260930 - 3_600, LAST_ANSWER_20260930)], + ) + window = freezer.model_response_window(predictions, START) + assert window == "2026-06-12 to 2026-09-29" + released = "2026-06-12 to 2026-09-30" + with pytest.raises( + SystemExit, + match=r"\(2026-06-12 to 2026-09-30\) is not the predictions' " + r"\(2026-06-12 to 2026-09-29\)", + ): + freezer.model_response_window(predictions, START, released) + # A configured window that names the last answer passes. + assert freezer.model_response_window(predictions, START, window) == window + + +def test_rows_without_timestamps_cannot_move_the_end(tmp_path): + """The first waves predate the timestamp columns; their blank rows leave + the end on the last timed answer.""" + rows = [(None, None)] * 5 + [(utc(2026, 7, 2, 7, 50), utc(2026, 7, 2, 8, 40))] + predictions = write_predictions(tmp_path / "predictions.csv", rows) + assert freezer.model_response_window(predictions, START) == ( + "2026-06-12 to 2026-07-02" + ) + + +def test_predictions_that_time_no_answer_are_refused(tmp_path): + predictions = write_predictions(tmp_path / "predictions.csv", [(None, None)]) + with pytest.raises(SystemExit, match="times no answer"): + freezer.model_response_window(predictions, START) + untimed = tmp_path / "untimed.csv" + pd.DataFrame({"model": ["model"], "prediction": [1.0]}).to_csv(untimed) + with pytest.raises(SystemExit, match="no request_completed_at, request_started"): + freezer.model_response_window(untimed, START) + + +def test_a_start_after_the_first_timed_request_is_refused(tmp_path): + """The start is configured, but the recorded requests bound it: a request + on July 2 cannot sit in a window that opens on July 3.""" + predictions = write_predictions( + tmp_path / "predictions.csv", + [(utc(2026, 7, 2, 23, 59, 59), utc(2026, 7, 3, 0, 0, 1))], + ) + assert freezer.model_response_window(predictions, "2026-07-02") == ( + "2026-07-02 to 2026-07-03" + ) + with pytest.raises(SystemExit, match="request on 2026-07-02, before"): + freezer.model_response_window(predictions, "2026-07-03") + + +def test_a_request_started_after_the_last_answer_is_refused(tmp_path): + """A request that started after every completion (a row with a start and + no completion) would sit outside a window that ends on the last answer.""" + predictions = write_predictions( + tmp_path / "predictions.csv", + [(utc(2026, 9, 28, 20), utc(2026, 9, 28, 21)), (utc(2026, 9, 30, 1), None)], + ) + with pytest.raises(SystemExit, match="request on 2026-09-30, after its last"): + freezer.model_response_window(predictions, START) + + +def test_the_frozen_gzip_dates_like_the_staged_csv(tmp_path): + """The freeze dates the staged predictions.csv; the snapshot keeps its + deterministic gzip, which dates the same.""" + rows = [(None, None), (LAST_ANSWER_20260930 - 60, LAST_ANSWER_20260930)] + staged = write_predictions(tmp_path / "predictions.csv", rows) + frozen = tmp_path / "frozen" / "predictions.csv.gz" + frozen.parent.mkdir() + freezer.gzip_deterministic(staged, frozen, stored_name="predictions.csv") + assert freezer.model_response_window(frozen, START) == ( + freezer.model_response_window(staged, START) + ) + + +@pytest.mark.skipif(not hasattr(time, "tzset"), reason="needs time.tzset") +@pytest.mark.parametrize( + "zone", ["UTC", "America/New_York", "America/Los_Angeles", "Pacific/Kiritimati"] +) +@pytest.mark.parametrize( + "last_answer, end", + [ + # Release 20260929's last answer: September 28 in US time. + (LAST_ANSWER_20260929, "2026-09-29"), + # Release 20260901's (Claude Fable 5.1): September 1 in US time. + (utc(2026, 9, 2, 0, 29, 19), "2026-09-02"), + (LAST_ANSWER_20260930, "2026-09-29"), + ], +) +def test_the_end_is_the_utc_date_on_any_machine_clock(tmp_path, zone, last_answer, end): + predictions = write_predictions( + tmp_path / "predictions.csv", [(last_answer - 60, last_answer)] + ) + saved = os.environ.get("TZ") + os.environ["TZ"] = zone + time.tzset() + try: + window = freezer.model_response_window(predictions, START) + finally: + if saved is None: + del os.environ["TZ"] + else: + os.environ["TZ"] = saved + time.tzset() + assert window == f"2026-06-12 to {end}" + + +def test_a_time_just_before_utc_midnight_stays_on_its_day(): + """``datetime.fromtimestamp`` rounds to the microsecond and would carry + the largest float before midnight into the next day.""" + midnight = utc(2026, 9, 30) + before = float.__sub__(midnight, 2.5e-7) + assert before < midnight + assert freezer._utc_date(before) == date(2026, 9, 29) + assert freezer._utc_date(midnight) == date(2026, 9, 30) + + +# --- The freeze -------------------------------------------------------------- + + +@pytest.fixture +def freeze_inputs(tmp_path, monkeypatch): + """A source run whose predictions end on release 20260930's last answer, + with every step of freeze_snapshot.main that writes stubbed to record + that it ran.""" + source = tmp_path / "run" + (source / "us").mkdir(parents=True) + write_predictions( + source / "us" / "predictions.csv", + [(None, None), (LAST_ANSWER_20260930 - 60, LAST_ANSWER_20260930)], + ) + for name in ("dashboard.json", "reference_outputs.csv.meta.json"): + (tmp_path / name).write_text("{}\n") + monkeypatch.setattr(freezer, "SOURCE_RUN", source) + monkeypatch.setattr(freezer, "SOURCE_US", source / "us") + monkeypatch.setattr( + freezer, "PUBLISHED_DASHBOARD_SOURCE", tmp_path / "dashboard.json" + ) + monkeypatch.setattr( + freezer, "REFERENCE_META_SOURCE", tmp_path / "reference_outputs.csv.meta.json" + ) + monkeypatch.setattr(freezer, "MODEL_RESPONSE_DATE", None) + ran = [] + for step in ( + "remove_stale_artifacts", + "freeze_run", + "freeze_committed_artifacts", + "freeze_annotations", + ): + monkeypatch.setattr(freezer, step, lambda step=step: ran.append(step)) + + def build_manifest(run_files, committed, annotation_files, response_window): + raise RuntimeError(f"manifest window: {response_window}") + + monkeypatch.setattr(freezer, "build_manifest", build_manifest) + return ran + + +def test_the_freeze_dates_the_manifest_from_the_predictions(freeze_inputs): + with pytest.raises( + RuntimeError, match=r"^manifest window: 2026-06-12 to 2026-09-29$" + ): + freezer.main() + assert freeze_inputs == [ + "remove_stale_artifacts", + "freeze_run", + "freeze_committed_artifacts", + "freeze_annotations", + ] + + +def test_the_freeze_refuses_a_release_date_window_before_any_write( + freeze_inputs, monkeypatch +): + """What freeze_gpt61sol configured for dashboard-data-20260930.""" + monkeypatch.setattr(freezer, "MODEL_RESPONSE_DATE", "2026-06-12 to 2026-09-30") + with pytest.raises(SystemExit, match="not the release date"): + freezer.main() + assert freeze_inputs == [] + + +# --- Properties -------------------------------------------------------------- + +# Requests as the runner records them: microsecond Unix times (time.time()), +# within the board's span, each completing at or after it starts. +EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) +FIRST_US = int((datetime(2026, 6, 12, tzinfo=timezone.utc) - EPOCH).total_seconds()) +LAST_US = int((datetime(2027, 6, 30, tzinfo=timezone.utc) - EPOCH).total_seconds()) +REQUEST = st.tuples( + st.integers(FIRST_US * 1_000_000, LAST_US * 1_000_000), + st.integers(0, 6 * 3_600 * 1_000_000), +).map(lambda pair: (pair[0], pair[0] + pair[1])) +ROWS = st.lists(st.one_of(st.none(), REQUEST), min_size=1, max_size=40).filter( + lambda rows: any(rows) +) + + +def exact_date(microseconds: int) -> date: + """The UTC date of an integer microsecond time, in exact arithmetic.""" + return (EPOCH + timedelta(microseconds=microseconds)).date() + + +def window_of(rows: list[tuple[int, int] | None], start: str = START) -> str: + pairs = [ + (None, None) if row is None else (row[0] / 1e6, row[1] / 1e6) for row in rows + ] + with tempfile.TemporaryDirectory() as scratch: + path = write_predictions(Path(scratch) / "predictions.csv", pairs) + return freezer.model_response_window(path, start) + + +@settings(max_examples=150, deadline=None) +@given(ROWS) +def test_the_end_is_the_latest_completion_and_the_window_holds_every_request(rows): + window = window_of(rows) + first, end = (date.fromisoformat(part) for part in window.split(" to ")) + timed = [row for row in rows if row is not None] + assert first == date.fromisoformat(START) + assert end == exact_date(max(completed for _, completed in timed)) + for started, completed in timed: + assert first <= exact_date(started) <= exact_date(completed) <= end + + +@settings(max_examples=75, deadline=None) +@given(ROWS, st.randoms(use_true_random=False)) +def test_the_window_ignores_row_order(rows, random): + shuffled = list(rows) + random.shuffle(shuffled) + assert window_of(shuffled) == window_of(rows) + + +@settings(max_examples=75, deadline=None) +@given(ROWS, REQUEST) +def test_a_later_answer_never_moves_the_end_earlier(rows, extra): + before = window_of(rows).split(" to ")[1] + after = window_of([*rows, extra]).split(" to ")[1] + assert after >= before + assert after == max(before, exact_date(extra[1]).isoformat()) diff --git a/tests/test_snapshot_artifacts.py b/tests/test_snapshot_artifacts.py index 5bafde8c..c64a7007 100644 --- a/tests/test_snapshot_artifacts.py +++ b/tests/test_snapshot_artifacts.py @@ -956,6 +956,62 @@ def test_reference_refresh_date_is_the_generation_date_not_the_snapshot_date(): assert refresh["date"] <= refresh["snapshot_date"] +def _frozen_run_dir(manifest: dict) -> Path: + run_label = manifest["source_run_labels"]["us"] + return ROOT / manifest["source_run_artifacts"][run_label]["path"] + + +def test_frozen_predictions_and_usage_summary_agree_on_the_last_answer(): + """Two computations of the last answer agree: the freezer's window, from + the frozen predictions, and the frozen usage summary's latest + last_request_at, which policybench.analysis aggregates per model.""" + from scripts.freeze_snapshot import _utc_date, model_response_window + + manifest = json.loads((SNAPSHOT_DIR / "manifest.json").read_text()) + run_dir = _frozen_run_dir(manifest) + start = manifest["model_response_date"].split(" to ")[0] + window = model_response_window(run_dir / "predictions.csv.gz", start) + usage = pd.read_csv(run_dir / "analysis" / "usage_summary.csv") + assert window == f"{start} to {_utc_date(usage['last_request_at'].max())}" + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason=( + "Release dashboard-data-20260930 dated the window's end by its tag " + "(2026-09-30); the last answer is 2026-09-29 UTC. Whether to refreeze " + "now or at the next freeze is Max's call (cos decision d831). " + "Remove this mark with the freeze that corrects the manifest." + ), +) +def test_model_response_window_ends_on_the_last_frozen_answer(): + """The manifest's window ends on the UTC date of the last answer the frozen + predictions record, not on the release date, and its reproducibility + notes state the same window.""" + from scripts.freeze_snapshot import _response_window_phrase, model_response_window + + manifest = json.loads((SNAPSHOT_DIR / "manifest.json").read_text()) + start = manifest["model_response_date"].split(" to ")[0] + window = model_response_window( + _frozen_run_dir(manifest) / "predictions.csv.gz", start + ) + assert manifest["model_response_date"] == window + notes = " ".join(manifest["reproducibility_notes"]) + assert f"between {_response_window_phrase(window)}, as models" in notes + assert f"recorded {window} response window" in notes + + +def test_rendered_paper_states_the_manifest_response_window(): + """The rendered manuscript's snapshot table names the manifest's window, so + a manifest corrected without a re-render fails here.""" + manifest = json.loads((SNAPSHOT_DIR / "manifest.json").read_text()) + web = ROOT / manifest["rendered_paper_artifacts"]["web"]["path"] + page = (web / "index.html").read_text() + cells = re.findall(r"Model response date\s*([^<]*)", page) + assert cells == [manifest["model_response_date"]] + + def test_app_copy_of_serving_configuration_matches_the_frozen_file(): """The scenario explorer bundles a copy of the frozen serving config.