From bd416072ebb143c2e95984cb8be750ce62c36850 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 15:38:30 -0400 Subject: [PATCH 1/3] Record Google's re-route of the gemini-3.5-flash and gemini-3.7-flash names Google's Gemini API changelog entry of 2026-10-08 deprecates gemini-3.7-flash and gemini-3.5-flash: both left the AI Studio model picker, and requests under them route to gemini-3.8-flash and gemini-3.6-flash. On 2026-10-13 a request under each old name answered with the successor's modelVersion. The board's rows for both names were recorded before their successors were released (gemini-3.5-flash's answers are unchanged since 2026-06-15; gemini-3.7-flash's were requested 2026-08-18), so they are the original models, and the names can no longer re-run them. The DeepSeek note above records the same situation for deepseek-v4-flash and deepseek-v4-pro. The comments beside the two MODELS entries and a runbook paragraph say so. litellm 1.88.1 sets the response model to the requested name and never reads Gemini's modelVersion, so a re-run would still record the old name as provider_resolved_model. A snapshot test therefore pins the frozen rows' canonical hash and checks that every request time they record precedes the successor's release. No score, reference or data changes. Co-Authored-By: Claude Opus 5.5 --- docs/runbook.md | 7 ++++++ policybench/config.py | 18 +++++++++++++ tests/test_snapshot_artifacts.py | 43 ++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/docs/runbook.md b/docs/runbook.md index aaac2903..24bc1e27 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -204,6 +204,13 @@ already moved to the August V4 Pro release, so neither row can be re-run under its name. V4.1 Flash is the `deepseek-v4.1-flash` row, which calls DeepSeek's current name `deepseek-flash`. +Google deprecated `gemini-3.5-flash` and `gemini-3.7-flash` on 2026-10-08 +([changelog](https://ai.google.dev/gemini-api/docs/changelog)). Requests under +those names now route to `gemini-3.6-flash` and `gemini-3.8-flash`, which are +rows of their own, so neither row can be re-run under its name. A re-run would +not show the re-route: litellm records the requested name as +`provider_resolved_model` and drops Gemini's `modelVersion`. + OpenAI made [GPT-5.6 generally available](https://openai.com/index/gpt-5-6/) across ChatGPT, Codex, and the API on 2026-07-09, with a global rollout over 24 hours. Because these models are new to the PolicyBench harness, run the serving diff --git a/policybench/config.py b/policybench/config.py index b829b466..b5b335f3 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -67,8 +67,26 @@ "gpt-5.4-mini": "gpt-5.4-mini", "gpt-5.4-nano": "gpt-5.4-nano", "gemini-3.1-pro-preview": "gemini/gemini-3.1-pro-preview", + # Google deprecated gemini-3.5-flash and gemini-3.7-flash on 2026-10-08 + # (ai.google.dev/gemini-api/docs/changelog, entry of that date, read + # 2026-10-13): both left the AI Studio model picker, and requests under + # gemini-3.5-flash now route to gemini-3.6-flash. On 2026-10-13 a + # gemini-3.5-flash request answered with modelVersion gemini-3.6-flash. + # The row's answers are unchanged since the frozen predictions were first + # committed on 2026-06-15, before Gemini 3.6 Flash was released + # (2026-07-21), so they are Gemini 3.5 Flash, and this name can no longer + # re-run them. "gemini-3.5-flash": "gemini/gemini-3.5-flash", "gemini-3.6-flash": "gemini/gemini-3.6-flash", + # gemini-3.7-flash now routes to gemini-3.8-flash: on 2026-10-13 a request + # under that name answered with modelVersion gemini-3.8-flash. The row's + # answers were requested on 2026-08-18, before Gemini 3.8 Flash was + # released (2026-09-02), so they are Gemini 3.7 Flash, and this name can + # no longer re-run them. Neither re-route shows in a run's output: + # litellm 1.88.1 sets the response model to the requested name and never + # reads Gemini's modelVersion, so provider_resolved_model still records + # gemini-3.7-flash or gemini-3.5-flash, and GET models/ still + # returns each retired model's own version string. "gemini-3.7-flash": "gemini/gemini-3.7-flash", "gemini-3.8-flash": "gemini/gemini-3.8-flash", "gemini-3.5-flash-lite": "gemini/gemini-3.5-flash-lite", diff --git a/tests/test_snapshot_artifacts.py b/tests/test_snapshot_artifacts.py index 589b8548..2aaf794d 100644 --- a/tests/test_snapshot_artifacts.py +++ b/tests/test_snapshot_artifacts.py @@ -1629,6 +1629,49 @@ def test_model_response_window_ends_on_the_last_frozen_answer(): assert f"recorded {window} response window" in notes +# Board rows recorded under Gemini names that Google has since routed to +# another model (see the comments beside them in policybench/config.py): the +# model each name now reaches, and the canonical hash of the rows as frozen. +RETIRED_GEMINI_ROWS = { + "gemini-3.5-flash": ( + "gemini-3.6-flash", + "a3d965789f249042f4168dd6245a01a9b58b1c1bed197eea465ffdd19de3eabe", + ), + "gemini-3.7-flash": ( + "gemini-3.8-flash", + "6a3c0abbff6dd12c094fac1c7f301cffc81386cbb0a53a864930681bddfc4f7f", + ), +} + + +@pytest.mark.parametrize("model", sorted(RETIRED_GEMINI_ROWS)) +def test_board_rows_under_retired_gemini_names_stay_as_recorded(model): + """Since 2026-10-08 a request under gemini-3.5-flash or gemini-3.7-flash + reaches Gemini 3.6 Flash or 3.8 Flash, and the run still records the + requested name as provider_resolved_model. So neither row may be re-run or + repaired: its answers stay the ones frozen, and every request time it + records precedes the successor's release (the gemini-3.5-flash rows + predate request-time capture and record none).""" + from policybench.paper_results import MODEL_RELEASE_DATES + from scripts.freeze_snapshot import ( + PREDICTION_EVIDENCE_COLUMNS, + _model_prediction_rows_sha256, + ) + + successor, frozen_rows_sha256 = RETIRED_GEMINI_ROWS[model] + manifest = json.loads((SNAPSHOT_DIR / "manifest.json").read_text()) + predictions = pd.read_csv( + _frozen_run_dir(manifest) / "predictions.csv.gz", + usecols=["model", "request_started_at", *PREDICTION_EVIDENCE_COLUMNS], + ) + assert _model_prediction_rows_sha256(predictions, model) == frozen_rows_sha256 + successor_released = pd.Timestamp( + MODEL_RELEASE_DATES[successor], tz="UTC" + ).timestamp() + started = predictions.loc[predictions["model"] == model, "request_started_at"] + assert (started.dropna() < successor_released).all() + + def test_rendered_paper_states_the_manifest_response_window(): """The rendered manuscript's snapshot table names the manifest's window, so a manifest corrected without a re-render fails here.""" From 4c2cd379f1acce3a49f504c0238d4f723b0133bc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 16:05:59 -0400 Subject: [PATCH 2/3] Limit the hidden re-route to synchronous runs; the Gemini batch path records modelVersion Review r1 (GPT-6.1 Sol): GeminiBatchAdapter calls google-genai directly and records response.model_version as provider_resolved_model (batch_eval.py), so a batch run would show the re-route. Only the litellm path, which synchronous runs and row repairs use, records the requested name. The config comment, runbook paragraph and test docstring now say so. Co-Authored-By: Claude Opus 5.5 --- docs/runbook.md | 8 +++++--- policybench/config.py | 11 ++++++----- tests/test_snapshot_artifacts.py | 8 ++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/runbook.md b/docs/runbook.md index 24bc1e27..825252a6 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -207,9 +207,11 @@ DeepSeek's current name `deepseek-flash`. Google deprecated `gemini-3.5-flash` and `gemini-3.7-flash` on 2026-10-08 ([changelog](https://ai.google.dev/gemini-api/docs/changelog)). Requests under those names now route to `gemini-3.6-flash` and `gemini-3.8-flash`, which are -rows of their own, so neither row can be re-run under its name. A re-run would -not show the re-route: litellm records the requested name as -`provider_resolved_model` and drops Gemini's `modelVersion`. +rows of their own, so neither row can be re-run under its name. A synchronous +re-run or row repair would not show the re-route: litellm 1.88.1 sets the +response model to the requested name, which PolicyBench records as +`provider_resolved_model`, and ignores Gemini's `modelVersion`. The Gemini +batch path, which calls google-genai directly, does record `modelVersion`. OpenAI made [GPT-5.6 generally available](https://openai.com/index/gpt-5-6/) across ChatGPT, Codex, and the API on 2026-07-09, with a global rollout over 24 diff --git a/policybench/config.py b/policybench/config.py index b5b335f3..c85c5dd2 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -82,11 +82,12 @@ # under that name answered with modelVersion gemini-3.8-flash. The row's # answers were requested on 2026-08-18, before Gemini 3.8 Flash was # released (2026-09-02), so they are Gemini 3.7 Flash, and this name can - # no longer re-run them. Neither re-route shows in a run's output: - # litellm 1.88.1 sets the response model to the requested name and never - # reads Gemini's modelVersion, so provider_resolved_model still records - # gemini-3.7-flash or gemini-3.5-flash, and GET models/ still - # returns each retired model's own version string. + # no longer re-run them. A synchronous run, which row repairs also use, + # would not show either re-route: litellm 1.88.1 sets the response model + # to the requested name and never reads Gemini's modelVersion, so + # provider_resolved_model records gemini-3.7-flash or gemini-3.5-flash. + # The Gemini batch path (google-genai) does record modelVersion. GET + # models/ still returns each retired model's own version string. "gemini-3.7-flash": "gemini/gemini-3.7-flash", "gemini-3.8-flash": "gemini/gemini-3.8-flash", "gemini-3.5-flash-lite": "gemini/gemini-3.5-flash-lite", diff --git a/tests/test_snapshot_artifacts.py b/tests/test_snapshot_artifacts.py index 2aaf794d..30b919e2 100644 --- a/tests/test_snapshot_artifacts.py +++ b/tests/test_snapshot_artifacts.py @@ -1647,11 +1647,11 @@ def test_model_response_window_ends_on_the_last_frozen_answer(): @pytest.mark.parametrize("model", sorted(RETIRED_GEMINI_ROWS)) def test_board_rows_under_retired_gemini_names_stay_as_recorded(model): """Since 2026-10-08 a request under gemini-3.5-flash or gemini-3.7-flash - reaches Gemini 3.6 Flash or 3.8 Flash, and the run still records the - requested name as provider_resolved_model. So neither row may be re-run or - repaired: its answers stay the ones frozen, and every request time it + reaches Gemini 3.6 Flash or 3.8 Flash, and a synchronous run still records + the requested name as provider_resolved_model. So neither row may be re-run + or repaired: its answers stay the ones frozen, and every request time it records precedes the successor's release (the gemini-3.5-flash rows - predate request-time capture and record none).""" + predate request-time capture and record none, as batch runs do).""" from policybench.paper_results import MODEL_RELEASE_DATES from scripts.freeze_snapshot import ( PREDICTION_EVIDENCE_COLUMNS, From 6d91fb2a0a9895c22c446dec3be6a03c3ed282a3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 18:23:59 -0400 Subject: [PATCH 3/3] Pin the retired Gemini rows by a hash of their CSV text CI failed both cases: freeze_snapshot's canonical row hash gives a3d96578/6a3c0abb on macOS (Python 3.12 and 3.14 alike) and 9393d47c/049865e7 on the Linux runner, for the same predictions.csv.gz (the manifest's sha256 check passed there), with pandas 3.0.1 and numpy 2.4.2 on both. The test now reads the answer cells as strings, parses only predictions with Python's float, and adds provider_response_id, so pandas' dtype inference and float parsing no longer enter the hash. Co-Authored-By: Claude Opus 5.5 --- tests/test_snapshot_artifacts.py | 48 ++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/tests/test_snapshot_artifacts.py b/tests/test_snapshot_artifacts.py index 30b919e2..997d80e8 100644 --- a/tests/test_snapshot_artifacts.py +++ b/tests/test_snapshot_artifacts.py @@ -1631,19 +1631,38 @@ def test_model_response_window_ends_on_the_last_frozen_answer(): # Board rows recorded under Gemini names that Google has since routed to # another model (see the comments beside them in policybench/config.py): the -# model each name now reaches, and the canonical hash of the rows as frozen. +# model each name now reaches, and the hash of the rows' answers as frozen. RETIRED_GEMINI_ROWS = { "gemini-3.5-flash": ( "gemini-3.6-flash", - "a3d965789f249042f4168dd6245a01a9b58b1c1bed197eea465ffdd19de3eabe", + "d47d9d6595ccc0fcb230ead7ed13afe45c57122d6e037c9e7a60feb4406aa5ff", ), "gemini-3.7-flash": ( "gemini-3.8-flash", - "6a3c0abbff6dd12c094fac1c7f301cffc81386cbb0a53a864930681bddfc4f7f", + "614925842d04dea7122cda55b7f35a15053a709865fd7d7322626ef0f10d06f8", ), } +def _answers_sha256(predictions: pd.DataFrame, model: str) -> str: + """Hash a model's answers from the CSV text: the cells are read as strings, + and only predictions are parsed, by Python's float. freeze_snapshot's + canonical row hash gives one value on macOS and another on the Linux CI + runner for the same file, so it cannot be pinned here.""" + rows = sorted( + ( + row.scenario_id, + row.variable, + repr(float(row.prediction)) if row.prediction.strip() else "", + row.explanation.strip(), + row.raw_response.strip(), + row.provider_response_id.strip(), + ) + for row in predictions[predictions["model"] == model].itertuples(index=False) + ) + return hashlib.sha256(json.dumps(rows, ensure_ascii=False).encode()).hexdigest() + + @pytest.mark.parametrize("model", sorted(RETIRED_GEMINI_ROWS)) def test_board_rows_under_retired_gemini_names_stay_as_recorded(model): """Since 2026-10-08 a request under gemini-3.5-flash or gemini-3.7-flash @@ -1653,23 +1672,30 @@ def test_board_rows_under_retired_gemini_names_stay_as_recorded(model): records precedes the successor's release (the gemini-3.5-flash rows predate request-time capture and record none, as batch runs do).""" from policybench.paper_results import MODEL_RELEASE_DATES - from scripts.freeze_snapshot import ( - PREDICTION_EVIDENCE_COLUMNS, - _model_prediction_rows_sha256, - ) - successor, frozen_rows_sha256 = RETIRED_GEMINI_ROWS[model] + successor, answers_sha256 = RETIRED_GEMINI_ROWS[model] manifest = json.loads((SNAPSHOT_DIR / "manifest.json").read_text()) predictions = pd.read_csv( _frozen_run_dir(manifest) / "predictions.csv.gz", - usecols=["model", "request_started_at", *PREDICTION_EVIDENCE_COLUMNS], + usecols=[ + "model", + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_response_id", + "request_started_at", + ], + dtype=str, + keep_default_na=False, ) - assert _model_prediction_rows_sha256(predictions, model) == frozen_rows_sha256 + assert _answers_sha256(predictions, model) == answers_sha256 successor_released = pd.Timestamp( MODEL_RELEASE_DATES[successor], tz="UTC" ).timestamp() started = predictions.loc[predictions["model"] == model, "request_started_at"] - assert (started.dropna() < successor_released).all() + assert all(float(time) < successor_released for time in started if time) def test_rendered_paper_states_the_manifest_response_window():