Record Google's re-route of the gemini-3.5-flash and gemini-3.7-flash names

docs/runbook.md

@@ -204,6 +204,15 @@ already moved to the August V4 Pro release, so neither row can be re-run under
 its name. V4.1 Flash is the `deepseek-v4.1-flash` row, which calls
 DeepSeek's current name `deepseek-flash`.
 
+Google deprecated `gemini-3.5-flash` and `gemini-3.7-flash` on 2026-10-08
+([changelog](https://ai.google.dev/gemini-api/docs/changelog)). Requests under
+those names now route to `gemini-3.6-flash` and `gemini-3.8-flash`, which are
+rows of their own, so neither row can be re-run under its name. A synchronous
+re-run or row repair would not show the re-route: litellm 1.88.1 sets the
+response model to the requested name, which PolicyBench records as
+`provider_resolved_model`, and ignores Gemini's `modelVersion`. The Gemini
+batch path, which calls google-genai directly, does record `modelVersion`.
+
 OpenAI made [GPT-5.6 generally available](https://openai.com/index/gpt-5-6/)
 across ChatGPT, Codex, and the API on 2026-07-09, with a global rollout over 24
 hours. Because these models are new to the PolicyBench harness, run the serving

policybench/config.py

@@ -67,8 +67,27 @@
     "gpt-5.4-mini": "gpt-5.4-mini",
     "gpt-5.4-nano": "gpt-5.4-nano",
     "gemini-3.1-pro-preview": "gemini/gemini-3.1-pro-preview",
+    # Google deprecated gemini-3.5-flash and gemini-3.7-flash on 2026-10-08
+    # (ai.google.dev/gemini-api/docs/changelog, entry of that date, read
+    # 2026-10-13): both left the AI Studio model picker, and requests under
+    # gemini-3.5-flash now route to gemini-3.6-flash. On 2026-10-13 a
+    # gemini-3.5-flash request answered with modelVersion gemini-3.6-flash.
+    # The row's answers are unchanged since the frozen predictions were first
+    # committed on 2026-06-15, before Gemini 3.6 Flash was released
+    # (2026-07-21), so they are Gemini 3.5 Flash, and this name can no longer
+    # re-run them.
     "gemini-3.5-flash": "gemini/gemini-3.5-flash",
     "gemini-3.6-flash": "gemini/gemini-3.6-flash",
+    # gemini-3.7-flash now routes to gemini-3.8-flash: on 2026-10-13 a request
+    # under that name answered with modelVersion gemini-3.8-flash. The row's
+    # answers were requested on 2026-08-18, before Gemini 3.8 Flash was
+    # released (2026-09-02), so they are Gemini 3.7 Flash, and this name can
+    # no longer re-run them. A synchronous run, which row repairs also use,
+    # would not show either re-route: litellm 1.88.1 sets the response model
+    # to the requested name and never reads Gemini's modelVersion, so
+    # provider_resolved_model records gemini-3.7-flash or gemini-3.5-flash.
+    # The Gemini batch path (google-genai) does record modelVersion. GET
+    # models/<name> still returns each retired model's own version string.
     "gemini-3.7-flash": "gemini/gemini-3.7-flash",
     "gemini-3.8-flash": "gemini/gemini-3.8-flash",
     "gemini-3.5-flash-lite": "gemini/gemini-3.5-flash-lite",

tests/test_snapshot_artifacts.py

@@ -1629,6 +1629,75 @@ def test_model_response_window_ends_on_the_last_frozen_answer():
     assert f"recorded {window} response window" in notes
 
 
+# Board rows recorded under Gemini names that Google has since routed to
+# another model (see the comments beside them in policybench/config.py): the
+# model each name now reaches, and the hash of the rows' answers as frozen.
+RETIRED_GEMINI_ROWS = {
+    "gemini-3.5-flash": (
+        "gemini-3.6-flash",
+        "d47d9d6595ccc0fcb230ead7ed13afe45c57122d6e037c9e7a60feb4406aa5ff",
+    ),
+    "gemini-3.7-flash": (
+        "gemini-3.8-flash",
+        "614925842d04dea7122cda55b7f35a15053a709865fd7d7322626ef0f10d06f8",
+    ),
+}
+
+
+def _answers_sha256(predictions: pd.DataFrame, model: str) -> str:
+    """Hash a model's answers from the CSV text: the cells are read as strings,
+    and only predictions are parsed, by Python's float. freeze_snapshot's
+    canonical row hash gives one value on macOS and another on the Linux CI
+    runner for the same file, so it cannot be pinned here."""
+    rows = sorted(
+        (
+            row.scenario_id,
+            row.variable,
+            repr(float(row.prediction)) if row.prediction.strip() else "",
+            row.explanation.strip(),
+            row.raw_response.strip(),
+            row.provider_response_id.strip(),
+        )
+        for row in predictions[predictions["model"] == model].itertuples(index=False)
+    )
+    return hashlib.sha256(json.dumps(rows, ensure_ascii=False).encode()).hexdigest()
+
+
+@pytest.mark.parametrize("model", sorted(RETIRED_GEMINI_ROWS))
+def test_board_rows_under_retired_gemini_names_stay_as_recorded(model):
+    """Since 2026-10-08 a request under gemini-3.5-flash or gemini-3.7-flash
+    reaches Gemini 3.6 Flash or 3.8 Flash, and a synchronous run still records
+    the requested name as provider_resolved_model. So neither row may be re-run
+    or repaired: its answers stay the ones frozen, and every request time it
+    records precedes the successor's release (the gemini-3.5-flash rows
+    predate request-time capture and record none, as batch runs do)."""
+    from policybench.paper_results import MODEL_RELEASE_DATES
+
+    successor, answers_sha256 = RETIRED_GEMINI_ROWS[model]
+    manifest = json.loads((SNAPSHOT_DIR / "manifest.json").read_text())
+    predictions = pd.read_csv(
+        _frozen_run_dir(manifest) / "predictions.csv.gz",
+        usecols=[
+            "model",
+            "scenario_id",
+            "variable",
+            "prediction",
+            "explanation",
+            "raw_response",
+            "provider_response_id",
+            "request_started_at",
+        ],
+        dtype=str,
+        keep_default_na=False,
+    )
+    assert _answers_sha256(predictions, model) == answers_sha256
+    successor_released = pd.Timestamp(
+        MODEL_RELEASE_DATES[successor], tz="UTC"
+    ).timestamp()
+    started = predictions.loc[predictions["model"] == model, "request_started_at"]
+    assert all(float(time) < successor_released for time in started if time)
+
+
 def test_rendered_paper_states_the_manifest_response_window():
     """The rendered manuscript's snapshot table names the manifest's window, so
     a manifest corrected without a re-render fails here."""