diff --git a/docs/runbook.md b/docs/runbook.md index aaac2903..825252a6 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -204,6 +204,15 @@ already moved to the August V4 Pro release, so neither row can be re-run under its name. V4.1 Flash is the `deepseek-v4.1-flash` row, which calls DeepSeek's current name `deepseek-flash`. +Google deprecated `gemini-3.5-flash` and `gemini-3.7-flash` on 2026-10-08 +([changelog](https://ai.google.dev/gemini-api/docs/changelog)). Requests under +those names now route to `gemini-3.6-flash` and `gemini-3.8-flash`, which are +rows of their own, so neither row can be re-run under its name. A synchronous +re-run or row repair would not show the re-route: litellm 1.88.1 sets the +response model to the requested name, which PolicyBench records as +`provider_resolved_model`, and ignores Gemini's `modelVersion`. The Gemini +batch path, which calls google-genai directly, does record `modelVersion`. + OpenAI made [GPT-5.6 generally available](https://openai.com/index/gpt-5-6/) across ChatGPT, Codex, and the API on 2026-07-09, with a global rollout over 24 hours. Because these models are new to the PolicyBench harness, run the serving diff --git a/policybench/config.py b/policybench/config.py index b829b466..c85c5dd2 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -67,8 +67,27 @@ "gpt-5.4-mini": "gpt-5.4-mini", "gpt-5.4-nano": "gpt-5.4-nano", "gemini-3.1-pro-preview": "gemini/gemini-3.1-pro-preview", + # Google deprecated gemini-3.5-flash and gemini-3.7-flash on 2026-10-08 + # (ai.google.dev/gemini-api/docs/changelog, entry of that date, read + # 2026-10-13): both left the AI Studio model picker, and requests under + # gemini-3.5-flash now route to gemini-3.6-flash. On 2026-10-13 a + # gemini-3.5-flash request answered with modelVersion gemini-3.6-flash. + # The row's answers are unchanged since the frozen predictions were first + # committed on 2026-06-15, before Gemini 3.6 Flash was released + # (2026-07-21), so they are Gemini 3.5 Flash, and this name can no longer + # re-run them. "gemini-3.5-flash": "gemini/gemini-3.5-flash", "gemini-3.6-flash": "gemini/gemini-3.6-flash", + # gemini-3.7-flash now routes to gemini-3.8-flash: on 2026-10-13 a request + # under that name answered with modelVersion gemini-3.8-flash. The row's + # answers were requested on 2026-08-18, before Gemini 3.8 Flash was + # released (2026-09-02), so they are Gemini 3.7 Flash, and this name can + # no longer re-run them. A synchronous run, which row repairs also use, + # would not show either re-route: litellm 1.88.1 sets the response model + # to the requested name and never reads Gemini's modelVersion, so + # provider_resolved_model records gemini-3.7-flash or gemini-3.5-flash. + # The Gemini batch path (google-genai) does record modelVersion. GET + # models/ still returns each retired model's own version string. "gemini-3.7-flash": "gemini/gemini-3.7-flash", "gemini-3.8-flash": "gemini/gemini-3.8-flash", "gemini-3.5-flash-lite": "gemini/gemini-3.5-flash-lite", diff --git a/tests/test_snapshot_artifacts.py b/tests/test_snapshot_artifacts.py index 589b8548..997d80e8 100644 --- a/tests/test_snapshot_artifacts.py +++ b/tests/test_snapshot_artifacts.py @@ -1629,6 +1629,75 @@ def test_model_response_window_ends_on_the_last_frozen_answer(): assert f"recorded {window} response window" in notes +# Board rows recorded under Gemini names that Google has since routed to +# another model (see the comments beside them in policybench/config.py): the +# model each name now reaches, and the hash of the rows' answers as frozen. +RETIRED_GEMINI_ROWS = { + "gemini-3.5-flash": ( + "gemini-3.6-flash", + "d47d9d6595ccc0fcb230ead7ed13afe45c57122d6e037c9e7a60feb4406aa5ff", + ), + "gemini-3.7-flash": ( + "gemini-3.8-flash", + "614925842d04dea7122cda55b7f35a15053a709865fd7d7322626ef0f10d06f8", + ), +} + + +def _answers_sha256(predictions: pd.DataFrame, model: str) -> str: + """Hash a model's answers from the CSV text: the cells are read as strings, + and only predictions are parsed, by Python's float. freeze_snapshot's + canonical row hash gives one value on macOS and another on the Linux CI + runner for the same file, so it cannot be pinned here.""" + rows = sorted( + ( + row.scenario_id, + row.variable, + repr(float(row.prediction)) if row.prediction.strip() else "", + row.explanation.strip(), + row.raw_response.strip(), + row.provider_response_id.strip(), + ) + for row in predictions[predictions["model"] == model].itertuples(index=False) + ) + return hashlib.sha256(json.dumps(rows, ensure_ascii=False).encode()).hexdigest() + + +@pytest.mark.parametrize("model", sorted(RETIRED_GEMINI_ROWS)) +def test_board_rows_under_retired_gemini_names_stay_as_recorded(model): + """Since 2026-10-08 a request under gemini-3.5-flash or gemini-3.7-flash + reaches Gemini 3.6 Flash or 3.8 Flash, and a synchronous run still records + the requested name as provider_resolved_model. So neither row may be re-run + or repaired: its answers stay the ones frozen, and every request time it + records precedes the successor's release (the gemini-3.5-flash rows + predate request-time capture and record none, as batch runs do).""" + from policybench.paper_results import MODEL_RELEASE_DATES + + successor, answers_sha256 = RETIRED_GEMINI_ROWS[model] + manifest = json.loads((SNAPSHOT_DIR / "manifest.json").read_text()) + predictions = pd.read_csv( + _frozen_run_dir(manifest) / "predictions.csv.gz", + usecols=[ + "model", + "scenario_id", + "variable", + "prediction", + "explanation", + "raw_response", + "provider_response_id", + "request_started_at", + ], + dtype=str, + keep_default_na=False, + ) + assert _answers_sha256(predictions, model) == answers_sha256 + successor_released = pd.Timestamp( + MODEL_RELEASE_DATES[successor], tz="UTC" + ).timestamp() + started = predictions.loc[predictions["model"] == model, "request_started_at"] + assert all(float(time) < successor_released for time in started if time) + + def test_rendered_paper_states_the_manifest_response_window(): """The rendered manuscript's snapshot table names the manifest's window, so a manifest corrected without a re-render fails here."""