From e9abf5208a3c71b21b1433eda57adfcce3ef8491 Mon Sep 17 00:00:00 2001 From: daanschouten-kaiko Date: Fri, 12 Jun 2026 18:31:19 +0200 Subject: [PATCH] Add gemini 3.1 flash lite both as model-to-be-evaluated as LLM judge. --- configs/medmarks-endpoints.toml | 9 +++++++++ medarc_verifiers/utils/judge_helpers.py | 20 ++++++++++++++++++-- tests/test_judge_helpers.py | 19 +++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/configs/medmarks-endpoints.toml b/configs/medmarks-endpoints.toml index be95de32..0c408ce2 100644 --- a/configs/medmarks-endpoints.toml +++ b/configs/medmarks-endpoints.toml @@ -111,6 +111,15 @@ api_client_type = "openai_chat_completions" temperature = 1 +[[endpoint]] +endpoint_id = "gemini-3_1-flash-lite" +model = "google/gemini-3.1-flash-lite" +api_client_type = "openai_chat_completions" + +[endpoint.sampling_args] +temperature = 1 + + [[endpoint]] endpoint_id = "gemma-3-12b-it" model = "google/gemma-3-12b-it" diff --git a/medarc_verifiers/utils/judge_helpers.py b/medarc_verifiers/utils/judge_helpers.py index 10f703b5..36b8ad4e 100644 --- a/medarc_verifiers/utils/judge_helpers.py +++ b/medarc_verifiers/utils/judge_helpers.py @@ -1,9 +1,21 @@ from dataclasses import dataclass, field import os from typing import Dict, Iterable, Optional, Tuple, Union +from urllib.parse import urlparse from .sampling_args import sanitize_sampling_args_for_openai from .prime_inference import PRIME_INFERENCE_URL, _resolve_include_usage +# Google's OpenAI-compatibility endpoint strictly validates request payloads and +# rejects fields it does not support (e.g. top_k) with 400 INVALID_ARGUMENT. +_GOOGLE_OPENAI_COMPAT_HOST = "generativelanguage.googleapis.com" +_GOOGLE_UNSUPPORTED_SAMPLING_KEYS = ("top_k", "min_p") + + +def _is_google_openai_compat(base_url: str | None) -> bool: + if not base_url: + return False + return urlparse(base_url).hostname == _GOOGLE_OPENAI_COMPAT_HOST + def _normalize_judge_name(name: str) -> str: return name.lower().replace("_", "-").replace("/", "-") @@ -80,7 +92,7 @@ def as_dict(self, *, include_usage: bool = True) -> Dict[str, Union[float, int, top_k=64, ), JudgeSamplingDefaults( - name="gemini-3", + name=("gemini-3", "gemini-3.1"), temperature=1.0, top_p=0.95, top_k=64, @@ -149,7 +161,8 @@ def judge_sampling_args_and_headers( Args: judge_name: The name of the judge model. - base_url: The base URL for the API. Used for Prime Inference detection. + base_url: The base URL for the API. Used for Prime Inference detection and + to drop sampling params rejected by Google's OpenAI-compatibility endpoint. timeout: Request timeout in seconds. include_usage: Whether to include usage reporting in extra_body. If None (default), checks MEDARC_INCLUDE_USAGE env var, then @@ -174,6 +187,9 @@ def judge_sampling_args_and_headers( effective_include_usage = _resolve_include_usage(include_usage, is_prime_inference) payload = judge_defaults.as_dict(include_usage=effective_include_usage) + if _is_google_openai_compat(base_url): + for key in _GOOGLE_UNSUPPORTED_SAMPLING_KEYS: + payload.pop(key, None) if reasoning_effort is not None and "reasoning_effort" in payload: if isinstance(reasoning_effort, str) and reasoning_effort.lower().strip() == "none": payload.pop("reasoning_effort", None) diff --git a/tests/test_judge_helpers.py b/tests/test_judge_helpers.py index 5086e276..545925fc 100644 --- a/tests/test_judge_helpers.py +++ b/tests/test_judge_helpers.py @@ -124,6 +124,25 @@ def test_judge_sampling_defaults_supports_multiple_names_per_defaults( assert result_47 == result_46 +def test_judge_sampling_drops_params_unsupported_by_google_openai_compat( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Google's OpenAI-compat endpoint rejects unknown fields like top_k with 400.""" + monkeypatch.delenv("PRIME_TEAM_ID", raising=False) + monkeypatch.delenv("MEDARC_INCLUDE_USAGE", raising=False) + + google_url = "https://generativelanguage.googleapis.com/v1beta/openai/" + result, _ = judge_sampling_args_and_headers("gemini-3.1-flash-lite", base_url=google_url) + assert "top_k" not in result.get("extra_body", {}) + assert "min_p" not in result.get("extra_body", {}) + assert result["temperature"] == 1.0 + assert result["top_p"] == 0.95 + + # Other base URLs keep the provider-recommended top_k. + result, _ = judge_sampling_args_and_headers("gemini-3.1-flash-lite", base_url="https://api.example.com/v1") + assert result["extra_body"]["top_k"] == 64 + + def test_judge_sampling_override_temperature_non_reasoning(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("PRIME_TEAM_ID", raising=False) monkeypatch.delenv("MEDARC_INCLUDE_USAGE", raising=False)