From d85417187ed5c4e6669368036d2204fbf4652931 Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Fri, 18 Sep 2026 22:46:11 +0800 Subject: [PATCH] refactor(models): remove PR labels from runtime errors Replace internal PR and issue references in user-visible dialect, capability, and legacy-bypass errors with stable descriptions of the current runtime behavior. --- sieval/cli/leaderboard/session.py | 4 ++-- sieval/cli/validation.py | 2 +- sieval/core/models/dialect_registry.py | 8 ++++---- sieval/core/models/dialects/anthropic_messages.py | 10 +++++++--- sieval/core/models/dialects/openai_chat.py | 4 ++-- sieval/core/models/sglang_gen_model.py | 6 +++--- tests/unit/cli/test_validation.py | 4 ++-- .../models/dialects/test_anthropic_messages.py | 9 +++++++-- .../unit/core/models/dialects/test_openai_chat.py | 15 ++++++++++++++- tests/unit/core/models/test_dialect_registry.py | 6 +++--- 10 files changed, 45 insertions(+), 23 deletions(-) diff --git a/sieval/cli/leaderboard/session.py b/sieval/cli/leaderboard/session.py index 53e4ab6b..df41795c 100644 --- a/sieval/cli/leaderboard/session.py +++ b/sieval/cli/leaderboard/session.py @@ -832,7 +832,7 @@ def reconcile( CheckStage.REQUEST, verifier, requirement.reason - or "PR-1 compatibility requires response-time verification", + or "legacy compatibility requires response-time verification", ) return outcomes @@ -3301,7 +3301,7 @@ def replace_inline( raise ValueError( f"Model binding '{binding_id}' uses the temporary " "sglang_legacy bypass and cannot declare canonical " - "capabilities before the sglang_native PR-5 binder" + "capabilities while sglang_native has no executable binder" ) legacy_bypass.add(binding_id) continue diff --git a/sieval/cli/validation.py b/sieval/cli/validation.py index 7749630f..5361e2e6 100644 --- a/sieval/cli/validation.py +++ b/sieval/cli/validation.py @@ -308,7 +308,7 @@ def _validate_capabilities(cfg: dict, result: ValidationResult) -> None: result.errors.append( f"Model '{name}': the temporary sglang_legacy bypass " "cannot declare canonical capabilities before the " - "sglang_native PR-5 binder" + "sglang_native executable binder is available" ) continue diff --git a/sieval/core/models/dialect_registry.py b/sieval/core/models/dialect_registry.py index 5724014a..9340eb34 100644 --- a/sieval/core/models/dialect_registry.py +++ b/sieval/core/models/dialect_registry.py @@ -546,12 +546,12 @@ def bind_dialect( def _not_implemented_message(dialect_id: str) -> str: if dialect_id == "sglang_native": return ( - "dialect 'sglang_native' has no PR-1 binder; the one-cycle SGLang " - "compatibility path remains an explicit legacy bypass" + "dialect 'sglang_native' has no executable binder; use the explicit " + "'sglang_legacy' compatibility bypass" ) if dialect_id == "vllm_native": - return "dialect 'vllm_native' is reserved and explicitly deferred after #25" - return f"dialect {dialect_id!r} is reserved for a later #25 adapter PR" + return "dialect 'vllm_native' is reserved and has no executable binder" + return f"dialect {dialect_id!r} is reserved and has no executable binder" def dialect_registry_to_json() -> dict[str, JSONValue]: diff --git a/sieval/core/models/dialects/anthropic_messages.py b/sieval/core/models/dialects/anthropic_messages.py index 07763f74..19520c25 100644 --- a/sieval/core/models/dialects/anthropic_messages.py +++ b/sieval/core/models/dialects/anthropic_messages.py @@ -205,7 +205,7 @@ def _validate_multimodal_config(options: object) -> None: ) ), "hosted_tools": Unsupported( - "PR 3 does not bind Anthropic server-hosted tools" + "anthropic_messages does not support Anthropic server-hosted tools" ), "structured_output": Supported( DialectCapabilityBinding( @@ -2167,7 +2167,10 @@ def validate_request( "or 'none'", ) elif path == "tools.hosted": - audit.rejected(path, "Anthropic hosted tools are not active in PR 3") + audit.rejected( + path, + "Anthropic hosted tools are not supported by anthropic_messages", + ) elif ( path == "structured_output.format" and req.structured_output.format != "json_schema" @@ -2203,7 +2206,8 @@ def validate_request( else: audit.rejected( path, - "PR 3 exposes no raw Anthropic request passthrough", + "anthropic_messages does not support raw Anthropic request " + "passthrough", ) def prepare(self, req: Request, audit: RequestAudit) -> PreparedRequest: diff --git a/sieval/core/models/dialects/openai_chat.py b/sieval/core/models/dialects/openai_chat.py index cb182a14..3f77a643 100644 --- a/sieval/core/models/dialects/openai_chat.py +++ b/sieval/core/models/dialects/openai_chat.py @@ -130,7 +130,7 @@ def _validate_multimodal_config(options: object) -> None: ) ), "hosted_tools": Unsupported( - "PR 1 does not bind hosted tools on Chat Completions" + "openai_chat does not support hosted tools on Chat Completions" ), "structured_output": Supported( DialectCapabilityBinding( @@ -766,7 +766,7 @@ def validate_request( "openai_chat supports only summary='none' as a documented no-op", ) elif path == "tools.hosted": - audit.rejected(path, "hosted tools are not active in PR 1") + audit.rejected(path, "hosted tools are not supported by openai_chat") elif ( path == "input.modality.image.media_type" and isinstance(req.input, ChatInput) diff --git a/sieval/core/models/sglang_gen_model.py b/sieval/core/models/sglang_gen_model.py index 1d308e74..c2a044e1 100644 --- a/sieval/core/models/sglang_gen_model.py +++ b/sieval/core/models/sglang_gen_model.py @@ -232,14 +232,14 @@ async def arun(self, req: Request) -> Response: def with_dialect(self, dialect_id: str, runtime_plan: RuntimeBindingPlan) -> Model: del dialect_id, runtime_plan raise RuntimeError( - "sglang_legacy cannot rebind before the sglang_native PR-5 binder" + "sglang_legacy cannot rebind because sglang_native has no executable binder" ) def with_provenance_plan(self, provenance_plan: RuntimeBindingPlan) -> Self: del provenance_plan raise RuntimeError( - "sglang_legacy has no runtime plan for provenance before the " - "sglang_native PR-5 binder" + "sglang_legacy has no runtime plan for provenance because " + "sglang_native has no executable binder" ) def _legacy_lifecycle_owner(self) -> "SglangGenModel": diff --git a/tests/unit/cli/test_validation.py b/tests/unit/cli/test_validation.py index 674f762e..270d600e 100644 --- a/tests/unit/cli/test_validation.py +++ b/tests/unit/cli/test_validation.py @@ -531,8 +531,8 @@ def test_sglang_legacy_bypass_rejects_capability_declarations( @pytest.mark.parametrize( ("dialect", "message"), [ - ("sglang_native", "legacy bypass"), - ("vllm_native", "explicitly deferred"), + ("sglang_native", "no executable binder"), + ("vllm_native", "no executable binder"), ], ) def test_registered_but_inactive_dialect_is_rejected(self, dialect, message): diff --git a/tests/unit/core/models/dialects/test_anthropic_messages.py b/tests/unit/core/models/dialects/test_anthropic_messages.py index 9f0fe3fc..a39f7859 100644 --- a/tests/unit/core/models/dialects/test_anthropic_messages.py +++ b/tests/unit/core/models/dialects/test_anthropic_messages.py @@ -194,6 +194,11 @@ def test_decision_row_is_complete_and_logprobs_are_explicitly_absent(self) -> No assert isinstance(CAPABILITY_DECISIONS["sampled_logprobs"], Unsupported) assert isinstance(CAPABILITY_DECISIONS["top_logprobs"], Unsupported) assert "does not return" in CAPABILITY_DECISIONS["sampled_logprobs"].reason + hosted = CAPABILITY_DECISIONS["hosted_tools"] + assert isinstance(hosted, Unsupported) + assert hosted.reason == ( + "anthropic_messages does not support Anthropic server-hosted tools" + ) def test_supported_capabilities_are_first_class_registry_outcomes(self) -> None: spec = DIALECT_SPECS["anthropic_messages"] @@ -1379,7 +1384,7 @@ class TestPreIOGuards: ), ( _request(tools=ToolParams(hosted=(HostedToolSpec("web_search"),))), - "unavailable capability|not active", + "unavailable capability|not supported by anthropic_messages", ), ( _request( @@ -1430,7 +1435,7 @@ class TestPreIOGuards: "anthropic_messages", {"service_tier": "auto"} ) ), - "no raw Anthropic request passthrough", + "does not support raw Anthropic request passthrough", ), ( _request( diff --git a/tests/unit/core/models/dialects/test_openai_chat.py b/tests/unit/core/models/dialects/test_openai_chat.py index 8a38c32c..a21fc1fd 100644 --- a/tests/unit/core/models/dialects/test_openai_chat.py +++ b/tests/unit/core/models/dialects/test_openai_chat.py @@ -13,7 +13,12 @@ import pytest import sieval.core.models.dialects.openai_chat as openai_chat_module -from sieval.core.models.capabilities import Capability, ReasoningOptions, Supported +from sieval.core.models.capabilities import ( + Capability, + ReasoningOptions, + Supported, + Unsupported, +) from sieval.core.models.dialect import ( DialectError, OutputContractError, @@ -489,6 +494,14 @@ def test_legacy_capability_view_matches_supported_reasoning_controls(self) -> No assert Capability.Reasoning in dialect.capabilities assert Capability.ReasoningEffort in dialect.capabilities + def test_hosted_tool_rejection_uses_stable_dialect_wording(self) -> None: + decision = CAPABILITY_DECISIONS["hosted_tools"] + + assert isinstance(decision, Unsupported) + assert decision.reason == ( + "openai_chat does not support hosted tools on Chat Completions" + ) + def test_reasoning_config_rejects_non_noop_summary(self) -> None: decision = CAPABILITY_DECISIONS["reasoning"] assert isinstance(decision, Supported) diff --git a/tests/unit/core/models/test_dialect_registry.py b/tests/unit/core/models/test_dialect_registry.py index fd25a2dd..6711bdd4 100644 --- a/tests/unit/core/models/test_dialect_registry.py +++ b/tests/unit/core/models/test_dialect_registry.py @@ -264,7 +264,7 @@ def test_capability_decisions_are_available_only_for_active_dialects(self) -> No assert set(capability_decisions_for("anthropic_messages")) == set( CAPABILITY_KEYS ) - with pytest.raises(DialectNotImplemented, match="later #25 adapter"): + with pytest.raises(DialectNotImplemented, match="no executable binder"): capability_decisions_for("google_genai") @@ -272,8 +272,8 @@ class TestDialectBinders: @pytest.mark.parametrize( ("dialect_id", "message"), [ - ("vllm_native", "explicitly deferred"), - ("sglang_native", "legacy bypass"), + ("vllm_native", "no executable binder"), + ("sglang_native", "sglang_legacy.*bypass"), ], ) def test_reserved_dialects_fail_with_named_error(