Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions sieval/cli/leaderboard/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -832,7 +832,7 @@ def reconcile(
CheckStage.REQUEST,
verifier,
requirement.reason
or "PR-1 compatibility requires response-time verification",
or "legacy compatibility requires response-time verification",
)
return outcomes

Expand Down Expand Up @@ -3301,7 +3301,7 @@ def replace_inline(
raise ValueError(
f"Model binding '{binding_id}' uses the temporary "
"sglang_legacy bypass and cannot declare canonical "
"capabilities before the sglang_native PR-5 binder"
"capabilities while sglang_native has no executable binder"
)
legacy_bypass.add(binding_id)
continue
Expand Down
2 changes: 1 addition & 1 deletion sieval/cli/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ def _validate_capabilities(cfg: dict, result: ValidationResult) -> None:
result.errors.append(
f"Model '{name}': the temporary sglang_legacy bypass "
"cannot declare canonical capabilities before the "
"sglang_native PR-5 binder"
"sglang_native executable binder is available"
)
continue

Expand Down
8 changes: 4 additions & 4 deletions sieval/core/models/dialect_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -546,12 +546,12 @@ def bind_dialect(
def _not_implemented_message(dialect_id: str) -> str:
if dialect_id == "sglang_native":
return (
"dialect 'sglang_native' has no PR-1 binder; the one-cycle SGLang "
"compatibility path remains an explicit legacy bypass"
"dialect 'sglang_native' has no executable binder; use the explicit "
"'sglang_legacy' compatibility bypass"
)
if dialect_id == "vllm_native":
return "dialect 'vllm_native' is reserved and explicitly deferred after #25"
return f"dialect {dialect_id!r} is reserved for a later #25 adapter PR"
return "dialect 'vllm_native' is reserved and has no executable binder"
return f"dialect {dialect_id!r} is reserved and has no executable binder"


def dialect_registry_to_json() -> dict[str, JSONValue]:
Expand Down
10 changes: 7 additions & 3 deletions sieval/core/models/dialects/anthropic_messages.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ def _validate_multimodal_config(options: object) -> None:
)
),
"hosted_tools": Unsupported(
"PR 3 does not bind Anthropic server-hosted tools"
"anthropic_messages does not support Anthropic server-hosted tools"
),
"structured_output": Supported(
DialectCapabilityBinding(
Expand Down Expand Up @@ -2167,7 +2167,10 @@ def validate_request(
"or 'none'",
)
elif path == "tools.hosted":
audit.rejected(path, "Anthropic hosted tools are not active in PR 3")
audit.rejected(
path,
"Anthropic hosted tools are not supported by anthropic_messages",
)
elif (
path == "structured_output.format"
and req.structured_output.format != "json_schema"
Expand Down Expand Up @@ -2203,7 +2206,8 @@ def validate_request(
else:
audit.rejected(
path,
"PR 3 exposes no raw Anthropic request passthrough",
"anthropic_messages does not support raw Anthropic request "
"passthrough",
)

def prepare(self, req: Request, audit: RequestAudit) -> PreparedRequest:
Expand Down
4 changes: 2 additions & 2 deletions sieval/core/models/dialects/openai_chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ def _validate_multimodal_config(options: object) -> None:
)
),
"hosted_tools": Unsupported(
"PR 1 does not bind hosted tools on Chat Completions"
"openai_chat does not support hosted tools on Chat Completions"
),
"structured_output": Supported(
DialectCapabilityBinding(
Expand Down Expand Up @@ -766,7 +766,7 @@ def validate_request(
"openai_chat supports only summary='none' as a documented no-op",
)
elif path == "tools.hosted":
audit.rejected(path, "hosted tools are not active in PR 1")
audit.rejected(path, "hosted tools are not supported by openai_chat")
elif (
path == "input.modality.image.media_type"
and isinstance(req.input, ChatInput)
Expand Down
6 changes: 3 additions & 3 deletions sieval/core/models/sglang_gen_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,14 +232,14 @@ async def arun(self, req: Request) -> Response:
def with_dialect(self, dialect_id: str, runtime_plan: RuntimeBindingPlan) -> Model:
del dialect_id, runtime_plan
raise RuntimeError(
"sglang_legacy cannot rebind before the sglang_native PR-5 binder"
"sglang_legacy cannot rebind because sglang_native has no executable binder"
)

def with_provenance_plan(self, provenance_plan: RuntimeBindingPlan) -> Self:
del provenance_plan
raise RuntimeError(
"sglang_legacy has no runtime plan for provenance before the "
"sglang_native PR-5 binder"
"sglang_legacy has no runtime plan for provenance because "
"sglang_native has no executable binder"
)

def _legacy_lifecycle_owner(self) -> "SglangGenModel":
Expand Down
4 changes: 2 additions & 2 deletions tests/unit/cli/test_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -531,8 +531,8 @@ def test_sglang_legacy_bypass_rejects_capability_declarations(
@pytest.mark.parametrize(
("dialect", "message"),
[
("sglang_native", "legacy bypass"),
("vllm_native", "explicitly deferred"),
("sglang_native", "no executable binder"),
("vllm_native", "no executable binder"),
],
)
def test_registered_but_inactive_dialect_is_rejected(self, dialect, message):
Expand Down
9 changes: 7 additions & 2 deletions tests/unit/core/models/dialects/test_anthropic_messages.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,11 @@ def test_decision_row_is_complete_and_logprobs_are_explicitly_absent(self) -> No
assert isinstance(CAPABILITY_DECISIONS["sampled_logprobs"], Unsupported)
assert isinstance(CAPABILITY_DECISIONS["top_logprobs"], Unsupported)
assert "does not return" in CAPABILITY_DECISIONS["sampled_logprobs"].reason
hosted = CAPABILITY_DECISIONS["hosted_tools"]
assert isinstance(hosted, Unsupported)
assert hosted.reason == (
"anthropic_messages does not support Anthropic server-hosted tools"
)

def test_supported_capabilities_are_first_class_registry_outcomes(self) -> None:
spec = DIALECT_SPECS["anthropic_messages"]
Expand Down Expand Up @@ -1379,7 +1384,7 @@ class TestPreIOGuards:
),
(
_request(tools=ToolParams(hosted=(HostedToolSpec("web_search"),))),
"unavailable capability|not active",
"unavailable capability|not supported by anthropic_messages",
),
(
_request(
Expand Down Expand Up @@ -1430,7 +1435,7 @@ class TestPreIOGuards:
"anthropic_messages", {"service_tier": "auto"}
)
),
"no raw Anthropic request passthrough",
"does not support raw Anthropic request passthrough",
),
(
_request(
Expand Down
15 changes: 14 additions & 1 deletion tests/unit/core/models/dialects/test_openai_chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,12 @@
import pytest

import sieval.core.models.dialects.openai_chat as openai_chat_module
from sieval.core.models.capabilities import Capability, ReasoningOptions, Supported
from sieval.core.models.capabilities import (
Capability,
ReasoningOptions,
Supported,
Unsupported,
)
from sieval.core.models.dialect import (
DialectError,
OutputContractError,
Expand Down Expand Up @@ -489,6 +494,14 @@ def test_legacy_capability_view_matches_supported_reasoning_controls(self) -> No
assert Capability.Reasoning in dialect.capabilities
assert Capability.ReasoningEffort in dialect.capabilities

def test_hosted_tool_rejection_uses_stable_dialect_wording(self) -> None:
decision = CAPABILITY_DECISIONS["hosted_tools"]

assert isinstance(decision, Unsupported)
assert decision.reason == (
"openai_chat does not support hosted tools on Chat Completions"
)

def test_reasoning_config_rejects_non_noop_summary(self) -> None:
decision = CAPABILITY_DECISIONS["reasoning"]
assert isinstance(decision, Supported)
Expand Down
6 changes: 3 additions & 3 deletions tests/unit/core/models/test_dialect_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,16 +264,16 @@ def test_capability_decisions_are_available_only_for_active_dialects(self) -> No
assert set(capability_decisions_for("anthropic_messages")) == set(
CAPABILITY_KEYS
)
with pytest.raises(DialectNotImplemented, match="later #25 adapter"):
with pytest.raises(DialectNotImplemented, match="no executable binder"):
capability_decisions_for("google_genai")


class TestDialectBinders:
@pytest.mark.parametrize(
("dialect_id", "message"),
[
("vllm_native", "explicitly deferred"),
("sglang_native", "legacy bypass"),
("vllm_native", "no executable binder"),
("sglang_native", "sglang_legacy.*bypass"),
],
)
def test_reserved_dialects_fail_with_named_error(
Expand Down
Loading