Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 25 additions & 3 deletions raven/providers/litellm_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,9 @@ def _normalize_tool_call_id(tool_call_id: Any) -> Any:

@staticmethod
def _sanitize_messages(
messages: list[dict[str, Any]], extra_keys: frozenset[str] = frozenset()
messages: list[dict[str, Any]],
extra_keys: frozenset[str] = frozenset(),
ensure_tool_reasoning: bool = False,
) -> list[dict[str, Any]]:
"""Strip non-standard keys and ensure assistant messages have a content key."""
allowed = _ALLOWED_MSG_KEYS | extra_keys
Expand Down Expand Up @@ -396,8 +398,18 @@ def map_id(value: Any) -> Any:

if "tool_call_id" in clean and clean["tool_call_id"]:
clean["tool_call_id"] = map_id(clean["tool_call_id"])
if ensure_tool_reasoning and clean.get("role") == "assistant" and clean.get("tool_calls"):
clean.setdefault("reasoning_content", "")
return sanitized

def _requires_tool_reasoning_replay(self, original_model: str, resolved_model: str) -> bool:
"""Return whether the wire model requires reasoning keys on tool continuations."""
for model in (original_model, resolved_model):
upstream_model = self._strip_gateway_prefix(model).lower()
if upstream_model.startswith("deepseek/deepseek-v4-"):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-blocking scope note: this matches direct DeepSeek and OpenRouter, and cannot match AiHubMix -- which is in Raven's own onboarding catalog and fronts DeepSeek's official API.

AiHubMix's spec sets strip_model_prefix=True and via_driver="openai" (raven/providers/registry.py:254), so a stored aihubmix/deepseek-v4-pro resolves to openai/deepseek-v4-pro, and _strip_gateway_prefix -- which strips the gateway's model_prefix, i.e. openai/ -- leaves deepseek-v4-pro. Neither argument starts with deepseek/, so the replay never fires. The same applies to an OpenAI-compatible provider pointed at api.deepseek.com, where the id is stored under the custom provider's prefix.

That is not a regression -- those routes are broken today too -- and I have not measured whether AiHubMix relays DeepSeek's 400 verbatim, so I am not asking you to widen the match blind. Just worth knowing the issue stays open for those users. If you want the cheap version, matching "deepseek-v4-" in upstream_model instead of startswith("deepseek/deepseek-v4-") would cover them, at the cost of also firing for self-hosted deepseek-ai/DeepSeek-V4-* on SiliconFlow etc., where the extra key is inert anyway.

return True
return False

async def chat(
self,
messages: list[dict[str, Any]],
Expand All @@ -424,6 +436,7 @@ async def chat(
original_model = model or self.default_model
model = self._resolve_model(original_model)
extra_msg_keys = self._extra_msg_keys(original_model, model)
ensure_tool_reasoning = self._requires_tool_reasoning_replay(original_model, model)

if self._supports_cache_control(original_model):
if not self.disable_auto_cache_control:
Expand All @@ -440,7 +453,11 @@ async def chat(

kwargs: dict[str, Any] = {
"model": model,
"messages": self._sanitize_messages(self._sanitize_empty_content(messages), extra_keys=extra_msg_keys),
"messages": self._sanitize_messages(
self._sanitize_empty_content(messages),
extra_keys=extra_msg_keys,
ensure_tool_reasoning=ensure_tool_reasoning,
),
"temperature": temperature,
# Per-read httpx cap forwarded to the underlying client. This alone
# cannot bound a backend that trickles bytes forever (the read timer
Expand Down Expand Up @@ -529,6 +546,7 @@ async def chat_stream(
original_model = model or self.default_model
model = self._resolve_model(original_model)
extra_msg_keys = self._extra_msg_keys(original_model, model)
ensure_tool_reasoning = self._requires_tool_reasoning_replay(original_model, model)

if self._supports_cache_control(original_model):
if not self.disable_auto_cache_control:
Expand All @@ -538,7 +556,11 @@ async def chat_stream(

kwargs: dict[str, Any] = {
"model": model,
"messages": self._sanitize_messages(self._sanitize_empty_content(messages), extra_keys=extra_msg_keys),
"messages": self._sanitize_messages(
self._sanitize_empty_content(messages),
extra_keys=extra_msg_keys,
ensure_tool_reasoning=ensure_tool_reasoning,
),
"temperature": temperature,
"stream": True,
# OpenAI-compatible providers only emit the trailing usage chunk
Expand Down
105 changes: 105 additions & 0 deletions tests/test_litellm_provider_stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
- chat() and chat_stream() both forward the provider's api_key to acompletion
as an explicit kwarg, rather than relying on it having been exported to the
environment
- DeepSeek V4 tool continuations replay an empty reasoning_content field

Mocks patch `raven.providers.litellm_provider.acompletion` because the
provider module imports `from litellm import acompletion` at top level, so
Expand Down Expand Up @@ -270,6 +271,110 @@ async def fake_acompletion(**kwargs: Any):
assert captured["api_key"] == "k-main"


@pytest.mark.parametrize(
("model", "provider_name"),
[
("deepseek/deepseek-v4-flash", "deepseek"),
("deepseek/deepseek-v4-pro", "deepseek"),
("openrouter/deepseek/deepseek-v4-pro", "openrouter"),
],
)
@pytest.mark.parametrize("declare_tools", [True, False], ids=["declared", "omitted"])
@pytest.mark.asyncio
async def test_chat_replays_empty_reasoning_for_deepseek_v4_tool_calls(
monkeypatch: pytest.MonkeyPatch,
model: str,
provider_name: str,
declare_tools: bool,
) -> None:
captured: dict[str, Any] = {}

async def fake_acompletion(**kwargs: Any):
captured.update(kwargs)
return _FakeResponse("ok")

monkeypatch.setattr("raven.providers.litellm_provider.acompletion", fake_acompletion)

messages = [
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "probe", "arguments": "{}"},
}
],
},
{"role": "tool", "content": "done", "tool_call_id": "call_1"},
]
tools = [{"type": "function", "function": {"name": "probe", "parameters": {}}}] if declare_tools else None
provider = LiteLLMProvider(api_key="test-key", provider_name=provider_name, default_model=model)

await provider.chat(messages=messages, tools=tools)

assert captured["messages"][0]["reasoning_content"] == ""


@pytest.mark.parametrize("declare_tools", [True, False], ids=["declared", "omitted"])
@pytest.mark.asyncio
async def test_chat_stream_replays_empty_reasoning_for_deepseek_v4_tool_calls(
monkeypatch: pytest.MonkeyPatch,
declare_tools: bool,
) -> None:
captured: dict[str, Any] = {}

async def fake_acompletion(**kwargs: Any):
captured.update(kwargs)
return _fake_stream([_chunk("ok")])

monkeypatch.setattr("raven.providers.litellm_provider.acompletion", fake_acompletion)

messages = [{"role": "assistant", "content": "", "tool_calls": [{"id": "call_1"}]}]
tools = [{"type": "function", "function": {"name": "probe", "parameters": {}}}] if declare_tools else None
provider = LiteLLMProvider(
api_key="test-key",
provider_name="deepseek",
default_model="deepseek/deepseek-v4-flash",
)

async for _ in provider.chat_stream(messages=messages, tools=tools):
pass

assert captured["messages"][0]["reasoning_content"] == ""


@pytest.mark.parametrize(
("model", "provider_name"),
[
("deepseek/deepseek-chat", "deepseek"),
("openai/gpt-4o", "openai"),
],
)
@pytest.mark.asyncio
async def test_chat_leaves_non_thinking_tool_call_messages_unchanged(
monkeypatch: pytest.MonkeyPatch,
model: str,
provider_name: str,
) -> None:
captured: dict[str, Any] = {}

async def fake_acompletion(**kwargs: Any):
captured.update(kwargs)
return _FakeResponse("ok")

monkeypatch.setattr("raven.providers.litellm_provider.acompletion", fake_acompletion)

messages = [{"role": "assistant", "content": "", "tool_calls": [{"id": "call_1"}]}]
tools = [{"type": "function", "function": {"name": "probe", "parameters": {}}}]
provider = LiteLLMProvider(api_key="test-key", provider_name=provider_name, default_model=model)

await provider.chat(messages=messages, tools=tools)

assert "reasoning_content" not in captured["messages"][0]


# --------- generation settings reach the request body (regression) ----------
#
# chat_stream used to declare literal defaults (max_tokens=4096,
Expand Down
Loading