Skip to content

Add raw multimodal content parts support #133

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
eligotts wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Add raw multimodal content parts support #133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 59 additions & 12 deletions renderers/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ async def generate(
priority: int | None = None,
extra_headers: dict[str, str] | None = None,
max_prompt_len: int | None = None,
raw_multimodal: bool = False,
) -> dict[str, Any]:
"""Tokenize messages, call vLLM /inference/v1/generate, parse the response.

Expand All @@ -245,6 +246,9 @@ async def generate(
sidecar, then serializes it to vLLM's ``features`` schema (mm_hashes,
mm_placeholders, kwargs_data) before POSTing. The serializer imports
``vllm.*`` lazily so text-only consumers never pay for the import.
With ``raw_multimodal=True``, rendering skips image processing and the
request carries raw ``content_parts`` instead; vLLM must return the
expanded prompt as ``prompt_token_ids``.

``max_prompt_len`` controls the pre-flight overflow check. When the
rendered prompt is strictly longer than the cap, the request is never
Expand All @@ -255,10 +259,14 @@ async def generate(
cache a ``None`` cap and the pre-flight silently disables. Engine 4xx
that still slip through propagate raw — converting them into a domain
error is the calling client's job (its error shape is engine-specific).
Raw multimodal calls skip this pre-flight because only vLLM knows the
expanded prompt length.

Returns a dict with: request_id, prompt_ids, completion_ids,
completion_logprobs, content, reasoning_content, tool_calls,
finish_reason, routed_experts, multi_modal_data, prompt_attribution.
Returns a dict with: request_id, prompt_ids, renderer_prompt_ids,
completion_ids, completion_logprobs, content, reasoning_content,
tool_calls, finish_reason, routed_experts, multi_modal_data,
prompt_attribution. ``renderer_prompt_ids`` is the unexpanded logical
prompt on raw multimodal calls and ``None`` otherwise.

``prompt_attribution`` is the renderer's :class:`RenderedTokens` for
the prompt — either the one this call computed via
Expand Down Expand Up @@ -289,7 +297,15 @@ def _prepare():
multi_modal_data,
prompt_attribution,
)
rendered = renderer.render(messages, tools=tools, add_generation_prompt=True)
render_kwargs: dict[str, Any] = {}
if raw_multimodal:
render_kwargs["process_multimodal"] = False
rendered = renderer.render(
messages,
tools=tools,
add_generation_prompt=True,
**render_kwargs,
)
return (
rendered.token_ids,
renderer.get_stop_token_ids(),
Expand All @@ -301,12 +317,13 @@ def _prepare():
renderer, _prepare
)

if max_prompt_len is None:
max_prompt_len = await _resolve_max_prompt_len(client, model)
if max_prompt_len is not None and len(prompt_ids) > max_prompt_len:
raise OverlongPromptError(
prompt_len=len(prompt_ids), max_prompt_len=max_prompt_len
)
if not raw_multimodal:
if max_prompt_len is None:
max_prompt_len = await _resolve_max_prompt_len(client, model)
if max_prompt_len is not None and len(prompt_ids) > max_prompt_len:
raise OverlongPromptError(
prompt_len=len(prompt_ids), max_prompt_len=max_prompt_len
)

sp: dict[str, Any] = dict(sampling_params or {})
sp["stop_token_ids"] = stop_token_ids
Expand All @@ -318,11 +335,14 @@ def _prepare():
"token_ids": prompt_ids,
"sampling_params": sp,
}
content_parts = _content_parts(messages) if raw_multimodal else None
features = (
_build_mm_features(renderer, mm_data)
if mm_data and not mm_data.is_empty()
if not raw_multimodal and mm_data and not mm_data.is_empty()
else None
)
if content_parts:
body["content_parts"] = content_parts
if features is not None:
body["features"] = features
if cache_salt is not None:
Expand Down Expand Up @@ -352,6 +372,11 @@ def _prepare():

choice = (data.get("choices") or [{}])[0]
completion_ids = choice.get("token_ids") or []
effective_prompt_ids = data.get("prompt_token_ids")
if content_parts and not isinstance(effective_prompt_ids, list):
raise MalformedGenerateResponseError(
"Engine response must include prompt_token_ids for raw multimodal input."
)

completion_logprobs = _parse_completion_logprobs(choice, completion_ids)

Expand Down Expand Up @@ -379,7 +404,8 @@ def _prepare():

return {
"request_id": data.get("request_id") or "",
"prompt_ids": list(prompt_ids),
"prompt_ids": list(effective_prompt_ids or prompt_ids),
"renderer_prompt_ids": list(prompt_ids) if content_parts else None,
"completion_ids": list(completion_ids),
"completion_logprobs": completion_logprobs,
"content": parsed.content,
Expand All @@ -403,6 +429,27 @@ def _prepare():
}


def _content_parts(messages: list[Message]) -> list[dict[str, Any]]:
"""Flatten raw media in prompt order for vLLM's token generate endpoint."""
parts: list[dict[str, Any]] = []
for message in messages:
content = message.get("content")
if not isinstance(content, list):
continue
for part in content:
if not isinstance(part, Mapping):
continue
part_type = part.get("type")
if part_type not in ("image_url", "audio_url", "video_url"):
continue
source = part.get(part_type)
url = source.get("url") if isinstance(source, Mapping) else part.get("url")
if not isinstance(url, str) or not url:
raise ValueError(f"{part_type} content part is missing a URL")
parts.append({"type": part_type, "url": url})
return parts


def _build_mm_features(
renderer: Renderer | RendererPool,
mm_data: MultiModalData,
Expand Down
26 changes: 21 additions & 5 deletions renderers/kimi_k25.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -590,6 +590,8 @@ class KimiK25Renderer:
The tokenizer should be ``moonshotai/Kimi-K2-Instruct`` (same as K2).
"""

supports_raw_multimodal = True

def __init__(
self,
tokenizer: PreTrainedTokenizer,
Expand Down Expand Up @@ -733,6 +735,7 @@ def render(
*,
tools: list[ToolSpec] | None = None,
add_generation_prompt: bool = False,
process_multimodal: bool = True,
) -> RenderedTokens:
"""Render messages to tokens, matching the K2.5 chat template.

Expand Down Expand Up @@ -820,7 +823,10 @@ def emit_image(
``<|media_content|>``, ``<|media_end|>``, the trailing
``\\n``) are template-injected scaffold.
"""
_, out, _num_patches, h = self._process_image(part)
if process_multimodal:
_, out, _num_patches, h = self._process_image(part)
else:
out = h = None
emit_special(
self._media_begin, msg_idx, is_sampled=is_sampled, is_content=False
)
Expand All @@ -839,6 +845,9 @@ def emit_image(
self._media_end, msg_idx, is_sampled=is_sampled, is_content=False
)
emit_text("\n", msg_idx, is_sampled=is_sampled, is_content=False)
if not process_multimodal:
return
assert out is not None and h is not None
mm_hashes.setdefault("image", []).append(h)
mm_placeholders.setdefault("image", []).append(
PlaceholderRange(offset=offset, length=1)
Expand Down Expand Up @@ -1026,6 +1035,7 @@ def bridge_to_next_turn(
*,
tools: list[ToolSpec] | None = None,
previous_multi_modal_data: MultiModalData | None = None,
process_multimodal: bool = True,
) -> "RenderedTokens | None":
if (
not previous_prompt_ids
Expand Down Expand Up @@ -1114,7 +1124,10 @@ def emit_image(
is_sampled: bool = False,
is_content: bool = False,
) -> None:
_, out, _num_patches, h = self._process_image(part)
if process_multimodal:
_, out, _num_patches, h = self._process_image(part)
else:
out = h = None
emit_special(self._media_begin, msg_idx)
emit_text("image", msg_idx)
emit_special(self._media_content, msg_idx)
Expand All @@ -1124,6 +1137,9 @@ def emit_image(
emit_special(self._media_pad, msg_idx, is_content=is_content)
emit_special(self._media_end, msg_idx)
emit_text("\n", msg_idx)
if not process_multimodal:
return
assert out is not None and h is not None
new_hashes.setdefault("image", []).append(h)
new_placeholders.setdefault("image", []).append(
PlaceholderRange(offset=offset, length=1)
Expand Down Expand Up @@ -1187,17 +1203,17 @@ def emit_image(
# below never mutates the caller's previous_multi_modal_data.
merged_hashes: dict[str, list[str]] = (
{k: list(v) for k, v in previous_multi_modal_data.mm_hashes.items()}
if previous_multi_modal_data
if process_multimodal and previous_multi_modal_data
else {}
)
merged_placeholders: dict[str, list[PlaceholderRange]] = (
{k: list(v) for k, v in previous_multi_modal_data.mm_placeholders.items()}
if previous_multi_modal_data
if process_multimodal and previous_multi_modal_data
else {}
)
merged_items: dict[str, list[dict[str, Any]]] = (
{k: list(v) for k, v in previous_multi_modal_data.mm_items.items()}
if previous_multi_modal_data
if process_multimodal and previous_multi_modal_data
else {}
)
for modality, vals in new_hashes.items():
Expand Down
70 changes: 44 additions & 26 deletions renderers/qwen35.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ def _default_enable_thinking(tokenizer) -> bool:
class Qwen35Renderer:
"""Deterministic message → token renderer for Qwen3.5 models."""

supports_raw_multimodal = True
_config_cls: type = Qwen35RendererConfig

def __init__(
Expand Down Expand Up @@ -352,6 +353,7 @@ def render(
*,
tools: list[ToolSpec] | None = None,
add_generation_prompt: bool = False,
process_multimodal: bool = True,
) -> RenderedTokens:
if not messages:
raise ValueError("No messages provided.")
Expand Down Expand Up @@ -423,7 +425,11 @@ def emit_image(part: dict[str, Any], msg_idx: int) -> None:
# image data, so they ARE body content (is_content=True);
# the surrounding ``<|vision_start|>`` / ``<|vision_end|>``
# specials are template scaffold.
_, out, n, h = self._process_image(part)
if process_multimodal:
_, out, n, h = self._process_image(part)
else:
out = h = None
n = 1
vision_counts["image"] += 1
if self.config.add_vision_id:
emit_text(
Expand All @@ -441,16 +447,18 @@ def emit_image(part: dict[str, Any], msg_idx: int) -> None:
self._image_pad, msg_idx, is_sampled=False, is_content=True
)
emit_special(self._vision_end, msg_idx, is_sampled=False, is_content=False)
mm_hashes.setdefault("image", []).append(h)
mm_placeholders.setdefault("image", []).append(
PlaceholderRange(offset=offset, length=n)
)
mm_items.setdefault("image", []).append(
{
"pixel_values": out["pixel_values"],
"image_grid_thw": out["image_grid_thw"],
}
)
if process_multimodal:
assert out is not None and h is not None
mm_hashes.setdefault("image", []).append(h)
mm_placeholders.setdefault("image", []).append(
PlaceholderRange(offset=offset, length=n)
)
mm_items.setdefault("image", []).append(
{
"pixel_values": out["pixel_values"],
"image_grid_thw": out["image_grid_thw"],
}
)

def emit_user_with_media(content_list: list[Any], msg_idx: int) -> None:
"""Emit a user message whose content list contains image parts.
Expand Down Expand Up @@ -684,6 +692,7 @@ def bridge_to_next_turn(
*,
tools: list[ToolSpec] | None = None,
previous_multi_modal_data: MultiModalData | None = None,
process_multimodal: bool = True,
) -> "RenderedTokens | None":
if (
not previous_prompt_ids
Expand Down Expand Up @@ -720,6 +729,7 @@ def bridge_to_next_turn(
# scratch correctly.
if (
self.config.add_vision_id
and process_multimodal
and previous_multi_modal_data is None
and self._vision_start in previous_ids
):
Expand Down Expand Up @@ -750,9 +760,11 @@ def bridge_to_next_turn(
# ``add_vision_id`` parity across turns.
prev_image_count = 0
prev_video_count = 0
if previous_multi_modal_data is not None:
if process_multimodal and previous_multi_modal_data is not None:
prev_image_count = len(previous_multi_modal_data.mm_items.get("image", []))
prev_video_count = len(previous_multi_modal_data.mm_items.get("video", []))
elif not process_multimodal:
prev_image_count = previous_ids.count(self._image_pad)
vision_counts = {"image": prev_image_count, "video": prev_video_count}

def emit_special(
Expand Down Expand Up @@ -795,7 +807,11 @@ def emit_text_segments(
content_mask.append(is_content)

def emit_image(part: dict[str, Any], msg_idx: int = -1) -> None:
_, out, n, h = self._process_image(part)
if process_multimodal:
_, out, n, h = self._process_image(part)
else:
out = h = None
n = 1
vision_counts["image"] += 1
if self.config.add_vision_id:
emit_text(f"Picture {vision_counts['image']}: ", msg_idx)
Expand All @@ -804,16 +820,18 @@ def emit_image(part: dict[str, Any], msg_idx: int = -1) -> None:
for _ in range(n):
emit_special(self._image_pad, msg_idx, is_content=True)
emit_special(self._vision_end, msg_idx)
new_hashes.setdefault("image", []).append(h)
new_placeholders.setdefault("image", []).append(
PlaceholderRange(offset=offset, length=n)
)
new_items.setdefault("image", []).append(
{
"pixel_values": out["pixel_values"],
"image_grid_thw": out["image_grid_thw"],
}
)
if process_multimodal:
assert out is not None and h is not None
new_hashes.setdefault("image", []).append(h)
new_placeholders.setdefault("image", []).append(
PlaceholderRange(offset=offset, length=n)
)
new_items.setdefault("image", []).append(
{
"pixel_values": out["pixel_values"],
"image_grid_thw": out["image_grid_thw"],
}
)

def emit_user_with_media(content_list: list[Any], msg_idx: int) -> None:
emit_special(self._im_start, msg_idx)
Expand Down Expand Up @@ -904,17 +922,17 @@ def flush_buf() -> None:
# below never mutates the caller's previous_multi_modal_data.
merged_hashes: dict[str, list[str]] = (
{k: list(v) for k, v in previous_multi_modal_data.mm_hashes.items()}
if previous_multi_modal_data
if process_multimodal and previous_multi_modal_data
else {}
)
merged_placeholders: dict[str, list[PlaceholderRange]] = (
{k: list(v) for k, v in previous_multi_modal_data.mm_placeholders.items()}
if previous_multi_modal_data
if process_multimodal and previous_multi_modal_data
else {}
)
merged_items: dict[str, list[dict[str, Any]]] = (
{k: list(v) for k, v in previous_multi_modal_data.mm_items.items()}
if previous_multi_modal_data
if process_multimodal and previous_multi_modal_data
else {}
)
for modality, vals in new_hashes.items():
Expand Down
Loading
Loading