diff --git a/.gitattributes b/.gitattributes index e746fd25e..caa5d11aa 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,6 +2,7 @@ *.png filter=lfs diff=lfs merge=lfs -text assets/banner.png -filter -diff -merge -text assets/docker.png -filter -diff -merge -text +server/docs/images/vision-mmproj-meme-example.png -filter -diff -merge -text *.jpg filter=lfs diff=lfs merge=lfs -text *.jpeg filter=lfs diff=lfs merge=lfs -text *.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 0da29ea33..0acbd0c9f 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -203,6 +203,23 @@ endif() # Use only the ggml subtree of llama.cpp (skip libllama). add_subdirectory(deps/llama.cpp/ggml EXCLUDE_FROM_ALL) +# Optional native mmproj vision (mtmd + llama for tokenization/vocab). +option(DFLASH27B_MMPROJ "Build native mmproj vision support via mtmd" OFF) +if(DFLASH27B_MMPROJ) + set(LLAMA_BUILD_COMMON OFF CACHE BOOL "" FORCE) + set(LLAMA_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(LLAMA_BUILD_TOOLS OFF CACHE BOOL "" FORCE) + set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + set(LLAMA_BUILD_SERVER OFF CACHE BOOL "" FORCE) + add_subdirectory(deps/llama.cpp EXCLUDE_FROM_ALL) + # mtmd is added as a sibling of llama.cpp (not via tools/) so + # LLAMA_INSTALL_VERSION must be visible in the parent scope. + if(NOT DEFINED LLAMA_INSTALL_VERSION) + set(LLAMA_INSTALL_VERSION "0.0.0") + endif() + add_subdirectory(deps/llama.cpp/tools/mtmd EXCLUDE_FROM_ALL) +endif() + if(DFLASH27B_GPU_BACKEND STREQUAL "hip") # The vendored ggml HIP shim still uses a few CUDA spellings that are not # mapped in this upstream snapshot. Keep the compatibility layer in this @@ -238,6 +255,7 @@ set(DFLASH27B_SRC_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/src/gemma4 ${CMAKE_CURRENT_SOURCE_DIR}/src/deepseek4 ${CMAKE_CURRENT_SOURCE_DIR}/src/server + ${CMAKE_CURRENT_SOURCE_DIR}/src/vision ) add_library(dflash_common STATIC @@ -319,6 +337,7 @@ add_library(dflash_common STATIC src/qwen35/qwen35_backend.cpp src/qwen35/qwen35_tensor_parallel.cpp src/qwen35/qwen35_layer_split_adapter.cpp + src/qwen35/qwen35_layer_split_vision.cpp src/qwen35/qwen35_dflash_target.cpp src/qwen35/qwen35_layer_split_dflash_target.cpp src/qwen35/layer_split_daemon_loop.cpp @@ -344,6 +363,7 @@ add_library(dflash_common STATIC src/server/prefix_cache.cpp src/server/disk_prefix_cache.cpp src/server/freeze_history.cpp + src/vision/vision_input.cpp # ── Jinja chat-template engine (vendored under deps/llama.cpp/common/) ── # Used by render_chat_template_jinja() to support --chat-template-file # in dflash_server. Mirrors llama.cpp's common_chat_template plumbing. @@ -357,6 +377,9 @@ add_library(dflash_common STATIC deps/llama.cpp/common/jinja/caps.cpp deps/llama.cpp/common/unicode.cpp ) +if(DFLASH27B_MMPROJ) + target_sources(dflash_common PRIVATE src/vision/vision_encoder.cpp) +endif() # BSA (Block-Sparse Attention) backs the speculative-prefill drafter scoring # path. Default ON so prefill is fast out of the box. Turn OFF if you don't # run the spec-prefill stack or are building without CUDA BF16 WMMA support @@ -579,6 +602,16 @@ if(DFLASH27B_ENABLE_BSA) endif() endif() +if(DFLASH27B_MMPROJ) + target_compile_definitions(dflash_common PRIVATE DFLASH_HAVE_MMPROJ=1) + set(DFLASH27B_MMPROJ_INCLUDE_DIRS + ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/include + ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/tools/mtmd + ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/vendor) + target_include_directories(dflash_common PRIVATE ${DFLASH27B_MMPROJ_INCLUDE_DIRS}) + target_link_libraries(dflash_common PRIVATE llama mtmd) +endif() + target_include_directories(dflash_common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include @@ -1133,6 +1166,10 @@ if(DFLASH27B_TESTS) else() target_link_libraries(test_server_unit PRIVATE hip::host) endif() + if(DFLASH27B_MMPROJ) + target_compile_definitions(test_server_unit PRIVATE DFLASH_HAVE_MMPROJ=1) + target_include_directories(test_server_unit PRIVATE ${DFLASH27B_MMPROJ_INCLUDE_DIRS}) + endif() if(CMAKE_CROSSCOMPILING) # Runtime discovery cannot execute a target binary on the build # host. Register one aggregate test instead; CMake prepends the @@ -1312,6 +1349,10 @@ if(DFLASH27B_SERVER) DFLASH27B_BACKEND_CUDA=1 DFLASH27B_CUDA_MIN_SM=${_dflash_cuda_min_sm}) endif() + if(DFLASH27B_MMPROJ) + target_compile_definitions(dflash_server PRIVATE DFLASH_HAVE_MMPROJ=1) + target_include_directories(dflash_server PRIVATE ${DFLASH27B_MMPROJ_INCLUDE_DIRS}) + endif() if(NOT WIN32) target_link_libraries(dflash_server PRIVATE dflash_common ggml ${DFLASH27B_GGML_BACKEND_TARGET} pthread) else() diff --git a/server/docs/VISION.md b/server/docs/VISION.md new file mode 100644 index 000000000..c34ac8004 --- /dev/null +++ b/server/docs/VISION.md @@ -0,0 +1,108 @@ +# Native mmproj vision + +Optional build (`-DDFLASH27B_MMPROJ=ON`) wires llama.cpp **mtmd** so +`dflash_server` can load a GGUF multimodal projector alongside the text model +and accept OpenAI-style `image_url` content in chat completions. + +## Quick start + +**Need:** CUDA GPU, a Qwen3.5/3.6 GGUF + matching `mmproj-F16.gguf`, and a +**full** [`lucebox-ggml`](https://github.com/Luce-Org/lucebox-ggml) tree. +Hub only vendors the ggml subset — `tools/mtmd` is not in-tree — so a stock +configure with `-DDFLASH27B_MMPROJ=ON` will fail until mtmd sources are present. + +```bash +# 1) Checkout this PR +git fetch origin pull/571/head:pr-571 && git checkout pr-571 + +# 2) Supply full llama.cpp (mtmd) for the build +cd server/deps +mv llama.cpp llama.cpp.vendored-ggml-only +git clone --depth 1 -b luce-dflash https://github.com/Luce-Org/lucebox-ggml.git llama.cpp +# Keep hub-local ggml patches (e.g. rocmfp4) on top of the full tree +cp -a llama.cpp.vendored-ggml-only/ggml/. llama.cpp/ggml/ + +# 3) Build with vision +cd .. +cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CUDA_ARCHITECTURES= \ + -DDFLASH27B_MMPROJ=ON -DDFLASH27B_SERVER=ON +cmake --build build --target dflash_server -j"$(nproc)" + +# 4) Run (same flags you already use for Qwen35, plus mmproj) +./build/dflash_server \ + --model /path/to/Qwen….gguf \ + --mmproj /path/to/mmproj-F16.gguf \ + # …draft / layer-split / port as usual… + +# Container equivalent: +# DFLASH_MMPROJ=/path/to/mmproj-F16.gguf +# Optional: --no-mmproj-offload / DFLASH_MMPROJ_NO_OFFLOAD=1 +``` + +After the build you can restore the slim vendor so the tree stays pullable: + +```bash +cd server/deps +rm -rf llama.cpp +mv llama.cpp.vendored-ggml-only llama.cpp +``` + +### Smoke + +```bash +# Capability flag +curl -s localhost:8080/props | jq '.capabilities.vision_supported' +# expect: true + +# Multimodal chat (data URI only today) +IMG_B64=$(base64 -w0 /path/to/test.jpg) # macOS: base64 -i test.jpg +curl -s localhost:8080/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d "{ + \"model\": \"qwen\", + \"messages\": [{ + \"role\": \"user\", + \"content\": [ + {\"type\": \"text\", \"text\": \"What do you see?\"}, + {\"type\": \"image_url\", \"image_url\": { + \"url\": \"data:image/jpeg;base64,${IMG_B64}\" + }} + ] + }], + \"max_tokens\": 128 +}" +``` + +Also check a plain text turn still works (and still uses DFlash when a draft is +configured). Without `--mmproj`, image requests should 400 cleanly. + +## Runtime + +Multimodal turns run AR decode; text-only turns keep DFlash speculative decode +when configured. `/props` reports `capabilities.vision_supported: true` when +the projector is loaded. + +Supported on both monolithic Qwen35 and layer-split backends +(`supports_multimodal()` is delegated through `LayerSplitBackend`). + +## Example + +Chat completion with an attached meme image — the model reads the visual +layout and answers in natural language: + +![Native mmproj vision: meme interpretation](images/vision-mmproj-meme-example.png) + +Request shape (abbreviated): + +```json +{ + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "What do you think this image means?"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}} + ] + }] +} +``` diff --git a/server/docs/images/vision-mmproj-meme-example.png b/server/docs/images/vision-mmproj-meme-example.png new file mode 100644 index 000000000..4f9451ef5 Binary files /dev/null and b/server/docs/images/vision-mmproj-meme-example.png differ diff --git a/server/scripts/entrypoint.sh b/server/scripts/entrypoint.sh index 2602517f5..0759cb1e2 100755 --- a/server/scripts/entrypoint.sh +++ b/server/scripts/entrypoint.sh @@ -315,6 +315,8 @@ fi # the KV footprint. Only emitted to the server CLI when nonzero so # unset reproduces the server's own default unchanged. : "${DFLASH_FA_WINDOW:=0}" +# Native mmproj vision projector (Qwen3-VL etc.). When set, passed as --mmproj. +: "${DFLASH_MMPROJ:=}" # ── auto-detect target ───────────────────────────────────────────────────── # Target .gguf is typically 10-30 GB (Q4_K_M). Drafts are 1-2 GB (Q8_0 / Q4) @@ -499,6 +501,8 @@ CMD=("$DFLASH_SERVER_BIN" "$DFLASH_TARGET" [ -n "$DRAFT_ARG" ] && CMD+=(--draft "$DRAFT_ARG") [ -n "$DRAFT_ARG" ] && CMD+=(--ddtree --ddtree-budget "$DFLASH_BUDGET") +[ -n "$DFLASH_MMPROJ" ] && CMD+=(--mmproj "$DFLASH_MMPROJ") +[ "${DFLASH_MMPROJ_NO_OFFLOAD:-0}" = "1" ] && [ -n "$DFLASH_MMPROJ" ] && CMD+=(--no-mmproj-offload) [ -n "$DFLASH_DEFAULT_MAX_TOKENS" ] && CMD+=(--default-max-tokens "$DFLASH_DEFAULT_MAX_TOKENS") [ -n "$DFLASH_MODEL_NAME" ] && CMD+=(--model-name "$DFLASH_MODEL_NAME") # `--lazy-draft` is silently dropped by the C++ server unless both diff --git a/server/src/common/attn_masks.h b/server/src/common/attn_masks.h index ec25ca561..5c808807f 100644 --- a/server/src/common/attn_masks.h +++ b/server/src/common/attn_masks.h @@ -75,4 +75,24 @@ inline void build_tree_mask(const DDTree & tree, int past_length, } } +// Bidirectional mask for vision image chunks (full attention within chunk + +// to all prior KV). Matches Qwen35Backend::build_bidirectional_mask. +inline void build_bidirectional_mask(std::vector & out, + int kv_len, int n_tokens, int kv_pos, + int kq_stride_pad, + int kv_pad_override = 0) { + const int kv_pad = (kv_pad_override > 0) ? kv_pad_override + : align_up(kv_len, kq_stride_pad); + const int q_pad = align_up(n_tokens, KQ_MASK_PAD); + out.assign((size_t)kv_pad * q_pad, F16_NEG_INF); + for (int q = 0; q < n_tokens; q++) { + for (int k = 0; k < kv_pos; k++) { + out[(size_t)q * kv_pad + k] = F16_ZERO; + } + for (int k = kv_pos; k < kv_pos + n_tokens; k++) { + out[(size_t)q * kv_pad + k] = F16_ZERO; + } + } +} + } // namespace dflash::common diff --git a/server/src/common/backend_args.h b/server/src/common/backend_args.h index 1607b1495..ed957a53b 100644 --- a/server/src/common/backend_args.h +++ b/server/src/common/backend_args.h @@ -36,6 +36,10 @@ struct BackendArgs { // Optional: speculative decode draft model (qwen35 only) const char * draft_path = nullptr; + // Native mmproj vision (qwen35) + const char * mmproj_path = nullptr; + bool mmproj_use_gpu = true; + // Device placement DevicePlacement device; DevicePlacement draft_device; diff --git a/server/src/common/backend_factory.cpp b/server/src/common/backend_factory.cpp index ab34f2a19..e9d617eba 100644 --- a/server/src/common/backend_factory.cpp +++ b/server/src/common/backend_factory.cpp @@ -242,6 +242,11 @@ std::unique_ptr create_backend( ? std::max(DFLASH27B_DRAFT_BLOCK_SIZE, args.ddtree_budget + 1) : DFLASH27B_DRAFT_BLOCK_SIZE; cfg.run_dflash = args.draft_path != nullptr; + cfg.mmproj_path = args.mmproj_path; + cfg.mmproj_use_gpu = args.mmproj_use_gpu; + if (const char * mt = std::getenv("DFLASH_MMPROJ_THREADS")) { + cfg.mmproj_threads = std::max(1, std::atoi(mt)); + } auto adapter = std::make_unique(cfg); auto backend = std::make_unique(std::move(adapter)); @@ -270,6 +275,8 @@ std::unique_ptr create_backend( cfg.ddtree_temp = args.ddtree_temp; cfg.ddtree_chain_seed = args.ddtree_chain_seed; cfg.use_feature_mirror = args.use_feature_mirror; + cfg.mmproj_path = args.mmproj_path; + cfg.mmproj_use_gpu = args.mmproj_use_gpu; auto backend = std::make_unique(cfg); if (!backend->init()) { diff --git a/server/src/common/layer_split_backend.cpp b/server/src/common/layer_split_backend.cpp index b2c0ebbde..4e819e628 100644 --- a/server/src/common/layer_split_backend.cpp +++ b/server/src/common/layer_split_backend.cpp @@ -25,6 +25,10 @@ bool LayerSplitBackend::init() { return adapter_->init(); } +bool LayerSplitBackend::supports_multimodal() const { + return adapter_ && adapter_->supports_multimodal(); +} + void LayerSplitBackend::print_ready_banner() const { std::printf("[daemon] ready\n"); std::fflush(stdout); @@ -56,7 +60,11 @@ GenerateResult LayerSplitBackend::run_from_state(const GenerateRequest & req, } DaemonIO out_io = io.with_token_callback(req.on_token); - if (base_pos + (int)req.prompt.size() + req.n_gen + 1 > adapter_->max_context()) { + const bool use_multimodal = + req.multimodal && !req.multimodal->images.empty(); + + if (!use_multimodal && + base_pos + (int)req.prompt.size() + req.n_gen + 1 > adapter_->max_context()) { result.fail(GenerateErrorCode::ContextOverflow); return result; } @@ -69,39 +77,65 @@ GenerateResult LayerSplitBackend::run_from_state(const GenerateRequest & req, adapter_->begin_request(req); if (reset_state) adapter_->reset_request_state(); - const int prompt_len = (int)req.prompt.size(); - const int adapter_chunk = adapter_->prefill_chunk_tokens(); - int last_tok = (base_pos > 0 && prompt_len == 0) + int last_tok = (base_pos > 0 && !use_multimodal && req.prompt.empty()) ? adapter_->current_last_token() : -1; - int consumed = 0; auto t_prefill_start = std::chrono::steady_clock::now(); - while (consumed < prompt_len) { - int n_tokens = prompt_len - consumed; - if (adapter_chunk > 0 && n_tokens > adapter_chunk) { - n_tokens = adapter_chunk; - } - if (req.snap_pos >= 0 && req.snap_slot >= 0 && - req.snap_pos > base_pos + consumed && - req.snap_pos < base_pos + consumed + n_tokens) { - n_tokens = req.snap_pos - (base_pos + consumed); + + if (use_multimodal) { + if (!adapter_->supports_multimodal()) { + result.fail(GenerateErrorCode::AdapterUnavailable, + "vision not configured"); + return result; } - std::vector chunk(req.prompt.begin() + consumed, - req.prompt.begin() + consumed + n_tokens); - if (!adapter_->prefill(chunk, base_pos + consumed, last_tok)) { + MultimodalPrompt mm = *req.multimodal; + const int committed = adapter_->prefill_multimodal(mm, last_tok); + if (committed < 0) { result.fail(GenerateErrorCode::PrefillFailed); return result; } - consumed += n_tokens; + if (committed + req.n_gen + 1 > adapter_->max_context()) { + result.fail(GenerateErrorCode::ContextOverflow); + return result; + } if (req.snap_pos >= 0 && req.snap_slot >= 0 && - base_pos + consumed == req.snap_pos) { - if (adapter_->snapshot_save(req.snap_slot)) { - std::printf("[snap] inline slot=%d cur_pos=%d\n", - req.snap_slot, req.snap_pos); - std::fflush(stdout); + adapter_->snapshot_save(req.snap_slot)) { + std::printf("[snap] inline slot=%d cur_pos=%d\n", + req.snap_slot, req.snap_pos); + std::fflush(stdout); + } + } else { + const int prompt_len = (int)req.prompt.size(); + const int adapter_chunk = adapter_->prefill_chunk_tokens(); + int consumed = 0; + while (consumed < prompt_len) { + int n_tokens = prompt_len - consumed; + if (adapter_chunk > 0 && n_tokens > adapter_chunk) { + n_tokens = adapter_chunk; + } + if (req.snap_pos >= 0 && req.snap_slot >= 0 && + req.snap_pos > base_pos + consumed && + req.snap_pos < base_pos + consumed + n_tokens) { + n_tokens = req.snap_pos - (base_pos + consumed); + } + std::vector chunk(req.prompt.begin() + consumed, + req.prompt.begin() + consumed + n_tokens); + if (!adapter_->prefill(chunk, base_pos + consumed, last_tok)) { + result.fail(GenerateErrorCode::PrefillFailed); + return result; + } + consumed += n_tokens; + if (req.snap_pos >= 0 && req.snap_slot >= 0 && + base_pos + consumed == req.snap_pos) { + if (adapter_->snapshot_save(req.snap_slot)) { + std::printf("[snap] inline slot=%d cur_pos=%d\n", + req.snap_slot, req.snap_pos); + std::fflush(stdout); + } } } } + result.prefill_s = std::chrono::duration( std::chrono::steady_clock::now() - t_prefill_start).count(); @@ -111,13 +145,17 @@ GenerateResult LayerSplitBackend::run_from_state(const GenerateRequest & req, return result; } auto t_decode_start = std::chrono::steady_clock::now(); - const bool use_dflash = !req.force_ar_decode && adapter_->can_dflash_decode(); + const int committed_pos = use_multimodal + ? adapter_->current_cur_pos() + : base_pos + (int)req.prompt.size(); + const bool force_ar = req.force_ar_decode || use_multimodal; + const bool use_dflash = !force_ar && adapter_->can_dflash_decode(); if (use_dflash) result.spec_decode_ran = true; float dflash_accept_rate = 0.0f; const bool ok = use_dflash ? adapter_->decode_dflash(req.prompt, base_pos, last_tok, req.n_gen, result.tokens, out_io, dflash_accept_rate) - : adapter_->decode_ar(last_tok, base_pos + (int)req.prompt.size(), req.n_gen, + : adapter_->decode_ar(last_tok, committed_pos, req.n_gen, history_prefix, result.tokens, out_io); if (use_dflash) result.accept_rate = dflash_accept_rate; diff --git a/server/src/common/layer_split_backend.h b/server/src/common/layer_split_backend.h index 390870164..4f87ac1a2 100644 --- a/server/src/common/layer_split_backend.h +++ b/server/src/common/layer_split_backend.h @@ -28,6 +28,12 @@ class LayerSplitAdapter { virtual int prefill_chunk_tokens() const { return 0; } virtual bool prefill(const std::vector & prompt, int base_pos, int & last_tok) = 0; + virtual bool supports_multimodal() const { return false; } + virtual int prefill_multimodal(MultimodalPrompt & mm, int & last_tok) { + (void)mm; + (void)last_tok; + return -1; + } // history_prefix is the full original request prompt (not the delta // prefill after a prefix-cache restore); it seeds sampler penalty history. virtual bool decode_ar(int last_tok, int committed, int n_gen, @@ -77,6 +83,7 @@ class LayerSplitAdapter { return false; } virtual int current_last_token() const { return -1; } + virtual int current_cur_pos() const { return 0; } virtual void shutdown() = 0; }; @@ -96,6 +103,8 @@ class LayerSplitBackend : public ModelBackend { bool unpark(ParkTarget target) override; bool is_target_parked() const override { return false; } + bool supports_multimodal() const override; + GenerateResult generate_impl(const GenerateRequest & req, const DaemonIO & io) override; diff --git a/server/src/common/model_backend.h b/server/src/common/model_backend.h index 8c072121d..d9ea69fac 100644 --- a/server/src/common/model_backend.h +++ b/server/src/common/model_backend.h @@ -13,12 +13,15 @@ #include #include #include +#include #include #include #include #include #include +#include "vision_types.h" + #include "ggml.h" #include "ggml-backend.h" #include "sampler.h" @@ -177,8 +180,58 @@ struct GenerateRequest { // path returns success but emits no tokens, so each backend can route the // retry through its existing AR path without copying retry policy. bool force_ar_decode = false; + // Native mmproj vision prompt. When set, backends run multimodal prefill + // instead of token-id prefill and force AR decode. + std::unique_ptr multimodal; + + GenerateRequest() = default; + GenerateRequest(const GenerateRequest & other); + GenerateRequest & operator=(const GenerateRequest & other); + GenerateRequest(GenerateRequest &&) noexcept = default; + GenerateRequest & operator=(GenerateRequest &&) noexcept = default; }; +inline GenerateRequest::GenerateRequest(const GenerateRequest & other) + : prompt(other.prompt), + n_gen(other.n_gen), + sampler(other.sampler), + do_sample(other.do_sample), + stream(other.stream), + snap_pos(other.snap_pos), + snap_slot(other.snap_slot), + on_token(other.on_token), + hint_tokens(other.hint_tokens), + stall_tool_prefix_tokens(other.stall_tool_prefix_tokens), + stall_action_suffix_tokens(other.stall_action_suffix_tokens), + stall_skip_tokens(other.stall_skip_tokens), + budget_hook(other.budget_hook), + force_ar_decode(other.force_ar_decode), + multimodal(other.multimodal + ? std::make_unique(*other.multimodal) + : nullptr) {} + +inline GenerateRequest & GenerateRequest::operator=(const GenerateRequest & other) { + if (this == &other) return *this; + prompt = other.prompt; + n_gen = other.n_gen; + sampler = other.sampler; + do_sample = other.do_sample; + stream = other.stream; + snap_pos = other.snap_pos; + snap_slot = other.snap_slot; + on_token = other.on_token; + hint_tokens = other.hint_tokens; + stall_tool_prefix_tokens = other.stall_tool_prefix_tokens; + stall_action_suffix_tokens = other.stall_action_suffix_tokens; + stall_skip_tokens = other.stall_skip_tokens; + budget_hook = other.budget_hook; + force_ar_decode = other.force_ar_decode; + multimodal = other.multimodal + ? std::make_unique(*other.multimodal) + : nullptr; + return *this; +} + // Stable, backend-independent generation failure categories. Backends should // use these for recurrent failures so callers do not need to understand // architecture-specific strings. `generate_error_code()` is the daemon/API @@ -434,6 +487,9 @@ struct ModelBackend { // the server before startup. virtual bool supports_remote_draft() const { return false; } + // Native mmproj vision. LayerSplitBackend overrides to proxy the adapter. + virtual bool supports_multimodal() const { return false; } + // Layer-split capability introspection. Non layer-split backends keep the // default false; LayerSplitBackend proxies model-adapter support. virtual bool supports_kvflash() const { return false; } diff --git a/server/src/common/vision_types.h b/server/src/common/vision_types.h new file mode 100644 index 000000000..118c3a5e4 --- /dev/null +++ b/server/src/common/vision_types.h @@ -0,0 +1,24 @@ +// Multimodal prompt types for native mmproj vision integration. + +#pragma once + +#include +#include +#include + +namespace dflash::common { + +// Raw image bytes (JPEG/PNG/etc.) decoded from a data: URL or API payload. +struct DecodedImage { + std::vector bytes; +}; + +// A prompt with mtmd media markers embedded in marked_text plus parallel +// image payloads. marked_text must contain one mtmd_default_marker() per +// image (in order) before tokenize() is called. +struct MultimodalPrompt { + std::string marked_text; + std::vector images; +}; + +} // namespace dflash::common diff --git a/server/src/qwen35/layer_split_forward.cpp b/server/src/qwen35/layer_split_forward.cpp index a66383cc9..37a7078b1 100644 --- a/server/src/qwen35/layer_split_forward.cpp +++ b/server/src/qwen35/layer_split_forward.cpp @@ -425,7 +425,8 @@ bool run_qwen35_layer_split_layers_from_activation( bool capture_ssm_intermediates, Qwen35SplitCaptureStats * capture_stats, bool kvflash_preallocated = false, - const Qwen35SplitTreeInputs * tree_inputs = nullptr) { + const Qwen35SplitTreeInputs * tree_inputs = nullptr, + const LayerSplitAttnPrefillOpts * attn_opts = nullptr) { if (shards.empty() || !acts.a || !acts.b || n_tokens_total <= 0) return false; if (kvflash && fa_window > 0) { std::fprintf(stderr, @@ -509,16 +510,23 @@ bool run_qwen35_layer_split_layers_from_activation( return false; } if (is_attn && shard->layer_graph.positions) { - pos_buf.assign((size_t)4 * n, 0); - for (int i = 0; i < n; i++) { - const int p = kv_start + i; - pos_buf[0 * n + i] = p; - pos_buf[1 * n + i] = p; - pos_buf[2 * n + i] = p; - pos_buf[3 * n + i] = 0; + if (attn_opts && attn_opts->positions) { + const int32_t * src = attn_opts->positions + (size_t)4 * (size_t)start; + pos_buf.assign(src, src + (size_t)4 * (size_t)n); + ggml_backend_tensor_set(shard->layer_graph.positions, pos_buf.data(), 0, + sizeof(int32_t) * pos_buf.size()); + } else { + pos_buf.assign((size_t)4 * n, 0); + for (int i = 0; i < n; i++) { + const int p = kv_start + i; + pos_buf[0 * n + i] = p; + pos_buf[1 * n + i] = p; + pos_buf[2 * n + i] = p; + pos_buf[3 * n + i] = 0; + } + ggml_backend_tensor_set(shard->layer_graph.positions, pos_buf.data(), 0, + sizeof(int32_t) * pos_buf.size()); } - ggml_backend_tensor_set(shard->layer_graph.positions, pos_buf.data(), 0, - sizeof(int32_t) * pos_buf.size()); } if (is_attn && kvflash) { if (!fill_qwen35_kvflash_inputs( @@ -527,12 +535,17 @@ bool run_qwen35_layer_split_layers_from_activation( return false; } } else if (is_attn && with_mask && shard->layer_graph.attn_mask) { - const int win_start_l = (fa_window > 0 && kv_start > fa_window) - ? (kv_start - fa_window) : 0; - const int win_len_l = kv_len - win_start_l; const int kv_pad_override = (int)shard->layer_graph.attn_mask->ne[0]; - build_causal_mask(mask_buf, win_len_l, n, kv_start, kq_stride_pad, - win_start_l, kv_pad_override); + if (attn_opts && attn_opts->bidirectional) { + build_bidirectional_mask(mask_buf, kv_len, n, kv_start, + kq_stride_pad, kv_pad_override); + } else { + const int win_start_l = (fa_window > 0 && kv_start > fa_window) + ? (kv_start - fa_window) : 0; + const int win_len_l = kv_len - win_start_l; + build_causal_mask(mask_buf, win_len_l, n, kv_start, kq_stride_pad, + win_start_l, kv_pad_override); + } ggml_backend_tensor_set(shard->layer_graph.attn_mask, mask_buf.data(), 0, sizeof(uint16_t) * mask_buf.size()); } @@ -627,12 +640,14 @@ bool run_qwen35_layer_split_forward_from_activation( KvFlashPager * kvflash, bool kvflash_preallocated, const Qwen35SplitTreeInputs * tree_inputs, - bool capture_ssm_intermediates) { + bool capture_ssm_intermediates, + const LayerSplitAttnPrefillOpts * attn_opts) { if (!run_qwen35_layer_split_layers_from_activation( shards, acts, base_pos, n_tokens_total, ubatch, kq_stride_pad, fa_window, captures_out, nullptr, nullptr, kvflash, capture_ssm_intermediates, - /*capture_stats=*/nullptr, kvflash_preallocated, tree_inputs)) { + /*capture_stats=*/nullptr, kvflash_preallocated, tree_inputs, + attn_opts)) { return false; } diff --git a/server/src/qwen35/layer_split_forward.h b/server/src/qwen35/layer_split_forward.h index c9ddb3b9a..8895755f5 100644 --- a/server/src/qwen35/layer_split_forward.h +++ b/server/src/qwen35/layer_split_forward.h @@ -22,6 +22,14 @@ namespace dflash::common { class KvFlashPager; +// Optional attention overrides for multimodal prefill (mRoPE positions, +// bidirectional image masks). positions uses layer-split layout: +// [dim0*n + i, dim1*n + i, dim2*n + i, dim3*n + i] for n query tokens. +struct LayerSplitAttnPrefillOpts { + const int32_t * positions = nullptr; + bool bidirectional = false; +}; + // Compute argmax(logits) for a slice of the activation tensor via // out_norm + lm_head projection. bool compute_target_split_argmax( @@ -83,7 +91,8 @@ bool run_qwen35_layer_split_forward_from_activation( KvFlashPager * kvflash = nullptr, bool kvflash_preallocated = false, const Qwen35SplitTreeInputs * tree_inputs = nullptr, - bool capture_ssm_intermediates = false); + bool capture_ssm_intermediates = false, + const LayerSplitAttnPrefillOpts * attn_opts = nullptr); bool run_qwen35_layer_split_tree_verify_from_activation( std::vector & shards, diff --git a/server/src/qwen35/qwen35_backend.cpp b/server/src/qwen35/qwen35_backend.cpp index 8c83bc2b8..83b69eac5 100644 --- a/server/src/qwen35/qwen35_backend.cpp +++ b/server/src/qwen35/qwen35_backend.cpp @@ -39,6 +39,12 @@ #include "kv_quant.h" +#ifdef DFLASH_HAVE_MMPROJ +#include "vision_encoder.h" +#include "mtmd.h" +#include "mtmd-helper.h" +#endif + namespace dflash::common { namespace { @@ -334,6 +340,18 @@ bool Qwen35Backend::init() { } } +#ifdef DFLASH_HAVE_MMPROJ + if (cfg_.mmproj_path) { + vision_ = std::make_unique(); + const int n_threads = env_int_or_default("DFLASH_MMPROJ_THREADS", 4); + if (!vision_->init(cfg_.target_path, cfg_.mmproj_path, + cfg_.mmproj_use_gpu, n_threads)) { + std::fprintf(stderr, "[qwen35] VisionEncoder init failed\n"); + return false; + } + } +#endif + return true; } @@ -806,7 +824,27 @@ GenerateResult Qwen35Backend::generate_impl(const GenerateRequest & req, // Prefill auto t_prefill_start = std::chrono::steady_clock::now(); - const int committed = do_prefill(req.prompt, out_io, req.snap_pos, req.snap_slot); + const bool use_multimodal = req.multimodal && !req.multimodal->images.empty(); +#ifdef DFLASH_HAVE_MMPROJ + const bool vision_ready = vision_ && vision_->ready(); +#else + const bool vision_ready = false; +#endif + int committed = -1; + if (use_multimodal) { + if (!vision_ready) { + result.fail(GenerateErrorCode::AdapterUnavailable, + "vision not configured"); + return result; + } + // Free oversized gallocr/BSA scratch from prior text-only requests before + // multimodal prefill reserves a fresh graph (avoids CUDA OOM on agent payloads). + release_scratch(); + committed = do_prefill_multimodal(*req.multimodal, out_io, + req.snap_pos, req.snap_slot); + } else { + committed = do_prefill(req.prompt, out_io, req.snap_pos, req.snap_slot); + } if (committed < 0) { result.fail(GenerateErrorCode::PrefillFailed); return result; @@ -824,7 +862,8 @@ GenerateResult Qwen35Backend::generate_impl(const GenerateRequest & req, // generation. Most requests never hit the tail because the // model closes naturally well before the budget edge. bool decode_ok = false; - if (req.force_ar_decode) { + const bool force_ar = req.force_ar_decode || use_multimodal; + if (force_ar) { decode_ok = do_ar_decode(committed, req.n_gen, result.tokens, out_io, req.budget_hook, &result.budget_forced_close, @@ -963,6 +1002,8 @@ GenerateResult Qwen35Backend::restore_and_generate_impl(int slot, } } + const bool use_multimodal = req.multimodal && !req.multimodal->images.empty(); + // Decode if (req.n_gen > 0) { auto t_decode_start = std::chrono::steady_clock::now(); @@ -973,7 +1014,8 @@ GenerateResult Qwen35Backend::restore_and_generate_impl(int slot, // generation. Most requests never hit the tail because the // model closes naturally well before the budget edge. bool decode_ok = false; - if (req.force_ar_decode) { + const bool force_ar = req.force_ar_decode || use_multimodal; + if (force_ar) { decode_ok = do_ar_decode(committed, req.n_gen, result.tokens, out_io, req.budget_hook, &result.budget_forced_close, @@ -1281,6 +1323,245 @@ int Qwen35Backend::do_prefill(const std::vector & tokens, return committed; } +#ifdef DFLASH_HAVE_MMPROJ + +bool Qwen35Backend::do_prefill_embed_chunk(int kv_pos, int n_tokens, + const float * embeds, int hidden, + const std::vector & pos_buf, + bool with_mask, + bool last_token_logits_only, + bool non_causal_within_chunk) { + if (!build_target_step(sg_, w_, cache_, target_backend_, + /*kv_start=*/kv_pos, /*n_tokens=*/n_tokens, + with_mask, /*capture=*/true, + /*capture_delta_intermediate=*/false, + /*fa_window=*/0, + /*last_token_logits_only=*/last_token_logits_only, + cfg_.kq_stride_pad, + should_capture_moe_router(), + /*kvflash_mask=*/false)) { + std::fprintf(stderr, "multimodal prefill build @%d\n", kv_pos); + return false; + } + + ggml_backend_tensor_set(sg_.inp_embed, embeds, 0, + sizeof(float) * (size_t)hidden * (size_t)n_tokens); + ggml_backend_tensor_set(sg_.positions, pos_buf.data(), 0, + sizeof(int32_t) * pos_buf.size()); + + if (sg_.attn_mask) { + const int kv_len = kv_pos + n_tokens; + std::vector mask_buf; + const int kv_pad_override = (int)sg_.attn_mask->ne[0]; + if (non_causal_within_chunk) { + build_bidirectional_mask(mask_buf, kv_len, n_tokens, kv_pos, + cfg_.kq_stride_pad, kv_pad_override); + } else { + build_causal_mask(mask_buf, kv_len, n_tokens, kv_pos, + cfg_.kq_stride_pad, 0, kv_pad_override); + } + ggml_backend_tensor_set(sg_.attn_mask, mask_buf.data(), 0, + sizeof(uint16_t) * mask_buf.size()); + } + + auto st = ggml_backend_graph_compute(target_backend_, sg_.gf); + if (st != GGML_STATUS_SUCCESS) { + std::fprintf(stderr, "multimodal prefill compute @%d failed\n", kv_pos); + return false; + } + after_target_compute(sg_, kv_pos, n_tokens); + return true; +} + +int Qwen35Backend::do_prefill_multimodal(MultimodalPrompt & mm, + const DaemonIO & io, + int snap_pos, int snap_slot, + int kv_offset) { + (void)io; + if (!vision_ || !vision_->ready()) { + std::fprintf(stderr, "[vision] encoder not initialized\n"); + return -1; + } + if (kv_offset != 0) { + std::fprintf(stderr, "[vision] multimodal restore prefill unsupported\n"); + return -1; + } + + const int hidden = w_.n_embd; + const int vocab = w_.n_vocab; + prefill_last_logits_valid_ = false; + reset_recurrent_state(cache_); + + mtmd_input_chunks * chunks = vision_->tokenize(mm.marked_text, mm.images); + if (!chunks) return -1; + + struct ChunksGuard { + mtmd_input_chunks * p; + ~ChunksGuard() { if (p) mtmd_input_chunks_free(p); } + } chunks_guard{chunks}; + + const size_t n_chunks = mtmd_input_chunks_size(chunks); + if (n_chunks == 0) return 0; + + const int max_verify_tokens = cfg_.ddtree_mode + ? std::max(dw_.block_size, cfg_.ddtree_budget + 1) + : dw_.block_size; + if (!migrate_prefill_cache(w_, cfg_.device.max_ctx, + max_verify_tokens, + target_backend_, cache_)) { + std::fprintf(stderr, "multimodal prefill: cache migration failed: %s\n", + dflash27b_last_error()); + return -1; + } + + int prefill_ubatch = 512; + if (const char * s = std::getenv("DFLASH27B_PREFILL_UBATCH")) { + prefill_ubatch = std::max(1, std::atoi(s)); + } + + std::vector embed_buf; + int committed = kv_offset; + + for (size_t ci = 0; ci < n_chunks; ci++) { + const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, ci); + const auto chunk_type = mtmd_input_chunk_get_type(chunk); + const bool is_last_chunk = (ci + 1 == n_chunks); + const int kv_pos = committed; + + if (chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT) { + size_t n_tokens_sz = 0; + const llama_token * toks = + mtmd_input_chunk_get_tokens_text(chunk, &n_tokens_sz); + const int n_tokens = (int)n_tokens_sz; + if (n_tokens <= 0) continue; + + if (n_tokens > prefill_ubatch) { + std::fprintf(stderr, + "[vision] chunking text mtmd_chunk=%zu tokens=%d ubatch=%d\n", + ci, n_tokens, prefill_ubatch); + } + + std::vector tokens(n_tokens); + for (int i = 0; i < n_tokens; i++) tokens[(size_t)i] = (int32_t)toks[i]; + + for (int start = 0; start < n_tokens; ) { + const int sub_n = std::min(prefill_ubatch, n_tokens - start); + const bool is_last_sub = (start + sub_n >= n_tokens); + const bool sub_last_logits_only = !(is_last_chunk && is_last_sub); + + embed_buf.resize((size_t)hidden * (size_t)sub_n); + if (!w_.embedder.embed(tokens.data() + start, sub_n, + embed_buf.data())) { + return -1; + } + + std::vector pos_buf((size_t)4 * (size_t)sub_n, 0); + for (int i = 0; i < sub_n; i++) { + const int p = kv_pos + start + i; + pos_buf[4 * i + 0] = p; + pos_buf[4 * i + 1] = p; + pos_buf[4 * i + 2] = p; + pos_buf[4 * i + 3] = 0; + } + + const bool with_mask = (cfg_.kq_stride_pad > KQ_MASK_PAD) || (sub_n > 1); + if (!do_prefill_embed_chunk(kv_pos + start, sub_n, embed_buf.data(), + hidden, pos_buf, with_mask, + sub_last_logits_only, + /*non_causal_within_chunk=*/false)) { + return -1; + } + start += sub_n; + } + + committed = kv_pos + n_tokens; + } else if (chunk_type == MTMD_INPUT_CHUNK_TYPE_IMAGE || + chunk_type == MTMD_INPUT_CHUNK_TYPE_AUDIO) { + const int n_tokens = (int)mtmd_input_chunk_get_n_tokens(chunk); + if (n_tokens <= 0) continue; + + if (!vision_->encode_chunk(chunk)) return -1; + float * embd = vision_->output_embeddings(); + if (!embd) return -1; + + std::vector pos_buf; + if (vision_->uses_mrope() && + chunk_type == MTMD_INPUT_CHUNK_TYPE_IMAGE) { + const auto * image_tokens = mtmd_input_chunk_get_tokens_image(chunk); + if (!image_tokens) return -1; + std::vector rel_pos((size_t)n_tokens); + vision_->get_image_decoder_pos(image_tokens, rel_pos.data(), + (size_t)n_tokens); + pos_buf.resize((size_t)4 * (size_t)n_tokens, 0); + for (int i = 0; i < n_tokens; i++) { + pos_buf[4 * i + 0] = kv_pos + (int)rel_pos[(size_t)i].t; + pos_buf[4 * i + 1] = kv_pos + (int)rel_pos[(size_t)i].y; + pos_buf[4 * i + 2] = kv_pos + (int)rel_pos[(size_t)i].x; + pos_buf[4 * i + 3] = 0; + } + } else { + pos_buf.resize((size_t)4 * (size_t)n_tokens, 0); + for (int i = 0; i < n_tokens; i++) { + const int p = kv_pos + i; + pos_buf[4 * i + 0] = p; + pos_buf[4 * i + 1] = p; + pos_buf[4 * i + 2] = p; + pos_buf[4 * i + 3] = 0; + } + } + + const bool non_causal = vision_->uses_non_causal(chunk); + const bool with_mask = non_causal || (cfg_.kq_stride_pad > KQ_MASK_PAD) || + (n_tokens > 1); + if (!do_prefill_embed_chunk(kv_pos, n_tokens, embd, hidden, + pos_buf, with_mask, + /*last_token_logits_only=*/!is_last_chunk, + non_causal)) { + return -1; + } + + committed = kv_pos + (int)mtmd_input_chunk_get_n_pos(chunk); + } else { + std::fprintf(stderr, "[vision] unsupported mtmd chunk type\n"); + return -1; + } + + cache_.cur_pos = committed; + + const int chunk_kv_len = committed - kv_pos; + if (chunk_kv_len > 0) { + if (remote_draft_.active() && !draft_parked_) { + if (!sync_remote_draft_features(kv_pos, chunk_kv_len)) return -1; + } else if (feature_mirror_.target_feat && !draft_parked_) { + draft_feature_mirror_sync_range(cache_.target_feat, cache_.target_feat_cap, + feature_mirror_, kv_pos, chunk_kv_len); + } + } + + int32_t last_tok = -1; + ggml_backend_tensor_get(sg_.argmax_tokens, &last_tok, 0, sizeof(int32_t)); + cache_.last_tok = last_tok; + if (is_last_chunk) { + prefill_last_logits_offset_ = + (size_t)(mtmd_input_chunk_get_n_tokens(chunk) - 1) * + (size_t)vocab * sizeof(float); + prefill_last_logits_valid_ = true; + } + } + + if (snap_slot >= 0 && snap_pos == committed) { + if (snapshot_save(snap_slot)) { + std::printf("[snap] multimodal end-of-prefill slot=%d cur_pos=%d\n", + snap_slot, committed); + std::fflush(stdout); + } + } + + return committed; +} + +#endif // DFLASH_HAVE_MMPROJ + // ── kvflash helpers ───────────────────────────────────────────────── void Qwen35Backend::kvflash_sync_prefill(int committed, diff --git a/server/src/qwen35/qwen35_backend.h b/server/src/qwen35/qwen35_backend.h index d14f931f7..38c7f2d2a 100644 --- a/server/src/qwen35/qwen35_backend.h +++ b/server/src/qwen35/qwen35_backend.h @@ -34,6 +34,10 @@ #include #include +namespace dflash::common { +class VisionEncoder; +} + namespace dflash::common { class Qwen35TensorParallelContext; @@ -70,6 +74,10 @@ struct Qwen35Config { float ddtree_temp = 1.0f; bool ddtree_chain_seed = true; bool use_feature_mirror = false; + + // Native mmproj vision + const char * mmproj_path = nullptr; + bool mmproj_use_gpu = true; }; // ── Backend class ─────────────────────────────────────────────────────── @@ -123,6 +131,11 @@ class Qwen35Backend : public ModelBackend { bool supports_dflash_spec_decode() const override { return true; } DFlashTarget * dflash_target() override; bool supports_remote_draft() const override { return true; } +#ifdef DFLASH_HAVE_MMPROJ + bool supports_multimodal() const override { return vision_ != nullptr; } +#else + bool supports_multimodal() const override { return false; } +#endif void shutdown() override; @@ -266,6 +279,10 @@ class Qwen35Backend : public ModelBackend { // ── DFlashTarget adapter (lazy-built) ──────────────────────────── std::unique_ptr dflash_target_; +#ifdef DFLASH_HAVE_MMPROJ + std::unique_ptr vision_; +#endif + // ── Internal helpers ───────────────────────────────────────────── // Prefill a prompt and return the number of tokens committed to KV. // kv_offset > 0 resumes from a restored snapshot: tokens are placed at @@ -275,6 +292,17 @@ class Qwen35Backend : public ModelBackend { int snap_pos = -1, int snap_slot = -1, int kv_offset = 0); + int do_prefill_multimodal(MultimodalPrompt & mm, + const DaemonIO & io, + int snap_pos = -1, int snap_slot = -1, + int kv_offset = 0); + + bool do_prefill_embed_chunk(int kv_pos, int n_tokens, + const float * embeds, int hidden, + const std::vector & pos_buf, + bool with_mask, bool last_token_logits_only, + bool non_causal_within_chunk); + // Speculative decode loop: draft → verify → accept until EOS/max. // When budget_hook is non-null and (n_gen - generated) drops to the // hard-limit boundary, breaks out of the spec-decode loop and tails diff --git a/server/src/qwen35/qwen35_layer_split_adapter.cpp b/server/src/qwen35/qwen35_layer_split_adapter.cpp index a323e6a93..d6dd09890 100644 --- a/server/src/qwen35/qwen35_layer_split_adapter.cpp +++ b/server/src/qwen35/qwen35_layer_split_adapter.cpp @@ -14,6 +14,7 @@ #include "qwen35/qwen35_layer_split_dflash_target.h" #include "qwen3/qwen3_drafter.h" #include "qwen3/qwen3_kvflash_scorer.h" +#include "vision/vision_encoder.h" #include "kv_quant.h" #include "ggml-cuda.h" @@ -107,6 +108,20 @@ bool Qwen35LayerSplitAdapter::init() { if (cfg_.draft_path && cfg_.run_dflash && !load_draft()) { return false; } + +#ifdef DFLASH_HAVE_MMPROJ + if (cfg_.mmproj_path && cfg_.mmproj_path[0]) { + vision_ = std::make_unique(); + if (!vision_->init(cfg_.target_path, cfg_.mmproj_path, + cfg_.mmproj_use_gpu, cfg_.mmproj_threads)) { + std::fprintf(stderr, "[target-split][vision] mmproj init failed\n"); + return false; + } + std::fprintf(stderr, "[target-split][vision] mmproj loaded: %s\n", + cfg_.mmproj_path); + } +#endif + prefix_snapshots_.resize(PREFIX_SLOTS); for (auto & slot : prefix_snapshots_) { slot.resize(shards_.size()); @@ -1274,6 +1289,11 @@ int Qwen35LayerSplitAdapter::current_last_token() const { return shards_.front().cache.last_tok; } +int Qwen35LayerSplitAdapter::current_cur_pos() const { + if (shards_.empty()) return 0; + return shards_.front().cache.cur_pos; +} + bool Qwen35LayerSplitAdapter::decode_ar( int last_tok, int committed, int n_gen, const std::vector & history_prefix, @@ -1419,6 +1439,9 @@ DFlashTarget * Qwen35LayerSplitAdapter::dflash_target() { void Qwen35LayerSplitAdapter::shutdown() { dflash_target_.reset(); +#ifdef DFLASH_HAVE_MMPROJ + vision_.reset(); +#endif free_drafter(); for (int slot = 0; slot < (int)prefix_snapshots_.size(); ++slot) { snapshot_free(slot); diff --git a/server/src/qwen35/qwen35_layer_split_adapter.h b/server/src/qwen35/qwen35_layer_split_adapter.h index 7f5e7b129..a9af3bd6f 100644 --- a/server/src/qwen35/qwen35_layer_split_adapter.h +++ b/server/src/qwen35/qwen35_layer_split_adapter.h @@ -14,6 +14,7 @@ #include "placement/remote_target_shard_config.h" #include "qwen3/qwen3_drafter.h" #include "qwen35_target_shard_ipc.h" +#include "qwen35/layer_split_forward.h" #include "step_graph.h" #include "internal.h" @@ -24,6 +25,10 @@ #include #include +namespace dflash::common { +class VisionEncoder; +} + namespace dflash::common { struct Qwen35LayerSplitAdapterConfig { @@ -41,6 +46,9 @@ struct Qwen35LayerSplitAdapterConfig { int max_verify_tokens = DFLASH27B_DRAFT_BLOCK_SIZE; bool run_dflash = false; int draft_swa_window = 0; + const char * mmproj_path = nullptr; + bool mmproj_use_gpu = true; + int mmproj_threads = 4; }; class Qwen35LayerSplitAdapter : public LayerSplitAdapter { @@ -60,6 +68,8 @@ class Qwen35LayerSplitAdapter : public LayerSplitAdapter { int prefill_chunk_tokens() const override; bool prefill(const std::vector & prompt, int base_pos, int & last_tok) override; + bool supports_multimodal() const override; + int prefill_multimodal(MultimodalPrompt & mm, int & last_tok) override; bool decode_ar(int last_tok, int committed, int n_gen, const std::vector & history_prefix, std::vector & out_tokens, @@ -86,6 +96,7 @@ class Qwen35LayerSplitAdapter : public LayerSplitAdapter { ggml_backend_buffer_t buf, int cur_pos, int32_t last_tok) override; int current_last_token() const override; + int current_cur_pos() const override; bool supports_dflash_spec_decode() const override { return true; } DFlashTarget * dflash_target() override; @@ -114,6 +125,10 @@ class Qwen35LayerSplitAdapter : public LayerSplitAdapter { bool snapshot_draft_features(int slot); void free_draft_feature_snapshot(int slot); bool restore_draft_features(int slot); + bool prefill_activation_chunk(int kv_pos, int n_tokens, const float * embeds, + int hidden, const LayerSplitAttnPrefillOpts * attn_opts, + bool want_logits); + void sync_draft_features_range(int kv_pos, int n_tokens); Qwen35LayerSplitAdapterConfig cfg_; std::vector shards_; @@ -162,6 +177,7 @@ class Qwen35LayerSplitAdapter : public LayerSplitAdapter { std::mt19937_64 sampler_rng_{std::random_device{}()}; std::unique_ptr dflash_target_; std::vector prefill_last_logits_; + std::unique_ptr vision_; }; } // namespace dflash::common diff --git a/server/src/qwen35/qwen35_layer_split_vision.cpp b/server/src/qwen35/qwen35_layer_split_vision.cpp new file mode 100644 index 000000000..aa86c7e49 --- /dev/null +++ b/server/src/qwen35/qwen35_layer_split_vision.cpp @@ -0,0 +1,247 @@ +// Multimodal prefill for Qwen35 layer-split adapter. + +#include "qwen35_layer_split_adapter.h" + +#include "common/dflash_layer_split_runtime.h" +#include "dflash_feature_ring.h" +#include "internal.h" +#include "qwen35/layer_split_forward.h" +#include "vision/vision_encoder.h" + +#include +#include +#include + +namespace dflash::common { + +bool Qwen35LayerSplitAdapter::supports_multimodal() const { +#ifdef DFLASH_HAVE_MMPROJ + return vision_ && vision_->ready(); +#else + return false; +#endif +} + +void Qwen35LayerSplitAdapter::sync_draft_features_range(int kv_pos, int n_tokens) { + if (n_tokens <= 0 || !cfg_.run_dflash) return; + if (remote_draft_.active()) { + // Remote draft sync is handled inside spec-decode; skip here. + return; + } + if (feature_ring_.target_feat && !shards_.empty()) { + const auto & cache = shards_.front().cache; + if (cache.target_feat) { + draft_feature_mirror_sync_range(cache.target_feat, cache.target_feat_cap, + feature_ring_, kv_pos, n_tokens); + } + } +} + +bool Qwen35LayerSplitAdapter::prefill_activation_chunk( + int kv_pos, int n_tokens, const float * embeds, int hidden, + const LayerSplitAttnPrefillOpts * attn_opts, bool want_logits) { + if (shards_.empty() || n_tokens <= 0 || !embeds) return false; + if (use_mixed_target_split()) { + std::fprintf(stderr, "[target-split][vision] mixed target split unsupported\n"); + return false; + } + + int ubatch = cfg_.chunk > 0 ? cfg_.chunk : 512; + if (const char * s = std::getenv("DFLASH27B_PREFILL_UBATCH")) { + ubatch = std::max(1, std::atoi(s)); + } + + ActivationPair acts; + if (!activation_pair_init(acts, shards_.front().backend, hidden, n_tokens, + activation_type_)) { + std::fprintf(stderr, "[target-split][vision] activation alloc failed\n"); + return false; + } + + if (!set_activation_tensor_from_f32( + acts.a, embeds, 0, (size_t)hidden * (size_t)n_tokens)) { + activation_pair_free(acts); + return false; + } + + int last_tok = -1; + std::vector logits; + const bool ok = run_qwen35_layer_split_forward_from_activation( + shards_, acts, kv_pos, n_tokens, ubatch, last_tok, + cfg_.kq_stride_pad, cfg_.fa_window, + /*argmax_out=*/nullptr, + want_logits ? &prefill_last_logits_ : nullptr, + /*captures_out=*/nullptr, + kvflash_active() ? &kvflash_pager_ : nullptr, + /*kvflash_preallocated=*/false, + /*tree_inputs=*/nullptr, + /*capture_ssm_intermediates=*/false, + attn_opts); + activation_pair_free(acts); + if (!ok) return false; + + if (cfg_.run_dflash && !remote_draft_.active()) { + sync_draft_features_range(kv_pos, n_tokens); + } + return true; +} + +int Qwen35LayerSplitAdapter::prefill_multimodal(MultimodalPrompt & mm, + int & last_tok) { +#ifdef DFLASH_HAVE_MMPROJ + if (!vision_ || !vision_->ready()) { + std::fprintf(stderr, "[target-split][vision] encoder not initialized\n"); + return -1; + } + if (shards_.empty()) return -1; + + for (auto & shard : shards_) { + reset_recurrent_state(shard.cache); + } + + mtmd_input_chunks * chunks = vision_->tokenize(mm.marked_text, mm.images); + if (!chunks) return -1; + + struct ChunksGuard { + mtmd_input_chunks * p; + ~ChunksGuard() { if (p) mtmd_input_chunks_free(p); } + } chunks_guard{chunks}; + + const size_t n_chunks = mtmd_input_chunks_size(chunks); + if (n_chunks == 0) { + last_tok = shards_.front().cache.last_tok; + return 0; + } + + const int hidden = shards_.front().weights.n_embd; + int prefill_ubatch = 512; + if (const char * s = std::getenv("DFLASH27B_PREFILL_UBATCH")) { + prefill_ubatch = std::max(1, std::atoi(s)); + } + + std::vector embed_buf; + int committed = 0; + + for (size_t ci = 0; ci < n_chunks; ci++) { + const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, ci); + const auto chunk_type = mtmd_input_chunk_get_type(chunk); + const bool is_last_chunk = (ci + 1 == n_chunks); + const int kv_pos = committed; + + if (chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT) { + size_t n_tokens_sz = 0; + const llama_token * toks = + mtmd_input_chunk_get_tokens_text(chunk, &n_tokens_sz); + const int n_tokens = (int)n_tokens_sz; + if (n_tokens <= 0) continue; + + if (n_tokens > prefill_ubatch) { + std::fprintf(stderr, + "[target-split][vision] chunking text mtmd_chunk=%zu tokens=%d ubatch=%d\n", + ci, n_tokens, prefill_ubatch); + } + + std::vector tokens((size_t)n_tokens); + for (int i = 0; i < n_tokens; i++) { + tokens[(size_t)i] = (int32_t)toks[i]; + } + + for (int start = 0; start < n_tokens; ) { + const int sub_n = std::min(prefill_ubatch, n_tokens - start); + const bool is_last_sub = (start + sub_n >= n_tokens); + const bool want_logits = is_last_chunk && is_last_sub; + + embed_buf.resize((size_t)hidden * (size_t)sub_n); + if (!shards_.front().weights.embedder.embed( + tokens.data() + start, sub_n, embed_buf.data())) { + return -1; + } + + std::vector pos_buf((size_t)4 * (size_t)sub_n, 0); + for (int i = 0; i < sub_n; i++) { + const int p = kv_pos + start + i; + pos_buf[4 * i + 0] = p; + pos_buf[4 * i + 1] = p; + pos_buf[4 * i + 2] = p; + pos_buf[4 * i + 3] = 0; + } + LayerSplitAttnPrefillOpts attn; + attn.positions = pos_buf.data(); + + if (!prefill_activation_chunk(kv_pos + start, sub_n, + embed_buf.data(), hidden, + &attn, want_logits)) { + return -1; + } + start += sub_n; + } + committed = kv_pos + n_tokens; + } else if (chunk_type == MTMD_INPUT_CHUNK_TYPE_IMAGE || + chunk_type == MTMD_INPUT_CHUNK_TYPE_AUDIO) { + const int n_tokens = (int)mtmd_input_chunk_get_n_tokens(chunk); + if (n_tokens <= 0) continue; + + if (!vision_->encode_chunk(chunk)) return -1; + float * embd = vision_->output_embeddings(); + if (!embd) return -1; + + std::vector pos_buf; + LayerSplitAttnPrefillOpts attn; + if (vision_->uses_mrope() && + chunk_type == MTMD_INPUT_CHUNK_TYPE_IMAGE) { + const auto * image_tokens = + mtmd_input_chunk_get_tokens_image(chunk); + if (!image_tokens) return -1; + std::vector rel_pos((size_t)n_tokens); + vision_->get_image_decoder_pos(image_tokens, rel_pos.data(), + (size_t)n_tokens); + pos_buf.resize((size_t)4 * (size_t)n_tokens, 0); + for (int i = 0; i < n_tokens; i++) { + pos_buf[4 * i + 0] = + kv_pos + (int)rel_pos[(size_t)i].t; + pos_buf[4 * i + 1] = + kv_pos + (int)rel_pos[(size_t)i].y; + pos_buf[4 * i + 2] = + kv_pos + (int)rel_pos[(size_t)i].x; + pos_buf[4 * i + 3] = 0; + } + attn.positions = pos_buf.data(); + } else { + pos_buf.resize((size_t)4 * (size_t)n_tokens, 0); + for (int i = 0; i < n_tokens; i++) { + const int p = kv_pos + i; + pos_buf[4 * i + 0] = p; + pos_buf[4 * i + 1] = p; + pos_buf[4 * i + 2] = p; + pos_buf[4 * i + 3] = 0; + } + attn.positions = pos_buf.data(); + } + attn.bidirectional = vision_->uses_non_causal(chunk); + + if (!prefill_activation_chunk(kv_pos, n_tokens, embd, hidden, &attn, + /*want_logits=*/is_last_chunk)) { + return -1; + } + committed = kv_pos + (int)mtmd_input_chunk_get_n_pos(chunk); + } else { + std::fprintf(stderr, "[target-split][vision] unsupported mtmd chunk\n"); + return -1; + } + + for (auto & shard : shards_) { + shard.cache.cur_pos = committed; + } + } + + last_tok = shards_.front().cache.last_tok; + return committed; +#else + (void)mm; + (void)last_tok; + std::fprintf(stderr, "[target-split][vision] built without DFLASH_HAVE_MMPROJ\n"); + return -1; +#endif +} + +} // namespace dflash::common diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index 46081fa23..b26925897 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -23,6 +23,11 @@ #include "tool_hint.h" #include "common/sha1.h" #include "freeze_history.h" +#include "vision/vision_input.h" + +#ifdef DFLASH_HAVE_MMPROJ +#include "mtmd.h" +#endif #ifdef DFLASH_HAS_CURL #include @@ -737,6 +742,7 @@ json build_props_body(const ServerConfig & config, {"reasoning_supported", reasoning_supported}, {"speculative_supported", speculative_supported}, {"tools_supported", tools_supported}, + {"vision_supported", config.vision_supported}, }}, }; return body; @@ -850,10 +856,18 @@ std::vector normalize_chat_messages( cm.content = m["content"].get(); } else if (m.contains("content") && m["content"].is_array()) { for (const auto & part : m["content"]) { + if (!part.is_object()) continue; std::string ptype = part.value("type", ""); if (ptype == "text" || ptype == "input_text" || ptype == "output_text") { cm.content += part.value("text", ""); + } else if (ptype == "image_url" || ptype == "input_image" || + ptype == "image") { +#ifdef DFLASH_HAVE_MMPROJ + cm.content += mtmd_default_marker(); +#else + cm.content += "<__media__>"; +#endif } } } @@ -1764,6 +1778,22 @@ bool HttpServer::render_and_tokenize_request( } req.started_in_thinking = prompt_ends_in_open_think(rendered); + if (messages_contain_images(req.messages)) { + if (config_.mmproj_path.empty()) { + send_error(fd, 400, + "vision input requires --mmproj (no mmproj model loaded)"); + return false; + } + MultimodalPrompt extracted = extract_multimodal_from_messages(req.messages); + if (extracted.images.empty()) { + send_error(fd, 400, "failed to decode vision input"); + return false; + } + auto mm = std::make_unique(); + mm->marked_text = rendered; + mm->images = std::move(extracted.images); + req.multimodal = std::move(mm); + } req.prompt_tokens = tokenizer_.encode(rendered); return true; } @@ -3095,6 +3125,11 @@ void HttpServer::prepare_generation_inputs( : req.max_output; inputs.request.prompt = prepared.tokens; + if (req.multimodal) { + inputs.request.multimodal = + std::make_unique(*req.multimodal); + inputs.request.force_ar_decode = true; + } inputs.request.n_gen = inputs.generation_cap; inputs.request.sampler = req.sampler; inputs.request.do_sample = req.sampler.needs_logit_processing(); diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index 56f555369..c391a6293 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -181,6 +181,10 @@ struct ServerConfig { std::string chat_template_src; // literal Jinja source (loaded from file) std::string chat_template_path; // path it was loaded from (logged at startup) + // Native mmproj vision + std::string mmproj_path; + bool vision_supported = false; + // Expert frequency tracking (--freq): print frequency analysis at shutdown. bool freq_tracking = false; @@ -228,6 +232,8 @@ struct ParsedRequest { // Bandit: per-session adaptive keep_ratio opt-in std::string session_id; DiskPrefixCachePolicy disk_cache_policy; + // Native mmproj vision payload (populated when messages contain images). + std::unique_ptr multimodal; }; // Parse request sampler fields, applying model-card defaults where present. diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index 9cb7feafc..464195f5d 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -70,6 +70,8 @@ static void print_usage(const char * prog) { "\n" "Options:\n" " --draft Draft model for speculative decode\n" + " --mmproj Multimodal projector (mmproj) for native vision\n" + " --no-mmproj-offload Run mmproj on CPU (default: GPU offload)\n" " --port Listen port (default: 8080)\n" " --host Bind address (default: 0.0.0.0)\n" " --max-ctx Max context length (default: 131072)\n" @@ -220,6 +222,7 @@ int main(int argc, char ** argv) { double spark_vram_gib = 0.0; // --spark-vram: total VRAM target in GiB (0=use card) std::string cache_type_k; // explicit --cache-type-k override std::string cache_type_v; // explicit --cache-type-v override + std::string mmproj_path; // --mmproj bool target_device_seen = false; bool target_devices_seen = false; bool fast_rollback_forced_off = false; @@ -252,6 +255,13 @@ int main(int argc, char ** argv) { for (int i = 2; i < argc; i++) { if (std::strcmp(argv[i], "--draft") == 0 && i + 1 < argc) { bargs.draft_path = argv[++i]; + } else if (std::strcmp(argv[i], "--mmproj") == 0 && i + 1 < argc) { + mmproj_path = argv[++i]; + bargs.mmproj_path = mmproj_path.c_str(); + sconfig.mmproj_path = mmproj_path; + sconfig.vision_supported = true; + } else if (std::strcmp(argv[i], "--no-mmproj-offload") == 0) { + bargs.mmproj_use_gpu = false; } else if (std::strcmp(argv[i], "--port") == 0 && i + 1 < argc) { sconfig.port = std::atoi(argv[++i]); } else if (std::strcmp(argv[i], "--host") == 0 && i + 1 < argc) { @@ -598,6 +608,12 @@ int main(int argc, char ** argv) { } } + if (bargs.mmproj_path && bargs.draft_path) { + std::fprintf(stderr, + "[server] mmproj + draft loaded: DFlash for text-only requests, " + "AR decode for image requests\n"); + } + // Ask the factory to resolve model/placement facts and apply its feature // admission policy before any setup work. server_main only maps the // categorized result to the existing process exit convention. @@ -945,6 +961,8 @@ int main(int argc, char ** argv) { std::fprintf(stderr, "[server] │ port = %d\n", sconfig.port); std::fprintf(stderr, "[server] │ model = %s\n", bargs.model_path); std::fprintf(stderr, "[server] │ draft = %s\n", bargs.draft_path ? bargs.draft_path : "(none)"); + std::fprintf(stderr, "[server] │ mmproj = %s\n", + bargs.mmproj_path ? bargs.mmproj_path : "(none)"); std::fprintf(stderr, "[server] │ model_name = %s\n", sconfig.model_name.c_str()); std::fprintf(stderr, "[server] │ max_ctx = %d\n", sconfig.max_ctx); // max_tokens default for requests that omit the field. The request @@ -1081,6 +1099,7 @@ int main(int argc, char ** argv) { sconfig.fa_window = bargs.fa_window; sconfig.ddtree_budget = bargs.ddtree_budget; sconfig.speculative_enabled = bargs.ddtree_mode; + sconfig.vision_supported = !sconfig.mmproj_path.empty(); sconfig.target_sharding = bargs.device.is_layer_split(); // KV type: report the operator's choice if set, else the family default // the backend resolves (the tq3_0 auto policy was removed; laguna uses diff --git a/server/src/vision/vision_encoder.cpp b/server/src/vision/vision_encoder.cpp new file mode 100644 index 000000000..43d545bd4 --- /dev/null +++ b/server/src/vision/vision_encoder.cpp @@ -0,0 +1,139 @@ +// Vision encoder implementation (mtmd wrapper). + +#include "vision_encoder.h" + +#ifdef DFLASH_HAVE_MMPROJ + +#include "mtmd.h" +#include "mtmd-helper.h" +#include "llama.h" + +#include +#include + +namespace dflash::common { + +VisionEncoder::VisionEncoder() = default; +VisionEncoder::~VisionEncoder() = default; + +bool VisionEncoder::init(const char * model_path, + const char * mmproj_path, + bool use_gpu, + int n_threads) { + if (!model_path || !mmproj_path) { + std::fprintf(stderr, "[vision] model_path and mmproj_path are required\n"); + return false; + } + + llama_backend_init(); + + llama_model_params mparams = llama_model_default_params(); + mparams.n_gpu_layers = 0; // vocab/metadata only + mparams.use_mmap = true; + + llama_model * raw_model = llama_model_load_from_file(model_path, mparams); + if (!raw_model) { + std::fprintf(stderr, "[vision] failed to load llama_model from %s\n", model_path); + return false; + } + model_.reset(raw_model); + + mtmd_context_params cparams = mtmd_context_params_default(); + cparams.use_gpu = use_gpu; + cparams.n_threads = (n_threads > 0) ? n_threads : 4; + cparams.warmup = false; + if (const char * s = std::getenv("IMAGE_MIN_TOKENS")) { + const int v = std::atoi(s); + if (v > 0) cparams.image_min_tokens = v; + } + if (const char * s = std::getenv("IMAGE_MAX_TOKENS")) { + const int v = std::atoi(s); + if (v > 0) cparams.image_max_tokens = v; + } + + mtmd_context * raw_ctx = mtmd_init_from_file(mmproj_path, model_.get(), cparams); + if (!raw_ctx) { + std::fprintf(stderr, "[vision] failed to load mmproj from %s\n", mmproj_path); + model_.reset(); + return false; + } + ctx_.reset(raw_ctx); + + std::fprintf(stderr, "[vision] loaded mmproj %s (gpu=%s threads=%d)\n", + mmproj_path, use_gpu ? "on" : "off", cparams.n_threads); + return true; +} + +mtmd_input_chunks * VisionEncoder::tokenize( + const std::string & marked_text, + const std::vector & images) const { + if (!ctx_) return nullptr; + + std::vector bitmap_wrappers; + bitmap_wrappers.reserve(images.size()); + std::vector bitmap_ptrs; + bitmap_ptrs.reserve(images.size()); + + for (const auto & img : images) { + mtmd_bitmap * bmp = mtmd_helper_bitmap_init_from_buf( + ctx_.get(), img.bytes.data(), img.bytes.size()); + if (!bmp) { + std::fprintf(stderr, "[vision] failed to decode image bytes\n"); + return nullptr; + } + bitmap_wrappers.emplace_back(bmp); + bitmap_ptrs.push_back(bitmap_wrappers.back().ptr.get()); + } + + mtmd_input_chunks * chunks = mtmd_input_chunks_init(); + if (!chunks) return nullptr; + + mtmd_input_text text{}; + text.text = marked_text.c_str(); + text.add_special = false; + text.parse_special = true; + + const int32_t rc = mtmd_tokenize(ctx_.get(), chunks, &text, + bitmap_ptrs.data(), bitmap_ptrs.size()); + if (rc != 0) { + std::fprintf(stderr, "[vision] mtmd_tokenize failed rc=%d\n", rc); + mtmd_input_chunks_free(chunks); + return nullptr; + } + return chunks; +} + +bool VisionEncoder::encode_chunk(const mtmd_input_chunk * chunk) { + if (!ctx_ || !chunk) return false; + const int32_t rc = mtmd_encode_chunk(ctx_.get(), chunk); + if (rc != 0) { + std::fprintf(stderr, "[vision] mtmd_encode_chunk failed rc=%d\n", rc); + return false; + } + return true; +} + +float * VisionEncoder::output_embeddings() const { + return ctx_ ? mtmd_get_output_embd(ctx_.get()) : nullptr; +} + +bool VisionEncoder::uses_mrope() const { + return ctx_ && mtmd_decode_use_mrope(ctx_.get()); +} + +bool VisionEncoder::uses_non_causal(const mtmd_input_chunk * chunk) const { + return ctx_ && mtmd_decode_use_non_causal(ctx_.get(), chunk); +} + +void VisionEncoder::get_image_decoder_pos(const mtmd_image_tokens * image_tokens, + mtmd_decoder_pos * out_pos, + size_t n_tokens) const { + if (!image_tokens || !out_pos) return; + for (size_t i = 0; i < n_tokens; i++) { + out_pos[i] = mtmd_image_tokens_get_decoder_pos(image_tokens, i); + } +} + +} // namespace dflash::common + +#endif // DFLASH_HAVE_MMPROJ diff --git a/server/src/vision/vision_encoder.h b/server/src/vision/vision_encoder.h new file mode 100644 index 000000000..7d0cfdcc6 --- /dev/null +++ b/server/src/vision/vision_encoder.h @@ -0,0 +1,80 @@ +// Vision encoder wrapping mtmd for native mmproj inference. + +#pragma once + +#include "common/vision_types.h" + +#include +#include +#include +#include + +#ifdef DFLASH_HAVE_MMPROJ +#include "llama.h" +#include "mtmd.h" +#endif + +namespace dflash::common { + +#ifdef DFLASH_HAVE_MMPROJ + +class VisionEncoder { +public: + VisionEncoder(); + ~VisionEncoder(); + + VisionEncoder(const VisionEncoder &) = delete; + VisionEncoder & operator=(const VisionEncoder &) = delete; + + // Load mmproj + a CPU-only llama_model (vocab/metadata) for tokenization. + bool init(const char * model_path, + const char * mmproj_path, + bool use_gpu, + int n_threads); + + bool ready() const { return ctx_ != nullptr; } + + // Tokenize marked_text + raw image bytes into mtmd chunks. Caller owns + // the returned pointer and must free with mtmd_input_chunks_free(). + mtmd_input_chunks * tokenize(const std::string & marked_text, + const std::vector & images) const; + + // Run the vision projector for an image/audio chunk. + bool encode_chunk(const mtmd_input_chunk * chunk); + + // Embeddings from the last encode_chunk() call (not owned). + float * output_embeddings() const; + + bool uses_mrope() const; + bool uses_non_causal(const mtmd_input_chunk * chunk) const; + void get_image_decoder_pos(const mtmd_image_tokens * image_tokens, + mtmd_decoder_pos * out_pos, + size_t n_tokens) const; + +private: + struct LlamaModelDeleter { + void operator()(llama_model * m) const { + if (m) llama_model_free(m); + } + }; + struct MtmdContextDeleter { + void operator()(mtmd_context * c) const { + if (c) mtmd_free(c); + } + }; + + std::unique_ptr model_; + std::unique_ptr ctx_; +}; + +#else // !DFLASH_HAVE_MMPROJ + +class VisionEncoder { +public: + bool init(const char *, const char *, bool, int) { return false; } + bool ready() const { return false; } +}; + +#endif // DFLASH_HAVE_MMPROJ + +} // namespace dflash::common diff --git a/server/src/vision/vision_input.cpp b/server/src/vision/vision_input.cpp new file mode 100644 index 000000000..ae05eb519 --- /dev/null +++ b/server/src/vision/vision_input.cpp @@ -0,0 +1,176 @@ +// Vision input parsing implementation. + +#include "vision_input.h" + +#ifdef DFLASH_HAVE_MMPROJ +#include "mtmd.h" +#endif + +#include +#include + +namespace dflash::common { + +namespace { + +static const char * kBase64Chars = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +static bool is_base64(unsigned char c) { + return (std::isalnum(c) != 0) || (c == '+') || (c == '/'); +} + +static void append_image_marker(std::string & out) { +#ifdef DFLASH_HAVE_MMPROJ + out += mtmd_default_marker(); +#else + out += "<__media__>"; +#endif +} + +static bool part_is_image_type(const std::string & type) { + return type == "image_url" || type == "input_image" || type == "image"; +} + +static std::vector decode_image_url_object(const json & image_url) { + if (!image_url.is_object()) { + throw std::runtime_error("image_url must be an object"); + } + const std::string url = image_url.value("url", ""); + if (url.empty()) { + throw std::runtime_error("image_url.url is required"); + } + return decode_image_data_url(url); +} + +static void append_part_text(std::string & out, const json & part) { + const std::string type = part.value("type", ""); + if (type == "text" || type == "input_text" || type == "output_text") { + out += part.value("text", ""); + return; + } + if (part_is_image_type(type)) { + append_image_marker(out); + return; + } +} + +static void process_content_array(MultimodalPrompt & mm, const json & content) { + if (!content.is_array()) return; + for (const auto & part : content) { + if (!part.is_object()) continue; + const std::string type = part.value("type", ""); + if (part_is_image_type(type)) { + DecodedImage img; + if (type == "input_image") { + const std::string source = part.value("source", ""); + if (source == "base64" || part.contains("data")) { + const std::string data = part.value("data", ""); + if (data.empty()) { + throw std::runtime_error("input_image data is empty"); + } + img.bytes = base64_decode(data); + } else if (part.contains("image_url")) { + img.bytes = decode_image_url_object(part["image_url"]); + } else { + throw std::runtime_error("unsupported input_image source"); + } + } else { + img.bytes = decode_image_url_object(part.value("image_url", json::object())); + } + if (img.bytes.empty()) { + throw std::runtime_error("decoded image is empty"); + } + mm.images.push_back(std::move(img)); + append_image_marker(mm.marked_text); + continue; + } + append_part_text(mm.marked_text, part); + } +} + +} // namespace + +std::vector base64_decode(const std::string & encoded_string) { + const int in_len = (int)encoded_string.size(); + int i = 0; + int in_ = 0; + unsigned char char_array_4[4], char_array_3[3]; + std::vector ret; + ret.reserve((size_t)in_len * 3 / 4); + + while (in_len - in_ > 0 && encoded_string[in_] != '=' && is_base64(encoded_string[in_])) { + char_array_4[i++] = (unsigned char)encoded_string[in_++]; + if (i == 4) { + for (i = 0; i < 4; i++) { + const char * p = std::strchr(kBase64Chars, char_array_4[i]); + if (!p) throw std::runtime_error("invalid base64"); + char_array_4[i] = (unsigned char)(p - kBase64Chars); + } + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + for (i = 0; i < 3; i++) ret.push_back(char_array_3[i]); + i = 0; + } + } + + if (i) { + for (int j = i; j < 4; j++) char_array_4[j] = 0; + for (int j = 0; j < 4; j++) { + const char * p = std::strchr(kBase64Chars, char_array_4[j]); + if (!p) throw std::runtime_error("invalid base64"); + char_array_4[j] = (unsigned char)(p - kBase64Chars); + } + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + for (int j = 0; j < i - 1; j++) ret.push_back(char_array_3[j]); + } + return ret; +} + +std::vector decode_image_data_url(const std::string & url) { + const auto comma = url.find(','); + if (comma == std::string::npos) { + throw std::runtime_error("invalid image data URL"); + } + const std::string header = url.substr(0, comma); + if (header.rfind("data:image/", 0) != 0) { + throw std::runtime_error("image url must be data:image/...;base64,..."); + } + if (header.size() < 7 || header.substr(header.size() - 7) != ";base64") { + throw std::runtime_error("image url must be base64 encoded"); + } + return base64_decode(url.substr(comma + 1)); +} + +MultimodalPrompt extract_multimodal_from_messages(const json & messages) { + MultimodalPrompt mm; + if (!messages.is_array()) return mm; + + for (const auto & m : messages) { + if (!m.is_object()) continue; + if (m.contains("content") && m["content"].is_string()) { + mm.marked_text += m["content"].get(); + } else if (m.contains("content") && m["content"].is_array()) { + process_content_array(mm, m["content"]); + } + } + return mm; +} + +bool messages_contain_images(const json & messages) { + if (!messages.is_array()) return false; + for (const auto & m : messages) { + if (!m.is_object() || !m.contains("content") || !m["content"].is_array()) continue; + for (const auto & part : m["content"]) { + if (!part.is_object()) continue; + const std::string type = part.value("type", ""); + if (part_is_image_type(type)) return true; + } + } + return false; +} + +} // namespace dflash::common diff --git a/server/src/vision/vision_input.h b/server/src/vision/vision_input.h new file mode 100644 index 000000000..b31bc755b --- /dev/null +++ b/server/src/vision/vision_input.h @@ -0,0 +1,34 @@ +// Vision input parsing — base64 decode and multimodal extraction from API +// messages. Does not depend on mtmd; safe to compile without DFLASH_HAVE_MMPROJ. + +#pragma once + +#include "common/vision_types.h" + +#include +#include +#include +#include + +namespace dflash::common { + +using json = nlohmann::json; + +// Decode a base64 string into raw bytes. Throws std::runtime_error on +// invalid input. +std::vector base64_decode(const std::string & encoded); + +// Decode a data:image/...;base64,... URL. Throws on unsupported schemes. +std::vector decode_image_data_url(const std::string & url); + +// Walk an OpenAI-style messages array, decode image_url / input_image parts, +// and build a MultimodalPrompt whose marked_text concatenates text parts with +// mtmd_default_marker() inserted for each image (llama-server-common order). +// When DFLASH_HAVE_MMPROJ is unset, markers are still inserted but callers +// must reject image requests without a loaded mmproj. +MultimodalPrompt extract_multimodal_from_messages(const json & messages); + +// Returns true when `messages` contains at least one image part. +bool messages_contain_images(const json & messages); + +} // namespace dflash::common