Experimental Qwen3.6 vision DFlash

.gitattributes

@@ -2,6 +2,7 @@
 *.png filter=lfs diff=lfs merge=lfs -text
 assets/banner.png -filter -diff -merge -text
 assets/docker.png -filter -diff -merge -text
+server/docs/images/vision-mmproj-meme-example.png -filter -diff -merge -text
 *.jpg filter=lfs diff=lfs merge=lfs -text
 *.jpeg filter=lfs diff=lfs merge=lfs -text
 *.mp4 filter=lfs diff=lfs merge=lfs -text

README.md

@@ -24,6 +24,11 @@
 
 ---
 
+> **Experimental fork:** the branch `agent/qwen36-vision-dflash` is based on
+> [upstream PR #571](https://github.com/Luce-Org/lucebox/pull/571) and adds
+> opt-in DFlash decode after Qwen3.6 multimodal prefill. See
+> [the provenance, build instructions, and limitations](docs/experimental-qwen36-vision-dflash.md).
+
 ## Inference Engine Optimizations
 
 Each one is self-contained with setup instructions and benchmark notes.

docs/experimental-qwen36-vision-dflash.md

@@ -0,0 +1,88 @@
+# Experimental Qwen3.6 vision + DFlash
+
+This branch extends native Qwen3.6 `mmproj` support with opt-in DFlash decode
+after multimodal prefill. Autoregressive decoding remains the default.
+
+## Provenance
+
+The native vision implementation comes directly from upstream pull request
+[Luce-Org/lucebox#571](https://github.com/Luce-Org/lucebox/pull/571),
+`feat(vision): native mmproj multimodal chat for Qwen35`, authored by
+[David Roth](https://github.com/davidmroth).
+
+This branch was created from the PR head commit:
+
+- source repository: `davidmroth/lucebox-hub`
+- source branch: `feat/vision-native-mmproj`
+- commit: `122393626cf7bc488e6a27d2ea7069ff15316312`
+- upstream base: `Luce-Org/lucebox:main`
+
+The PR provides `--mmproj`, OpenAI-compatible image input parsing, mtmd image
+encoding, multimodal Qwen35 prefill, and an AR safety fallback for image
+requests. This fork preserves that implementation and adds:
+
+1. opt-in DFlash decode for single-GPU multimodal requests;
+2. correct selection of the final prefill argmax/logits row;
+3. correct logits offset when the final text chunk exceeds the prefill
+   micro-batch size.
+
+## Enabling vision DFlash
+
+Build with native vision enabled:
+
+```bash
+cmake -S server -B server/build \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_CUDA_ARCHITECTURES=86 \
+  -DDFLASH27B_SERVER=ON \
+  -DDFLASH27B_MMPROJ=ON
+cmake --build server/build --target dflash_server -j
+```
+
+PR #571 requires a full `llama.cpp`/mtmd dependency. The tested build used
+`Luce-Org/lucebox-ggml` branch `luce-dflash`, with this branch's patched
+`server/deps/llama.cpp/ggml` subtree overlaid onto it before configuring CMake.
+
+Run the experimental mode:
+
+```bash
+DFLASH_VISION_DFLASH=1 DFLASH27B_PREFILL_UBATCH=256 \
+server/build/dflash_server Qwen3.6-27B-Q4_K_M.gguf \
+  --draft dflash-draft-3.6-q4_k_m.gguf \
+  --mmproj mmproj-Qwen3.6-27B-Q8_0.gguf \
+  --ddtree --ddtree-budget 22 \
+  --chunk 256 --max-ctx 61440 \
+  --cache-type-k q4_0 --cache-type-v q4_0
+```
+
+Without `DFLASH_VISION_DFLASH=1`, image requests use the PR's AR behavior.
+Text-only requests continue to use DFlash normally. Layer-split/multi-GPU
+vision remains AR-only because its committed-position handling is different
+and was not validated by this experiment.
+
+## RTX 3090 notes
+
+The tested target was a 24 GB RTX 3090. A BF16 projector left insufficient
+headroom for a roughly 690 MiB multimodal prefill allocation after a 5K-token
+prompt. The official `ggml-org/Qwen3.6-27B-GGUF`
+`mmproj-Qwen3.6-27B-Q8_0.gguf` reduced projector residency enough to complete
+the same request while keeping GPU vision prefill fast.
+
+The tested llama-swap profile used:
+
+- 61,440-token context;
+- Q4_0 K/V cache;
+- prefill chunk and micro-batch size 256;
+- Q8_0 mmproj on GPU;
+- DDTree budget 22.
+
+## Limitations
+
+This is an experiment, not a claim of production-ready image acceleration.
+Observed image-request acceptance was often only 10-17%, so end-to-end decode
+speed stayed close to AR. A deterministic image test also produced a semantic
+divergence from AR. Use the default AR path when visual fidelity matters and
+enable vision DFlash only for evaluation.
+
+DFlash accelerates text decoding after image encoding. It does not accelerate
+the vision encoder or multimodal prefill.

server/CMakeLists.txt

@@ -203,6 +203,23 @@ endif()
 # Use only the ggml subtree of llama.cpp (skip libllama).
 add_subdirectory(deps/llama.cpp/ggml EXCLUDE_FROM_ALL)
 
+# Optional native mmproj vision (mtmd + llama for tokenization/vocab).
+option(DFLASH27B_MMPROJ "Build native mmproj vision support via mtmd" OFF)
+if(DFLASH27B_MMPROJ)
+    set(LLAMA_BUILD_COMMON    OFF CACHE BOOL "" FORCE)
+    set(LLAMA_BUILD_TESTS     OFF CACHE BOOL "" FORCE)
+    set(LLAMA_BUILD_TOOLS     OFF CACHE BOOL "" FORCE)
+    set(LLAMA_BUILD_EXAMPLES  OFF CACHE BOOL "" FORCE)
+    set(LLAMA_BUILD_SERVER    OFF CACHE BOOL "" FORCE)
+    add_subdirectory(deps/llama.cpp EXCLUDE_FROM_ALL)
+    # mtmd is added as a sibling of llama.cpp (not via tools/) so
+    # LLAMA_INSTALL_VERSION must be visible in the parent scope.
+    if(NOT DEFINED LLAMA_INSTALL_VERSION)
+        set(LLAMA_INSTALL_VERSION "0.0.0")
+    endif()
+    add_subdirectory(deps/llama.cpp/tools/mtmd EXCLUDE_FROM_ALL)
+endif()
+
 if(DFLASH27B_GPU_BACKEND STREQUAL "hip")
     # The vendored ggml HIP shim still uses a few CUDA spellings that are not
     # mapped in this upstream snapshot. Keep the compatibility layer in this
@@ -238,6 +255,7 @@ set(DFLASH27B_SRC_INCLUDE_DIRS
     ${CMAKE_CURRENT_SOURCE_DIR}/src/gemma4
     ${CMAKE_CURRENT_SOURCE_DIR}/src/deepseek4
     ${CMAKE_CURRENT_SOURCE_DIR}/src/server
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/vision
 )
 
 add_library(dflash_common STATIC
@@ -319,6 +337,7 @@ add_library(dflash_common STATIC
     src/qwen35/qwen35_backend.cpp
     src/qwen35/qwen35_tensor_parallel.cpp
     src/qwen35/qwen35_layer_split_adapter.cpp
+    src/qwen35/qwen35_layer_split_vision.cpp
     src/qwen35/qwen35_dflash_target.cpp
     src/qwen35/qwen35_layer_split_dflash_target.cpp
     src/qwen35/layer_split_daemon_loop.cpp
@@ -344,6 +363,7 @@ add_library(dflash_common STATIC
     src/server/prefix_cache.cpp
     src/server/disk_prefix_cache.cpp
     src/server/freeze_history.cpp
+    src/vision/vision_input.cpp
     # ── Jinja chat-template engine (vendored under deps/llama.cpp/common/) ──
     # Used by render_chat_template_jinja() to support --chat-template-file
     # in dflash_server. Mirrors llama.cpp's common_chat_template plumbing.
@@ -357,6 +377,9 @@ add_library(dflash_common STATIC
     deps/llama.cpp/common/jinja/caps.cpp
     deps/llama.cpp/common/unicode.cpp
 )
+if(DFLASH27B_MMPROJ)
+    target_sources(dflash_common PRIVATE src/vision/vision_encoder.cpp)
+endif()
 # BSA (Block-Sparse Attention) backs the speculative-prefill drafter scoring
 # path. Default ON so prefill is fast out of the box. Turn OFF if you don't
 # run the spec-prefill stack or are building without CUDA BF16 WMMA support
@@ -579,6 +602,16 @@ if(DFLASH27B_ENABLE_BSA)
     endif()
 endif()
 
+if(DFLASH27B_MMPROJ)
+    target_compile_definitions(dflash_common PRIVATE DFLASH_HAVE_MMPROJ=1)
+    set(DFLASH27B_MMPROJ_INCLUDE_DIRS
+        ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/include
+        ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/tools/mtmd
+        ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/vendor)
+    target_include_directories(dflash_common PRIVATE ${DFLASH27B_MMPROJ_INCLUDE_DIRS})
+    target_link_libraries(dflash_common PRIVATE llama mtmd)
+endif()
+
 target_include_directories(dflash_common
     PUBLIC
         ${CMAKE_CURRENT_SOURCE_DIR}/include
@@ -1133,6 +1166,10 @@ if(DFLASH27B_TESTS)
         else()
             target_link_libraries(test_server_unit PRIVATE hip::host)
         endif()
+        if(DFLASH27B_MMPROJ)
+            target_compile_definitions(test_server_unit PRIVATE DFLASH_HAVE_MMPROJ=1)
+            target_include_directories(test_server_unit PRIVATE ${DFLASH27B_MMPROJ_INCLUDE_DIRS})
+        endif()
         if(CMAKE_CROSSCOMPILING)
             # Runtime discovery cannot execute a target binary on the build
             # host. Register one aggregate test instead; CMake prepends the
@@ -1312,6 +1349,10 @@ if(DFLASH27B_SERVER)
                 DFLASH27B_BACKEND_CUDA=1
                 DFLASH27B_CUDA_MIN_SM=${_dflash_cuda_min_sm})
         endif()
+        if(DFLASH27B_MMPROJ)
+            target_compile_definitions(dflash_server PRIVATE DFLASH_HAVE_MMPROJ=1)
+            target_include_directories(dflash_server PRIVATE ${DFLASH27B_MMPROJ_INCLUDE_DIRS})
+        endif()
         if(NOT WIN32)
             target_link_libraries(dflash_server PRIVATE dflash_common ggml ${DFLASH27B_GGML_BACKEND_TARGET} pthread)
         else()

server/docs/VISION.md

@@ -0,0 +1,108 @@
+# Native mmproj vision
+
+Optional build (`-DDFLASH27B_MMPROJ=ON`) wires llama.cpp **mtmd** so
+`dflash_server` can load a GGUF multimodal projector alongside the text model
+and accept OpenAI-style `image_url` content in chat completions.
+
+## Quick start
+
+**Need:** CUDA GPU, a Qwen3.5/3.6 GGUF + matching `mmproj-F16.gguf`, and a
+**full** [`lucebox-ggml`](https://github.com/Luce-Org/lucebox-ggml) tree.
+Hub only vendors the ggml subset — `tools/mtmd` is not in-tree — so a stock
+configure with `-DDFLASH27B_MMPROJ=ON` will fail until mtmd sources are present.
+
+```bash
+# 1) Checkout this PR
+git fetch origin pull/571/head:pr-571 && git checkout pr-571
+
+# 2) Supply full llama.cpp (mtmd) for the build
+cd server/deps
+mv llama.cpp llama.cpp.vendored-ggml-only
+git clone --depth 1 -b luce-dflash https://github.com/Luce-Org/lucebox-ggml.git llama.cpp
+# Keep hub-local ggml patches (e.g. rocmfp4) on top of the full tree
+cp -a llama.cpp.vendored-ggml-only/ggml/. llama.cpp/ggml/
+
+# 3) Build with vision
+cd ..
+cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_CUDA_ARCHITECTURES=<your_sm> \
+  -DDFLASH27B_MMPROJ=ON -DDFLASH27B_SERVER=ON
+cmake --build build --target dflash_server -j"$(nproc)"
+
+# 4) Run (same flags you already use for Qwen35, plus mmproj)
+./build/dflash_server \
+  --model /path/to/Qwen….gguf \
+  --mmproj /path/to/mmproj-F16.gguf \
+  # …draft / layer-split / port as usual…
+
+# Container equivalent:
+#   DFLASH_MMPROJ=/path/to/mmproj-F16.gguf
+# Optional: --no-mmproj-offload / DFLASH_MMPROJ_NO_OFFLOAD=1
+```
+
+After the build you can restore the slim vendor so the tree stays pullable:
+
+```bash
+cd server/deps
+rm -rf llama.cpp
+mv llama.cpp.vendored-ggml-only llama.cpp
+```
+
+### Smoke
+
+```bash
+# Capability flag
+curl -s localhost:8080/props | jq '.capabilities.vision_supported'
+# expect: true
+
+# Multimodal chat (data URI only today)
+IMG_B64=$(base64 -w0 /path/to/test.jpg)   # macOS: base64 -i test.jpg
+curl -s localhost:8080/v1/chat/completions \
+  -H 'Content-Type: application/json' \
+  -d "{
+  \"model\": \"qwen\",
+  \"messages\": [{
+    \"role\": \"user\",
+    \"content\": [
+      {\"type\": \"text\", \"text\": \"What do you see?\"},
+      {\"type\": \"image_url\", \"image_url\": {
+        \"url\": \"data:image/jpeg;base64,${IMG_B64}\"
+      }}
+    ]
+  }],
+  \"max_tokens\": 128
+}"
+```
+
+Also check a plain text turn still works (and still uses DFlash when a draft is
+configured). Without `--mmproj`, image requests should 400 cleanly.
+
+## Runtime
+
+Multimodal turns run AR decode; text-only turns keep DFlash speculative decode
+when configured. `/props` reports `capabilities.vision_supported: true` when
+the projector is loaded.
+
+Supported on both monolithic Qwen35 and layer-split backends
+(`supports_multimodal()` is delegated through `LayerSplitBackend`).
+
+## Example
+
+Chat completion with an attached meme image — the model reads the visual
+layout and answers in natural language:
+
+![Native mmproj vision: meme interpretation](images/vision-mmproj-meme-example.png)
+
+Request shape (abbreviated):
+
+```json
+{
+  "messages": [{
+    "role": "user",
+    "content": [
+      {"type": "text", "text": "What do you think this image means?"},
+      {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}
+    ]
+  }]
+}
+```

server/scripts/entrypoint.sh

@@ -315,6 +315,8 @@ fi
 # the KV footprint. Only emitted to the server CLI when nonzero so
 # unset reproduces the server's own default unchanged.
 : "${DFLASH_FA_WINDOW:=0}"
+# Native mmproj vision projector (Qwen3-VL etc.). When set, passed as --mmproj.
+: "${DFLASH_MMPROJ:=}"
 
 # ── auto-detect target ─────────────────────────────────────────────────────
 # Target .gguf is typically 10-30 GB (Q4_K_M). Drafts are 1-2 GB (Q8_0 / Q4)
@@ -499,6 +501,8 @@ CMD=("$DFLASH_SERVER_BIN" "$DFLASH_TARGET"
 
 [ -n "$DRAFT_ARG" ]                && CMD+=(--draft "$DRAFT_ARG")
 [ -n "$DRAFT_ARG" ]                && CMD+=(--ddtree --ddtree-budget "$DFLASH_BUDGET")
+[ -n "$DFLASH_MMPROJ" ]            && CMD+=(--mmproj "$DFLASH_MMPROJ")
+[ "${DFLASH_MMPROJ_NO_OFFLOAD:-0}" = "1" ] && [ -n "$DFLASH_MMPROJ" ] && CMD+=(--no-mmproj-offload)
 [ -n "$DFLASH_DEFAULT_MAX_TOKENS" ] && CMD+=(--default-max-tokens "$DFLASH_DEFAULT_MAX_TOKENS")
 [ -n "$DFLASH_MODEL_NAME" ]         && CMD+=(--model-name "$DFLASH_MODEL_NAME")
 # `--lazy-draft` is silently dropped by the C++ server unless both

server/src/common/attn_masks.h

@@ -75,4 +75,24 @@ inline void build_tree_mask(const DDTree & tree, int past_length,
     }
 }
 
+// Bidirectional mask for vision image chunks (full attention within chunk +
+// to all prior KV).  Matches Qwen35Backend::build_bidirectional_mask.
+inline void build_bidirectional_mask(std::vector<uint16_t> & out,
+                                     int kv_len, int n_tokens, int kv_pos,
+                                     int kq_stride_pad,
+                                     int kv_pad_override = 0) {
+    const int kv_pad = (kv_pad_override > 0) ? kv_pad_override
+                                             : align_up(kv_len, kq_stride_pad);
+    const int q_pad  = align_up(n_tokens, KQ_MASK_PAD);
+    out.assign((size_t)kv_pad * q_pad, F16_NEG_INF);
+    for (int q = 0; q < n_tokens; q++) {
+        for (int k = 0; k < kv_pos; k++) {
+            out[(size_t)q * kv_pad + k] = F16_ZERO;
+        }
+        for (int k = kv_pos; k < kv_pos + n_tokens; k++) {
+            out[(size_t)q * kv_pad + k] = F16_ZERO;
+        }
+    }
+}
+
 }  // namespace dflash::common

server/src/common/backend_args.h

@@ -36,6 +36,10 @@ struct BackendArgs {
     // Optional: speculative decode draft model (qwen35 only)
     const char *    draft_path   = nullptr;
 
+    // Native mmproj vision (qwen35)
+    const char *    mmproj_path     = nullptr;
+    bool            mmproj_use_gpu  = true;
+
     // Device placement
     DevicePlacement device;
     DevicePlacement draft_device;

server/src/common/backend_factory.cpp

@@ -242,6 +242,11 @@ std::unique_ptr<ModelBackend> create_backend(
                 ? std::max<int>(DFLASH27B_DRAFT_BLOCK_SIZE, args.ddtree_budget + 1)
                 : DFLASH27B_DRAFT_BLOCK_SIZE;
             cfg.run_dflash         = args.draft_path != nullptr;
+            cfg.mmproj_path        = args.mmproj_path;
+            cfg.mmproj_use_gpu     = args.mmproj_use_gpu;
+            if (const char * mt = std::getenv("DFLASH_MMPROJ_THREADS")) {
+                cfg.mmproj_threads = std::max(1, std::atoi(mt));
+            }
 
             auto adapter = std::make_unique<Qwen35LayerSplitAdapter>(cfg);
             auto backend = std::make_unique<LayerSplitBackend>(std::move(adapter));
@@ -270,6 +275,8 @@ std::unique_ptr<ModelBackend> create_backend(
         cfg.ddtree_temp        = args.ddtree_temp;
         cfg.ddtree_chain_seed  = args.ddtree_chain_seed;
         cfg.use_feature_mirror = args.use_feature_mirror;
+        cfg.mmproj_path        = args.mmproj_path;
+        cfg.mmproj_use_gpu     = args.mmproj_use_gpu;
 
         auto backend = std::make_unique<Qwen35Backend>(cfg);
         if (!backend->init()) {