From 7ffd5bc06419787eb761d82e89bcfb8b35530bf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Sun, 16 Aug 2026 08:48:53 +0100 Subject: [PATCH 1/2] Load standalone Qwen3.5 MTP GGUFs passed with -md A predictor-only MTP GGUF reports the full block count (n_main + nextn_predict_layers) but only ships the NextN block, so loading one with -md failed: check_tensor_dims: tensor 'blk.0.attn_norm.weight' not found create_qwen35_tensors() and create_qwen35moe_tensors() create every main block as required. Detect the predictor-only case the same way create_step35_tensors() does and mark the absent blocks TENSOR_SKIP|TENSOR_NOT_REQUIRED. Qwen3.5 also has to use the common MTP package contract, otherwise the predictor-only GGUF is never classified as a companion, and the target is not classified TARGET_ONLY - which is what makes it export the hidden states the companion consumes. The remaining two hunks cover cases the above newly reaches: a predictor-only GGUF passed as -m now loads far enough to abort in the graph builder, and its empty main blocks reach split_recurrent_tensors() under -sm graph. --- common/common.cpp | 8 ++++++++ common/speculative.cpp | 3 ++- include/llama.h | 2 ++ src/llama-load-tensors.cpp | 20 ++++++++++++++++++-- src/llama-model.cpp | 12 ++++++++++-- 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index f241371fbf..153927bf34 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -4072,6 +4072,14 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) { return iparams; } + // a predictor-only MTP GGUF has no main blocks, so it cannot be the target model + if (llama_model_mtp_package(model) == LLAMA_MTP_PACKAGE_COMPANION) { + fprintf(stderr, "%s: error: '%s' is an MTP companion, pass it with -md instead\n", + __func__, params.model.c_str()); + llama_free_model(model); + return iparams; + } + auto cparams = common_context_params_to_llama(params); llama_context * lctx = llama_init_from_model(model, cparams); diff --git a/common/speculative.cpp b/common/speculative.cpp index b641f3ef4d..cb9ece044a 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -145,7 +145,8 @@ static bool common_speculative_are_compatible( } static bool common_speculative_target_has_appended_mtp_contract(const llama_model * model) { - return llama_model_is_step35(model) || llama_model_is_deepseek4(model); + return llama_model_is_step35(model) || llama_model_is_deepseek4(model) || + llama_model_is_qwen35_family(model); } static bool common_speculative_has_recognized_mtp_companion( diff --git a/include/llama.h b/include/llama.h index 177f1a7193..6ccb146af7 100644 --- a/include/llama.h +++ b/include/llama.h @@ -713,6 +713,8 @@ extern "C" { LLAMA_API bool llama_model_is_step35(const struct llama_model * model); + LLAMA_API bool llama_model_is_qwen35_family(const struct llama_model * model); + LLAMA_API bool llama_is_gemma4_mtp_file(const char * path); LLAMA_API bool llama_model_is_split_mode_graph(const struct llama_model * model); diff --git a/src/llama-load-tensors.cpp b/src/llama-load-tensors.cpp index 47300df048..917fb08606 100644 --- a/src/llama-load-tensors.cpp +++ b/src/llama-load-tensors.cpp @@ -1704,6 +1704,12 @@ bool create_tensors_helper::create_qwen35moe_tensors(const LLM_TN & tn) { const int64_t value_dim = head_v_dim * n_v_heads; const int64_t conv_dim = key_dim * 2 + value_dim; + // A predictor-only MTP GGUF reports the full block count but only ships the NextN block, skip the rest + const bool mtp_only = hparams.nextn_predict_layers > 0 && + ml.get_tensor_meta(tn(LLM_TENSOR_ATTN_NORM, "weight", 0).c_str()) == nullptr; + const int trunk_flags = mtp_only + ? llama_model_loader::TENSOR_SKIP | llama_model_loader::TENSOR_NOT_REQUIRED : 0; + for (int i = 0; i < n_layer; ++i) { const bool is_mtp_layer = hparams.nextn_predict_layers > 0 && static_cast(i) >= n_layer - hparams.nextn_predict_layers; @@ -1712,7 +1718,7 @@ bool create_tensors_helper::create_qwen35moe_tensors(const LLM_TN & tn) { auto & layer = model.layers[i]; - int flags = 0; + int flags = is_mtp_layer ? 0 : trunk_flags; if (!model.mtp && is_mtp_layer) { flags |= llama_model_loader::TENSOR_SKIP; } @@ -1813,6 +1819,12 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) { const int64_t value_dim = head_v_dim * n_v_heads; const int64_t conv_dim = key_dim * 2 + value_dim; + // A predictor-only MTP GGUF reports the full block count but only ships the NextN block, skip the rest + const bool mtp_only = hparams.nextn_predict_layers > 0 && + ml.get_tensor_meta(tn(LLM_TENSOR_ATTN_NORM, "weight", 0).c_str()) == nullptr; + const int trunk_flags = mtp_only + ? llama_model_loader::TENSOR_SKIP | llama_model_loader::TENSOR_NOT_REQUIRED : 0; + for (int i = 0; i < n_layer; ++i) { auto & layer = model.layers[i]; @@ -1821,7 +1833,7 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) { ggml_context * ctx_split = ctx_for_layer_split(i); - int flags = 0; + int flags = is_mtp_layer ? 0 : trunk_flags; // Skip loading MTP layers if the feature is disabled if (!model.mtp) { if (is_mtp_layer) { @@ -5395,6 +5407,10 @@ bool create_tensors_helper::create_tensors() { for ([[maybe_unused]] auto mem : mem_used) LLAMA_LOG_DEBUG(" %g", mem/1024./1024.); LLAMA_LOG_DEBUG("\n"); auto & layer = model.layers[il]; + // a predictor-only MTP GGUF has no tensors at all in its main blocks + if (!layer.attn_norm && !layer.wq && !layer.wqkv && !layer.ssm_in && !layer.wo) { + continue; + } auto ctx_split = ctx_for_layer_split(il); if (layer.attn_norm) { prepare_split_tensors(-1, ctx_split, layer.attn_norm, layer.split_attn_norm, mirror, mem_used); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 119f0ca390..69c5aa5c3c 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2319,6 +2319,10 @@ bool llama_model_is_step35(const llama_model * model) { return model && model->arch == LLM_ARCH_STEP35; } +bool llama_model_is_qwen35_family(const llama_model * model) { + return model && (model->arch == LLM_ARCH_QWEN35 || model->arch == LLM_ARCH_QWEN35MOE); +} + enum llama_mtp_package llama_model_mtp_package(const llama_model * model) { if (!model) { return LLAMA_MTP_PACKAGE_INVALID; @@ -2331,13 +2335,15 @@ enum llama_mtp_package llama_model_mtp_package(const llama_model * model) { const size_t n_nextn = model->hparams.nextn_predict_layers; const bool has_common_package_contract = llama_model_is_step35(model) || llama_model_is_deepseek4(model) || + llama_model_is_qwen35_family(model) || llama_model_is_gemma4_mtp_assistant(model); if (!has_common_package_contract) { return n_nextn > 0 ? LLAMA_MTP_PACKAGE_EMBEDDED : LLAMA_MTP_PACKAGE_NONE; } if (n_nextn == 0) { - if (llama_model_is_step35(model) || llama_model_is_deepseek4(model)) { + if (llama_model_is_step35(model) || llama_model_is_deepseek4(model) || + llama_model_is_qwen35_family(model)) { for (const auto & layer : model->layers) { if (layer.attn_norm != nullptr) { return LLAMA_MTP_PACKAGE_TARGET_ONLY; @@ -2353,7 +2359,9 @@ enum llama_mtp_package llama_model_mtp_package(const llama_model * model) { } const size_t first = n_layers - n_nextn; - const bool has_tail = model->layers[first].nextn.eh_proj != nullptr; + // Qwen3.5 9B/4B NextN blocks have no eh_proj, but every one of them has enorm + const bool has_tail = model->layers[first].nextn.eh_proj != nullptr || + (llama_model_is_qwen35_family(model) && model->layers[first].nextn.enorm != nullptr); bool has_trunk = false; for (size_t il = 0; il < first; ++il) { From 06c4dfc6d76ba19086a3f44397058e1b2e9f6661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Sun, 16 Aug 2026 23:13:58 +0100 Subject: [PATCH 2/2] Qwen3.5 MTP: require q_proj in predictor-only GGUFs, check companion arch Review follow-up. A dense NextN block loads q_proj as optional because it can be shared with the last main block. A predictor-only GGUF has no main blocks, so one built that way loaded with wq == nullptr and then hung. Require the tensor in that case so the load fails naming it. eh_proj, attn_q and the MLP are all optional on that block, so the tail probe stays on enorm, which is required - the comment there said only eh_proj. Adding Qwen3.5 to the common MTP package contract also made common_speculative_has_recognized_mtp_companion() accept any GGUF classified COMPANION, with no architecture check of the kind the Step and DeepSeek branches have. Add it, plus the predictor count. Dense and MoE are separate architectures, so the comparison is on the arch itself. --- common/speculative.cpp | 14 ++++++++++++++ src/llama-load-tensors.cpp | 4 +++- src/llama-model.cpp | 3 ++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index cb9ece044a..5656d1b963 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2163,6 +2163,20 @@ bool common_speculative_finalize_startup( __func__, n_heads); return false; } + } else if (llama_model_is_qwen35_family(model)) { + // dense and MoE are separate architectures, so compare the arch itself + if (std::strcmp(llama_model_arch_string(model), llama_model_arch_string(companion)) != 0) { + LOG_ERR("%s: Qwen3.5 MTP requires a companion of the same architecture, target is %s and companion is %s\n", + __func__, llama_model_arch_string(model), llama_model_arch_string(companion)); + return false; + } + + const int32_t n_heads = llama_model_n_nextn_layer(companion); + if (n_heads != 1) { + LOG_ERR("%s: Qwen3.5 MTP companion requires exactly one predictor layer, got %d\n", + __func__, n_heads); + return false; + } } if (common_speculative_has_recognized_mtp_companion(model, companion)) { diff --git a/src/llama-load-tensors.cpp b/src/llama-load-tensors.cpp index 917fb08606..c1e2f3b3f8 100644 --- a/src/llama-load-tensors.cpp +++ b/src/llama-load-tensors.cpp @@ -1841,6 +1841,8 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) { } } const int mtp_opt = is_mtp_layer ? llama_model_loader::TENSOR_NOT_REQUIRED : 0; + // q_proj may be shared with the last main block, which a predictor-only GGUF does not have + const int mtp_opt_q = mtp_only ? 0 : mtp_opt; layer.attn_norm = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, flags); layer.attn_post_norm = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), { n_embd }, flags); @@ -1848,7 +1850,7 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) { if (!hparams.is_recurrent(i)) { // Attention layers (MTP layer is always standard attention) - layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head * 2 }, flags | mtp_opt); + layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head * 2 }, flags | mtp_opt_q); layer.wk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, flags); layer.wv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, flags); layer.wo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, flags); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 69c5aa5c3c..378f026b7c 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2359,7 +2359,8 @@ enum llama_mtp_package llama_model_mtp_package(const llama_model * model) { } const size_t first = n_layers - n_nextn; - // Qwen3.5 9B/4B NextN blocks have no eh_proj, but every one of them has enorm + // A Qwen3.5 NextN block loads eh_proj, attn_q and the MLP as optional, so none of them + // marks a predictor tail on its own; enorm is required and is present in all of them. const bool has_tail = model->layers[first].nextn.eh_proj != nullptr || (llama_model_is_qwen35_family(model) && model->layers[first].nextn.enorm != nullptr);