Gemma4 ov int4 and stateful fixes

.github/workflows/build-openvino.yml

@@ -80,14 +80,14 @@ jobs:
         id: cmake_test_cpu
         run: |
           cd ${{ github.workspace }}
-          ctest --test-dir build/ReleaseOV -L main --verbose --timeout 2000
+          ctest --test-dir build/ReleaseOV -L main --verbose --timeout 4000
 
       - name: Test (GPU)
         id: cmake_test_gpu
         run: |
           cd ${{ github.workspace }}
           export GGML_OPENVINO_DEVICE=GPU
-          ctest --test-dir build/ReleaseOV -L main --verbose --timeout 3000
+          ctest --test-dir build/ReleaseOV -L main --verbose --output-on-failure --timeout 8000
 
   openvino-windows-2022:
     runs-on: windows-2022
@@ -163,4 +163,4 @@ jobs:
           call "%OPENVINO_ROOT%\setupvars.bat"
 
           cd build
-          ctest --test-dir ReleaseOV -L main -C Release --verbose --timeout 3000
+          ctest --test-dir ReleaseOV -L main -C Release --verbose --timeout 4000

ggml/src/ggml-openvino/ggml-decoder.cpp

@@ -115,6 +115,68 @@ bool is_same_shape(const ggml_tensor * a, const ggml_tensor * b) {
 bool is_conv_states_all_tensor(const ggml_tensor * tensor) {
     return tensor != nullptr && strncmp(tensor->name, "conv_states_all", strlen("conv_states_all")) == 0;
 }
+
+// Walk an attention op (FLASH_ATTN_EXT or SOFT_MAX) to the KV-cache leaf tensor it
+// reads. Mirrors the structure matched by get_attention_pattern_case() in
+// compute_llm_params(); kept name-free so SWA/full classification does not depend
+// on tensor names. Returns nullptr if the op is not a recognised attention op.
+const ggml_tensor * get_attention_cache_k_leaf(const ggml_tensor * node) {
+    if (node == nullptr) {
+        return nullptr;
+    }
+    const ggml_tensor * cache_k_permute = nullptr;
+    switch (node->op) {
+    case GGML_OP_FLASH_ATTN_EXT:
+        if (node->src[1] == nullptr) {
+            return nullptr;
+        }
+        // case 0: src[1] is PERMUTE of a VIEW of cache_k
+        if (node->src[1]->op == GGML_OP_PERMUTE && node->src[1]->src[0] != nullptr &&
+            node->src[1]->src[0]->op == GGML_OP_VIEW) {
+            cache_k_permute = node->src[1];
+            // case 1: src[1] is CPY of a PERMUTE of a VIEW of cache_k
+        } else if (node->src[1]->op == GGML_OP_CPY && node->src[1]->src[0] != nullptr &&
+                   node->src[1]->src[0]->op == GGML_OP_PERMUTE && node->src[1]->src[0]->src[0] != nullptr &&
+                   node->src[1]->src[0]->src[0]->op == GGML_OP_VIEW) {
+            cache_k_permute = node->src[1]->src[0];
+        }
+        break;
+    case GGML_OP_SOFT_MAX:
+        if (node->src[0] == nullptr) {
+            return nullptr;
+        }
+        // case 2: src[0] is MUL_MAT whose src[0] is PERMUTE of a VIEW of cache_k
+        if (node->src[0]->op == GGML_OP_MUL_MAT && node->src[0]->src[0] != nullptr &&
+            node->src[0]->src[0]->op == GGML_OP_PERMUTE) {
+            cache_k_permute = node->src[0]->src[0];
+            // case 3: src[0] is ADD whose src[0] is MUL_MAT whose src[0] is PERMUTE
+        } else if (node->src[0]->op == GGML_OP_ADD && node->src[0]->src[0] != nullptr &&
+                   node->src[0]->src[0]->op == GGML_OP_MUL_MAT && node->src[0]->src[0]->src[0] != nullptr &&
+                   node->src[0]->src[0]->src[0]->op == GGML_OP_PERMUTE) {
+            cache_k_permute = node->src[0]->src[0]->src[0];
+        }
+        break;
+    default:
+        return nullptr;
+    }
+    if (cache_k_permute == nullptr) {
+        return nullptr;
+    }
+    const ggml_tensor * cache_k_view = cache_k_permute->src[0];
+    if (cache_k_view == nullptr || cache_k_view->op != GGML_OP_VIEW) {
+        return nullptr;
+    }
+    return cache_k_view->src[0];
+}
+
+// The KV-cache layer index feeding an attention op, or nullopt if not applicable.
+std::optional<int> get_attention_op_kv_layer(const ggml_tensor * node) {
+    const ggml_tensor * cache_k = get_attention_cache_k_leaf(node);
+    if (cache_k == nullptr) {
+        return std::nullopt;
+    }
+    return extract_layer_from_name(cache_k->name);
+}
 }  // namespace
 
 static std::string get_tensor_ov_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) {
@@ -139,8 +201,23 @@ static std::string get_tensor_graph_input_ov_name(const GgmlOvDecoder * decoder,
     if (GgmlOvDecoder::is_inp_emb(tensor, op)) {
         return "embd";
     }
-    if (decoder->is_stateful() && GgmlOvDecoder::is_inp_mask(tensor, op)) {
-        return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa";
+    if (GgmlOvDecoder::is_inp_mask(tensor, op)) {
+        // llama core names both the full-attention mask and the SWA mask
+        // "attn_inp_kq_mask", so they cannot be told apart by name. Classify by the
+        // KV-cache layer the consuming attention op reads: an SWA layer's mask must
+        // be a distinct OV input, otherwise the two masks alias to one parameter.
+        // That aliasing is silently fine while both caches share the same n_kv, but
+        // once the growing full-attention cache is padded past the capped SWA ring
+        // buffer (context depth > 2 * n_swa) the single shared mask no longer
+        // matches the full-attention scores and attention fails to broadcast.
+        const auto layer = get_attention_op_kv_layer(op);
+        const bool is_swa = layer.has_value() && decoder->is_swa_layer(layer.value());
+        if (decoder->is_stateful()) {
+            return is_swa ? "self_kq_mask_swa" : "self_kq_mask";
+        }
+        if (is_swa) {
+            return get_tensor_ov_name(cgraph, tensor) + "_swa";
+        }
     }
     return get_tensor_ov_name(cgraph, tensor);
 }
@@ -455,6 +532,54 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
         return -1;
     };
 
+    // Pre-pass: classify SWA vs full-attention layers structurally, without relying
+    // on tensor names. llama core names every attention mask "attn_inp_kq_mask", so
+    // SWA and full masks are indistinguishable by name. The KV cache, however, is
+    // allocated per layer type: a sliding-window layer uses a small capped ring
+    // buffer while a full-attention layer spans the whole context, so its cache_k
+    // second dim (ne[1], the allocated context length) is strictly smaller. Collect
+    // every attention layer's cache size; the layers whose cache is smaller than the
+    // largest one are the SWA layers. A model with a single uniform cache size has no
+    // SWA/full split to make and yields an empty swa_layers set (the previous
+    // behaviour), so non-SWA models are unaffected.
+    //
+    // IMPORTANT: get_attention_cache_k_leaf() returns the leaf buffer BEHIND the
+    // VIEW (cache_k_view->src[0]), i.e. the tensor allocated once at model load, not
+    // the runtime n_kv VIEW that grows with context depth. So ne[1] here is a fixed
+    // constant per layer (e.g. gemma4: SWA=1024, full=8192) and the SWA<full ordering
+    // holds at EVERY depth, including depth ~1. Reading the growing VIEW instead would
+    // break this: at shallow depth the full-attention view is padded smaller than the
+    // capped SWA ring (inverting the test), and near the ring size the two coincide
+    // (making SWA undetectable). Classifying off the allocation avoids both.
+    {
+        std::vector<std::pair<int, int64_t>> attn_layer_cache_size;  // (layer, cache_k->ne[1])
+        int64_t max_cache_size = 0;
+        for (int i = 0; i < cgraph->n_nodes; i++) {
+            const ggml_tensor * cache_k = get_attention_cache_k_leaf(cgraph->nodes[i]);
+            if (cache_k == nullptr) {
+                continue;
+            }
+            auto layer = extract_layer_from_name(cache_k->name);
+            if (!layer.has_value()) {
+                continue;
+            }
+            attn_layer_cache_size.emplace_back(layer.value(), cache_k->ne[1]);
+            max_cache_size = std::max(max_cache_size, cache_k->ne[1]);
+        }
+        for (const auto & [layer, cache_size] : attn_layer_cache_size) {
+            if (cache_size < max_cache_size) {
+                if (std::find(model_params.swa_layers.begin(), model_params.swa_layers.end(), layer) ==
+                    model_params.swa_layers.end()) {
+                    model_params.swa_layers.push_back(layer);
+                }
+            }
+        }
+    }
+    auto is_swa_layer_local = [&model_params](int layer) {
+        return std::find(model_params.swa_layers.begin(), model_params.swa_layers.end(), layer) !=
+               model_params.swa_layers.end();
+    };
+
     bool rope_seen = false;
     for (int i = 0; i < cgraph->n_nodes; i++) {
         auto * node = cgraph->nodes[i];
@@ -500,11 +625,13 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
             ggml_tensor * cache_k = cache_k_view->src[0];
             int layer = extract_layer_from_name(cache_k->name).value();
 
-            std::string mask_name(mask->name);
+            // SWA vs full-attention is decided structurally in the pre-pass above
+            // (by KV cache size), not by the mask name — llama core gives every
+            // attention mask the same name "attn_inp_kq_mask".
+            const bool is_swa = is_swa_layer_local(layer);
 
             model_params.kv_buffer_ctx_id = ggml_backend_openvino_buffer_get_ctx_id(cache_k->buffer);
-            if (mask_name.find("swa") != std::string::npos) {
-                model_params.swa_layers.push_back(layer);
+            if (is_swa) {
                 model_params.ctx_per_seq_swa = cache_k->ne[1];
             } else {
                 model_params.ctx_per_seq = cache_k->ne[1];
@@ -517,7 +644,7 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
             memcpy(&offset, cache_k_view->op_params, sizeof(size_t));
             compute_params.seq_active_start = offset / seq_size;
 
-            if (mask_name.find("swa") != std::string::npos) {
+            if (is_swa) {
                 compute_params.attention_size_swa = mask->ne[0];
             } else {
                 compute_params.attention_size = mask->ne[0];
@@ -565,7 +692,7 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
         if (node->op == GGML_OP_GATED_DELTA_NET) {
             model_params.state_size = node->src[0]->ne[0];
         }
-        if (node->op == GGML_OP_SCALE && is_kvcache(node->view_src, nullptr)) {
+        if (node->op == GGML_OP_SCALE && node->view_src != nullptr && is_kvcache(node->view_src, nullptr)) {
             compute_params.cache_rs_reset_len = ggml_nelements(node) / node->view_src->ne[0];
             compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->ne[0];
         }
@@ -641,11 +768,17 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
         if (is_stateful()) {
             // Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size]
             // to stateful layout [1, seq, n_heads_kv, head_size].
+            // NOTE: Gemma4 uses per-layer-type head sizes (sliding_attention layers
+            // head_dim=256, full_attention layers global_head_dim=512), so the single
+            // scalar m_model_params.head_size cannot describe every layer. Derive the
+            // head size from THIS tensor's own combined dim instead, so SWA and full
+            // layers each get their correct head_size.
             assert(input_shape.size() == 4 && input_shape[0] == 1 && input_shape[1] == 1 &&
-                   input_shape[2].is_dynamic() &&
-                   input_shape[3] == (m_model_params.n_heads_kv * m_model_params.head_size));
-            input_shape = {input_shape[0], ov::Dimension::dynamic(), m_model_params.n_heads_kv,
-                           m_model_params.head_size};
+                   input_shape[2].is_dynamic() && input_shape[3].is_static() &&
+                   input_shape[3].get_length() % m_model_params.n_heads_kv == 0);
+            const int64_t combined_dim = input_shape[3].get_length();  // n_heads_kv * head_size
+            const int64_t head_size = combined_dim / m_model_params.n_heads_kv;
+            input_shape = {input_shape[0], ov::Dimension::dynamic(), m_model_params.n_heads_kv, head_size};
         }
 
     } else if (is_kv_idx(input, op)) {
@@ -958,6 +1091,39 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
         return weight_node;
     }
 
+    // 3D quantized MoE expert weights [k, m, n_expert]: flatten to a rank-2
+    // [n_expert, m*k] tensor and build the dequant subgraph with use_bias=true (the
+    // exact f16 zero-point form). This is the path hit by test-backend-ops and the
+    // host-buffer load; the backend-buffer path builds the same node in set_tensor.
+    // translate_mul_mat_id gathers experts on axis 0 of this node and splits m*k.
+    if (ggml_is_quantized(tensor->type) && tensor->ne[2] > 1) {
+        GGML_ASSERT(tensor->ne[3] == 1 && "4D quantized expert weights are not supported");
+        GGML_ASSERT(ggml_is_contiguous(tensor) && "expert weights must be contiguous to flatten");
+        const int64_t n_expert = tensor->ne[2];
+        const int64_t m = tensor->ne[1];
+        const int64_t k = tensor->ne[0];
+        ggml_tensor flat_tensor = *tensor;
+        flat_tensor.ne[0] = m * k;
+        flat_tensor.ne[1] = n_expert;
+        flat_tensor.ne[2] = 1;
+        flat_tensor.ne[3] = 1;
+        flat_tensor.nb[1] = ggml_row_size(tensor->type, m * k);
+        flat_tensor.nb[2] = ggml_nbytes(tensor);
+        flat_tensor.nb[3] = ggml_nbytes(tensor);
+        // GGML_OPENVINO_INT4_REQUANT: the down-projection experts are Q5_1/Q8_0
+        // (kept as u8 = 8-bit). Requantize them to symmetric int4 (see
+        // ggml_openvino_get_requant_type). Requant is gated behind use_bias=false; the
+        // symmetric-int4 dequant chain (Constant(i4)->Convert->Multiply) still folds to
+        // GatherMatmulCompressed. gate_up (Q4_K) keeps use_bias=true (already int4).
+        static const bool int4_requant = ggml_openvino_getenv_int("GGML_OPENVINO_INT4_REQUANT") != 0;
+        const bool requant_this =
+            int4_requant && (tensor->type == GGML_TYPE_Q5_1 || tensor->type == GGML_TYPE_Q8_0);
+        OvWeight flat_weight =
+            process_weight_tensor(&flat_tensor, tensor->data, nullptr, /*use_bias=*/!requant_this);
+        flat_weight.weight_node->set_friendly_name(tensor->name);
+        return flat_weight.weight_node;
+    }
+
     // There are three cases where we need to create a new weight node:
     // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor
     // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used
@@ -1673,8 +1839,22 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
         case GGML_OP_DIAG:
         case GGML_OP_TRI:
         case GGML_OP_REPEAT:
+        // Shape-preserving elementwise ops: the dynamic dim is unchanged from src[0].
+        // DIV/CLAMP are used in the MoE routing-weight normalization
+        // (sum_rows -> clamp -> div). If they are left untracked here the dynamic
+        // (token) dim is lost there, the captured prefill token count gets baked into
+        // the downstream reshapes, and every decoder layer after layer 0 turns static
+        // (which then triggers the GPU in-place-concat KV-cache corruption).
+        case GGML_OP_DIV:
+        case GGML_OP_CLAMP:
             m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]];
             break;
+        case GGML_OP_SUM_ROWS:
+            // SUM_ROWS reduces ggml axis 0 to size 1 and preserves all other axes, so the
+            // dynamic dim is preserved unless it was axis 0 (then it is summed away).
+            m_node_dynamic_dims[node] =
+                (m_node_dynamic_dims[node->src[0]] == 0) ? -1 : m_node_dynamic_dims[node->src[0]];
+            break;
         case GGML_OP_MUL_MAT_ID:
         case GGML_OP_SOLVE_TRI:
             m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]];