From 97a4774828dbdc0c5de522f6d6d9446cb390ef29 Mon Sep 17 00:00:00 2001 From: yzyyzyhhh <96101183+happyyzy@users.noreply.github.com> Date: Thu, 17 Sep 2026 21:40:54 +0800 Subject: [PATCH 1/6] feat: add Qualcomm Hexagon acceleration path --- .gitmodules | 2 +- ggml | 2 +- src/core/ggml_extend.cpp | 54 ++++---------- src/core/ggml_extend.h | 3 +- src/model/common/block.hpp | 8 +- src/model/common/ggml_block.hpp | 119 +++++++++++++++++++++++++++--- src/model/diffusion/flux.hpp | 98 ++++++++++++++++++++---- src/model/vae/auto_encoder_kl.hpp | 58 +++++++++++---- src/model_io/safetensors_io.cpp | 2 +- 9 files changed, 261 insertions(+), 85 deletions(-) diff --git a/.gitmodules b/.gitmodules index a26210d90..b30cc6e33 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "ggml"] path = ggml - url = https://github.com/leejet/ggml.git + url = https://github.com/happyyzy/ggml.git [submodule "examples/server/frontend"] path = examples/server/frontend url = https://github.com/leejet/sdcpp-webui.git diff --git a/ggml b/ggml index e20c3a14a..3f916ebb5 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit e20c3a14aa70ee84ca58499814206dd08d8026bc +Subproject commit 3f916ebb52dfa5c84276a832d2e3f28f4d2dc595 diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp index 7cf585392..0b64b9af8 100644 --- a/src/core/ggml_extend.cpp +++ b/src/core/ggml_extend.cpp @@ -135,20 +135,9 @@ ggml_tensor* ggml_ext_silu_act(ggml_context* ctx, ggml_tensor* x, bool gate_firs // return: [ne3, ne2, ne1, ne0/2] auto x_vec = ggml_ext_chunk(ctx, x, 2, 0, false); - ggml_tensor* gate; - if (gate_first) { - gate = x_vec[0]; - x = x_vec[1]; - } else { - x = x_vec[0]; - gate = x_vec[1]; - } - gate = ggml_cont(ctx, gate); - gate = ggml_silu_inplace(ctx, gate); - - x = ggml_mul(ctx, x, gate); // [ne3, ne2, ne1, ne0/2] - - return x; + ggml_tensor* gate = gate_first ? x_vec[0] : x_vec[1]; + ggml_tensor* up = gate_first ? x_vec[1] : x_vec[0]; + return ggml_swiglu_split(ctx, gate, up); } ggml_tensor* ggml_ext_group_norm_32(ggml_context* ctx, @@ -247,29 +236,14 @@ ggml_tensor* ggml_ext_linear_i8_tensorwise(ggml_context* ctx, ggml_tensor* b, int convrot_group_size, float scale) { - GGML_ASSERT(x->type == GGML_TYPE_F32 || (x->type == GGML_TYPE_I8 && scale == 1.f)); - if (scale != 1.f) { - x = ggml_ext_scale(ctx, x, scale); - } - - ggml_tensor* fused_bias = scale == 1.f ? b : nullptr; - if (x->ne[2] * x->ne[3] > 1024) { - int64_t ne2 = x->ne[2]; - int64_t ne3 = x->ne[3]; - x = ggml_reshape_2d(ctx, x, x->ne[0], x->ne[1] * x->ne[2] * x->ne[3]); - x = ggml_mul_mat_i8_tensorwise(ctx, w, x, weight_scale, fused_bias, convrot_group_size); - x = ggml_reshape_4d(ctx, x, x->ne[0], x->ne[1] / ne2 / ne3, ne2, ne3); - } else { - x = ggml_mul_mat_i8_tensorwise(ctx, w, x, weight_scale, fused_bias, convrot_group_size); - } - - if (scale != 1.f) { - x = ggml_ext_scale(ctx, x, 1.f / scale); - if (b != nullptr) { - x = ggml_add_inplace(ctx, x, b); - } - } - return x; + GGML_UNUSED(ctx); + GGML_UNUSED(x); + GGML_UNUSED(w); + GGML_UNUSED(weight_scale); + GGML_UNUSED(b); + GGML_UNUSED(convrot_group_size); + GGML_UNUSED(scale); + GGML_ABORT("I8 tensorwise matmul is not available in this ggml revision"); } ggml_tensor* ggml_ext_pad_ext(ggml_context* ctx, @@ -683,14 +657,16 @@ ggml_tensor* ggml_ext_group_norm(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, ggml_tensor* b, - int num_groups) { + int num_groups, + bool inplace) { if (ggml_n_dims(x) >= 3 && w != nullptr && b != nullptr) { w = ggml_reshape_4d(ctx, w, 1, 1, w->ne[0], 1); b = ggml_reshape_4d(ctx, b, 1, 1, b->ne[0], 1); } const float eps = 1e-6f; // default eps parameter - x = ggml_group_norm(ctx, x, num_groups, eps); + x = inplace ? ggml_group_norm_inplace(ctx, x, num_groups, eps) + : ggml_group_norm(ctx, x, num_groups, eps); if (w != nullptr && b != nullptr) { x = ggml_mul_inplace(ctx, x, w); // b = ggml_repeat(ctx, b, x); diff --git a/src/core/ggml_extend.h b/src/core/ggml_extend.h index 585d4582c..da3a3432d 100644 --- a/src/core/ggml_extend.h +++ b/src/core/ggml_extend.h @@ -219,7 +219,8 @@ ggml_tensor* ggml_ext_group_norm(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, ggml_tensor* b, - int num_groups = 32); + int num_groups = 32, + bool inplace = false); ggml_tensor* ggml_ext_timestep_embedding( ggml_context* ctx, diff --git a/src/model/common/block.hpp b/src/model/common/block.hpp index 09db9250a..8f585ba4c 100644 --- a/src/model/common/block.hpp +++ b/src/model/common/block.hpp @@ -56,12 +56,16 @@ class UpSampleBlock : public GGMLBlock { blocks["conv"] = std::shared_ptr(new Conv2d(channels, out_channels, {3, 3}, {1, 1}, {1, 1})); } + bool supports_upscale(GGMLRunnerContext* ctx, ggml_tensor* x) { + auto conv = std::dynamic_pointer_cast(blocks["conv"]); + return conv->supports_upscale(ctx, x, 2); + } + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { // x: [N, channels, h, w] auto conv = std::dynamic_pointer_cast(blocks["conv"]); - x = ggml_upscale(ctx->ggml_ctx, x, 2, GGML_SCALE_MODE_NEAREST); // [N, channels, h*2, w*2] - x = conv->forward(ctx, x); // [N, out_channels, h*2, w*2] + x = conv->forward_upscale(ctx, x, 2); // [N, out_channels, h*2, w*2] return x; } }; diff --git a/src/model/common/ggml_block.hpp b/src/model/common/ggml_block.hpp index 0b3cb8650..5a0c9be97 100644 --- a/src/model/common/ggml_block.hpp +++ b/src/model/common/ggml_block.hpp @@ -207,7 +207,7 @@ class Linear : public UnaryBlock { ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override { ggml_tensor* w = params["weight"]; ggml_tensor* weight_scale = has_weight_scale ? params["weight_scale"] : nullptr; - if (w->type == GGML_TYPE_F8_E4M3 || w->type == GGML_TYPE_F8_E5M2) { + if (w->type == GGML_TYPE_F8_E4M3) { bool supports_fp8_matmul = false; if (ctx->backend != nullptr) { ggml_tensor* fp8_matmul = ggml_mul_mat(ctx->ggml_ctx, w, x); @@ -241,7 +241,7 @@ class Linear : public UnaryBlock { const auto cache_key = std::make_pair(x, int8_convrot_group_size); auto cached = ctx->int8_convrot_cache.find(cache_key); if (cached == ctx->int8_convrot_cache.end()) { - x = ggml_quantize_i8_convrot(ctx->ggml_ctx, x, int8_convrot_group_size); + GGML_ABORT("I8 convrot is not available in this ggml revision"); ctx->int8_convrot_cache.emplace(cache_key, x); } else { x = cached->second; @@ -304,6 +304,30 @@ class Linear : public UnaryBlock { } return out; } + + ggml_tensor* forward_segmented(GGMLRunnerContext* ctx, + ggml_tensor* x0, + ggml_tensor* x1) { + ggml_tensor* w = params["weight"]; + if (ctx->weight_adapter == nullptr && scale == 1.f && + w->type == GGML_TYPE_F8_E4M3) { + ggml_tensor* out = ggml_mul_mat_segmented(ctx->ggml_ctx, w, x0, x1); + if (force_prec_f32) { + ggml_mul_mat_set_prec(out, GGML_PREC_F32); + } + if (ctx->backend != nullptr && ggml_backend_supports_op(ctx->backend, out)) { + if (has_weight_scale) { + out = ggml_mul(ctx->ggml_ctx, out, params["weight_scale"]); + } + if (bias) { + out = ggml_add_inplace(ctx->ggml_ctx, out, params["bias"]); + } + return out; + } + } + + return forward(ctx, ggml_concat(ctx->ggml_ctx, x0, x1, 0)); + } }; __STATIC_INLINE__ bool support_get_rows(ggml_type wtype) { @@ -434,6 +458,16 @@ class Conv2d : public UnaryBlock { forward_params.conv2d.scale = scale; return ctx->weight_adapter->forward_with_lora(ctx->ggml_ctx, ctx->backend, x, w, b, prefix, forward_params); } + if (b != nullptr && ctx->conv2d_direct_enabled && ctx->backend != nullptr && + !ctx->circular_x_enabled && !ctx->circular_y_enabled && scale == 1.f) { + ggml_tensor* out = ggml_conv_2d_direct_bias( + ctx->ggml_ctx, w, x, b, + stride.second, stride.first, padding.second, padding.first, + dilation.second, dilation.first); + if (ggml_backend_supports_op(ctx->backend, out)) { + return out; + } + } return ggml_ext_conv_2d(ctx->ggml_ctx, x, w, @@ -449,6 +483,42 @@ class Conv2d : public UnaryBlock { ctx->circular_y_enabled, scale); } + + ggml_tensor* forward_upscale(GGMLRunnerContext* ctx, + ggml_tensor* x, + int upscale_factor) { + ggml_tensor* out = try_forward_upscale(ctx, x, upscale_factor); + if (out == nullptr) { + return forward(ctx, ggml_upscale(ctx->ggml_ctx, x, upscale_factor, + GGML_SCALE_MODE_NEAREST)); + } + + return out; + } + + bool supports_upscale(GGMLRunnerContext* ctx, + ggml_tensor* x, + int upscale_factor) { + return try_forward_upscale(ctx, x, upscale_factor) != nullptr; + } + +private: + ggml_tensor* try_forward_upscale(GGMLRunnerContext* ctx, + ggml_tensor* x, + int upscale_factor) { + if (!ctx->conv2d_direct_enabled || ctx->backend == nullptr || + ctx->weight_adapter || ctx->circular_x_enabled || + ctx->circular_y_enabled || scale != 1.f) { + return nullptr; + } + + ggml_tensor* out = ggml_conv_2d_direct_upscale( + ctx->ggml_ctx, params["weight"], x, + bias ? params["bias"] : nullptr, upscale_factor, + stride.second, stride.first, padding.second, padding.first, + dilation.second, dilation.first); + return ggml_backend_supports_op(ctx->backend, out) ? out : nullptr; + } }; class Conv2d_grouped : public UnaryBlock { @@ -734,6 +804,23 @@ class GroupNorm : public GGMLBlock { bool affine; std::string prefix; + void get_affine_params(GGMLRunnerContext* ctx, + ggml_tensor** weight, + ggml_tensor** bias) { + *weight = nullptr; + *bias = nullptr; + if (!affine) { + return; + } + + *weight = params["weight"]; + *bias = params["bias"]; + if (ctx->weight_adapter) { + *weight = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, *weight, prefix + "weight"); + *bias = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, *bias, prefix + "bias"); + } + } + void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override { this->prefix = prefix; if (affine) { @@ -754,18 +841,30 @@ class GroupNorm : public GGMLBlock { eps(eps), affine(affine) {} - ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, + bool inplace = false) { ggml_tensor* w = nullptr; ggml_tensor* b = nullptr; - if (affine) { - w = params["weight"]; - b = params["bias"]; - if (ctx->weight_adapter) { - w = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, w, prefix + "weight"); - b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias"); + get_affine_params(ctx, &w, &b); + return ggml_ext_group_norm(ctx->ggml_ctx, x, w, b, num_groups, inplace); + } + + ggml_tensor* forward_silu(GGMLRunnerContext* ctx, ggml_tensor* x, + bool inplace = false) { + ggml_tensor* w = nullptr; + ggml_tensor* b = nullptr; + get_affine_params(ctx, &w, &b); + if (ctx->backend != nullptr && w != nullptr && b != nullptr) { + ggml_tensor* out = inplace + ? ggml_group_norm_affine_silu_inplace(ctx->ggml_ctx, x, w, b, num_groups, eps) + : ggml_group_norm_affine_silu(ctx->ggml_ctx, x, w, b, num_groups, eps); + if (ggml_backend_supports_op(ctx->backend, out)) { + return out; } } - return ggml_ext_group_norm(ctx->ggml_ctx, x, w, b, num_groups); + + x = ggml_ext_group_norm(ctx->ggml_ctx, x, w, b, num_groups, inplace); + return ggml_silu_inplace(ctx->ggml_ctx, x); } }; diff --git a/src/model/diffusion/flux.hpp b/src/model/diffusion/flux.hpp index 67e79d3d9..f5f9e7589 100644 --- a/src/model/diffusion/flux.hpp +++ b/src/model/diffusion/flux.hpp @@ -235,6 +235,18 @@ namespace Flux { x = ggml_mul(ctx->ggml_ctx, x, w); return x; } + + ggml_tensor* forward_rope(GGMLRunnerContext* ctx, + ggml_tensor* x, + ggml_tensor* pe) { + ggml_tensor* out = ggml_qknorm_rope( + ctx->ggml_ctx, x, params["scale"], pe, eps); + if (ctx->backend != nullptr && + ggml_backend_supports_op(ctx->backend, out)) { + return out; + } + return Rope::apply_rope(ctx->ggml_ctx, forward(ctx, x), pe); + } }; struct QKNorm : public GGMLBlock { @@ -261,6 +273,20 @@ namespace Flux { x = norm->forward(ctx, x); return x; } + + ggml_tensor* query_norm_rope(GGMLRunnerContext* ctx, + ggml_tensor* x, + ggml_tensor* pe) { + auto norm = std::dynamic_pointer_cast(blocks["query_norm"]); + return norm->forward_rope(ctx, x, pe); + } + + ggml_tensor* key_norm_rope(GGMLRunnerContext* ctx, + ggml_tensor* x, + ggml_tensor* pe) { + auto norm = std::dynamic_pointer_cast(blocks["key_norm"]); + return norm->forward_rope(ctx, x, pe); + } }; struct SelfAttention : public GGMLBlock { @@ -279,7 +305,9 @@ namespace Flux { blocks["proj"] = std::shared_ptr(new Linear(dim, dim, proj_bias)); } - std::vector pre_attention(GGMLRunnerContext* ctx, ggml_tensor* x) { + std::vector pre_attention(GGMLRunnerContext* ctx, + ggml_tensor* x, + ggml_tensor* pe = nullptr) { auto qkv_proj = std::dynamic_pointer_cast(blocks["qkv"]); auto norm = std::dynamic_pointer_cast(blocks["norm"]); @@ -291,8 +319,13 @@ namespace Flux { qkv->nb[0] * head_dim, qkv->nb[1], qkv->nb[2], (qkv->nb[0]) * qkv->ne[0] / 3); auto v = ggml_view_4d(ctx->ggml_ctx, qkv, head_dim, num_heads, qkv->ne[1], qkv->ne[2], qkv->nb[0] * head_dim, qkv->nb[1], qkv->nb[2], (qkv->nb[0]) * 2 * qkv->ne[0] / 3); - q = norm->query_norm(ctx, q); - k = norm->key_norm(ctx, k); + if (pe != nullptr) { + q = norm->query_norm_rope(ctx, q, pe); + k = norm->key_norm_rope(ctx, k, pe); + } else { + q = norm->query_norm(ctx, q); + k = norm->key_norm(ctx, k); + } return {q, k, v}; } @@ -310,9 +343,17 @@ namespace Flux { // x: [N, n_token, dim] // pe: [n_token, d_head/2, 2, 2] // return [N, n_token, dim] - auto qkv = pre_attention(ctx, x); // q,k,v: [N, n_token, n_head, d_head] - x = Rope::attention(ctx, qkv[0], qkv[1], qkv[2], pe, mask); // [N, n_token, dim] - x = post_attention(ctx, x); // [N, n_token, dim] + auto qkv = pre_attention(ctx, x, pe); + x = ggml_ext_attention_ext(ctx->ggml_ctx, + ctx->backend, + qkv[0], + qkv[1], + qkv[2], + num_heads, + mask, + true, + ctx->flash_attn_enabled); + x = post_attention(ctx, x); return x; } }; @@ -538,10 +579,20 @@ namespace Flux { ModulationOut txt_mod1 = txt_mods[0]; ModulationOut txt_mod2 = txt_mods[1]; + GGML_ASSERT(pe->ne[3] >= txt->ne[1] + img->ne[1]); + auto txt_pe = ggml_view_4d(ctx->ggml_ctx, + pe, + pe->ne[0], pe->ne[1], pe->ne[2], txt->ne[1], + pe->nb[1], pe->nb[2], pe->nb[3], 0); + auto img_pe = ggml_view_4d(ctx->ggml_ctx, + pe, + pe->ne[0], pe->ne[1], pe->ne[2], img->ne[1], + pe->nb[1], pe->nb[2], pe->nb[3], txt->ne[1] * pe->nb[3]); + // prepare image for attention auto img_modulated = img_norm1->forward(ctx, img); img_modulated = Flux::modulate(ctx->ggml_ctx, img_modulated, img_mod1.shift, img_mod1.scale); - auto img_qkv = img_attn->pre_attention(ctx, img_modulated); // q,k,v: [N, n_img_token, n_head, d_head] + auto img_qkv = img_attn->pre_attention(ctx, img_modulated, img_pe); auto img_q = img_qkv[0]; auto img_k = img_qkv[1]; auto img_v = img_qkv[2]; @@ -549,17 +600,25 @@ namespace Flux { // prepare txt for attention auto txt_modulated = txt_norm1->forward(ctx, txt); txt_modulated = Flux::modulate(ctx->ggml_ctx, txt_modulated, txt_mod1.shift, txt_mod1.scale); - auto txt_qkv = txt_attn->pre_attention(ctx, txt_modulated); // q,k,v: [N, n_txt_token, n_head, d_head] + auto txt_qkv = txt_attn->pre_attention(ctx, txt_modulated, txt_pe); auto txt_q = txt_qkv[0]; auto txt_k = txt_qkv[1]; auto txt_v = txt_qkv[2]; // run actual attention - auto q = ggml_concat(ctx->ggml_ctx, txt_q, img_q, 2); // [N, n_txt_token + n_img_token, n_head, d_head] - auto k = ggml_concat(ctx->ggml_ctx, txt_k, img_k, 2); // [N, n_txt_token + n_img_token, n_head, d_head] + auto q = ggml_concat(ctx->ggml_ctx, txt_q, img_q, 1); // [N*n_head, n_txt_token + n_img_token, d_head] + auto k = ggml_concat(ctx->ggml_ctx, txt_k, img_k, 1); // [N*n_head, n_txt_token + n_img_token, d_head] auto v = ggml_concat(ctx->ggml_ctx, txt_v, img_v, 2); // [N, n_txt_token + n_img_token, n_head, d_head] - auto attn = Rope::attention(ctx, q, k, v, pe, mask); // [N, n_txt_token + n_img_token, n_head*d_head] + auto attn = ggml_ext_attention_ext(ctx->ggml_ctx, + ctx->backend, + q, + k, + v, + img_attn->num_heads, + mask, + true, + ctx->flash_attn_enabled); auto txt_attn_out = ggml_view_3d(ctx->ggml_ctx, attn, attn->ne[0], @@ -682,9 +741,17 @@ namespace Flux { auto v = ggml_view_4d(ctx->ggml_ctx, qkv_mlp, head_dim, num_heads, qkv_mlp->ne[1], qkv_mlp->ne[2], qkv_mlp->nb[0] * head_dim, qkv_mlp->nb[1], qkv_mlp->nb[2], (qkv_mlp->nb[0]) * 2 * hidden_size); - q = norm->query_norm(ctx, q); - k = norm->key_norm(ctx, k); - auto attn = Rope::attention(ctx, q, k, v, pe, mask); // [N, n_token, hidden_size] + q = norm->query_norm_rope(ctx, q, pe); + k = norm->key_norm_rope(ctx, k, pe); + auto attn = ggml_ext_attention_ext(ctx->ggml_ctx, + ctx->backend, + q, + k, + v, + num_heads, + mask, + true, + ctx->flash_attn_enabled); // [N, n_token, hidden_size] auto mlp = ggml_view_3d(ctx->ggml_ctx, qkv_mlp, mlp_hidden_dim * mlp_mult_factor, qkv_mlp->ne[1], qkv_mlp->ne[2], qkv_mlp->nb[1], qkv_mlp->nb[2], hidden_size * 3 * qkv_mlp->nb[0]); if (use_yak_mlp) { @@ -694,8 +761,7 @@ namespace Flux { } else { mlp = ggml_ext_gelu(ctx->ggml_ctx, mlp, true); } - auto attn_mlp = ggml_concat(ctx->ggml_ctx, attn, mlp, 0); // [N, n_token, hidden_size + mlp_hidden_dim] - auto output = linear2->forward(ctx, attn_mlp); // [N, n_token, hidden_size] + auto output = linear2->forward_segmented(ctx, attn, mlp); // [N, n_token, hidden_size] output = ggml_add(ctx->ggml_ctx, x, ggml_mul(ctx->ggml_ctx, output, mod.gate)); return output; diff --git a/src/model/vae/auto_encoder_kl.hpp b/src/model/vae/auto_encoder_kl.hpp index 116a83219..390cc48a5 100644 --- a/src/model/vae/auto_encoder_kl.hpp +++ b/src/model/vae/auto_encoder_kl.hpp @@ -39,25 +39,33 @@ class ResnetBlock : public UnaryBlock { auto norm2 = std::dynamic_pointer_cast(blocks["norm2"]); auto conv2 = std::dynamic_pointer_cast(blocks["conv2"]); + const bool inplace_f16 = ctx->conv2d_direct_enabled && x->type == GGML_TYPE_F16; + const bool project_residual = out_channels != in_channels; + ggml_tensor* residual = x; + if (project_residual) { + auto nin_shortcut = std::dynamic_pointer_cast(blocks["nin_shortcut"]); + residual = nin_shortcut->forward(ctx, x); + } + auto h = x; - h = norm1->forward(ctx, h); - h = ggml_silu_inplace(ctx->ggml_ctx, h); // swish + h = norm1->forward_silu(ctx, h, inplace_f16 && project_residual); h = conv1->forward(ctx, h); // return h; - h = norm2->forward(ctx, h); - h = ggml_silu_inplace(ctx->ggml_ctx, h); // swish + h = norm2->forward_silu(ctx, h, inplace_f16); // dropout, skip for inference h = conv2->forward(ctx, h); - // skip connection - if (out_channels != in_channels) { - auto nin_shortcut = std::dynamic_pointer_cast(blocks["nin_shortcut"]); - - x = nin_shortcut->forward(ctx, x); // [N, out_channels, h, w] + if (inplace_f16) { + if (project_residual) { + // Keep the shortcut ahead of the in-place main branch in graph order. + h = ggml_add_inplace(ctx->ggml_ctx, residual, h); + } else { + h = ggml_add_inplace(ctx->ggml_ctx, h, residual); + } + } else { + h = ggml_add(ctx->ggml_ctx, h, residual); } - - h = ggml_add(ctx->ggml_ctx, h, x); return h; // [N, out_channels, h, w] } }; @@ -124,6 +132,9 @@ class AttnBlock : public UnaryBlock { if (use_linear) { h_ = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, h_, 1, 2, 0, 3)); // [N, h, w, in_channels] h_ = ggml_reshape_3d(ctx->ggml_ctx, h_, c, h * w, n); // [N, h * w, in_channels] + if (h_->type == GGML_TYPE_F16) { + h_ = ggml_cast(ctx->ggml_ctx, h_, GGML_TYPE_F32); + } q = q_proj->forward(ctx, h_); // [N, h * w, in_channels] k = k_proj->forward(ctx, h_); // [N, h * w, in_channels] @@ -144,11 +155,18 @@ class AttnBlock : public UnaryBlock { h_ = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, 1, nullptr, false, ctx->flash_attn_enabled); + if (!use_linear && x->type == GGML_TYPE_F16 && h_->type != GGML_TYPE_F16) { + h_ = ggml_cast(ctx->ggml_ctx, h_, GGML_TYPE_F16); + } + if (use_linear) { h_ = proj_out->forward(ctx, h_); // [N, h * w, in_channels] h_ = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, h_, 1, 0, 2, 3)); // [N, in_channels, h * w] h_ = ggml_reshape_4d(ctx->ggml_ctx, h_, w, h, c, n); // [N, in_channels, h, w] + if (x->type == GGML_TYPE_F16 && h_->type != GGML_TYPE_F16) { + h_ = ggml_cast(ctx->ggml_ctx, h_, GGML_TYPE_F16); + } } else { h_ = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, h_, 1, 0, 2, 3)); // [N, in_channels, h * w] h_ = ggml_reshape_4d(ctx->ggml_ctx, h_, w, h, c, n); // [N, in_channels, h, w] @@ -156,7 +174,8 @@ class AttnBlock : public UnaryBlock { h_ = proj_out->forward(ctx, h_); // [N, in_channels, h, w] } - h_ = ggml_add(ctx->ggml_ctx, h_, x); + h_ = x->type == GGML_TYPE_F16 ? ggml_add_inplace(ctx->ggml_ctx, h_, x) + : ggml_add(ctx->ggml_ctx, h_, x); return h_; } }; @@ -456,6 +475,8 @@ class Decoder : public GGMLBlock { auto norm_out = std::dynamic_pointer_cast(blocks["norm_out"]); auto conv_out = std::dynamic_pointer_cast(blocks["conv_out"]); + bool f16_activations = false; + // conv_in auto h = conv_in->forward(ctx, z); // [N, block_in, h, w] // sd::ggml_graph_cut::mark_graph_cut(h, "vae.decoder.prelude", "h"); @@ -482,14 +503,23 @@ class Decoder : public GGMLBlock { std::string name = "up." + std::to_string(i) + ".upsample"; auto up_sample = std::dynamic_pointer_cast(blocks[name]); + if (!f16_activations && h->type == GGML_TYPE_F32) { + ggml_tensor* h_f16 = ggml_cast(ctx->ggml_ctx, h, GGML_TYPE_F16); + if (up_sample->supports_upscale(ctx, h_f16)) { + h = h_f16; + f16_activations = true; + } + } h = up_sample->forward(ctx, h); // sd::ggml_graph_cut::mark_graph_cut(h, "vae.decoder.up." + std::to_string(i) + ".upsample", "h"); } } - h = norm_out->forward(ctx, h); - h = ggml_silu_inplace(ctx->ggml_ctx, h); // nonlinearity/swish + h = norm_out->forward_silu(ctx, h, f16_activations); h = conv_out->forward(ctx, h); // [N, out_ch, h*8, w*8] + if (f16_activations) { + h = ggml_cast(ctx->ggml_ctx, h, GGML_TYPE_F32); + } return h; } }; diff --git a/src/model_io/safetensors_io.cpp b/src/model_io/safetensors_io.cpp index 807c915aa..71cacf5b1 100644 --- a/src/model_io/safetensors_io.cpp +++ b/src/model_io/safetensors_io.cpp @@ -89,7 +89,7 @@ static ggml_type safetensors_dtype_to_ggml_type(const std::string& dtype) { } else if (dtype == "F8_E4M3") { ttype = GGML_TYPE_F8_E4M3; } else if (dtype == "F8_E5M2") { - ttype = GGML_TYPE_F8_E5M2; + ttype = GGML_TYPE_COUNT; } else if (dtype == "I32") { ttype = GGML_TYPE_I32; } else if (dtype == "I64") { From 5cf04dac21181442797966b8f18c335fb23458bf Mon Sep 17 00:00:00 2001 From: yzyyzyhhh <96101183+happyyzy@users.noreply.github.com> Date: Thu, 17 Sep 2026 21:40:54 +0800 Subject: [PATCH 2/6] docs: add Qualcomm branch landing page --- AGENTS.md | 183 -------------------------------- README.md | 307 ++++++++++++++++++++++++------------------------------ 2 files changed, 138 insertions(+), 352 deletions(-) delete mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index f45acb8be..000000000 --- a/AGENTS.md +++ /dev/null @@ -1,183 +0,0 @@ -# Instructions for stable-diffusion.cpp - -This document is for AI coding agents working in this repository. It should -describe agent-specific workflow, repository routing, editing boundaries, and -project-specific pitfalls. - -For general contribution rules, including PR scope, commit conventions, code -style, dependency updates, security hygiene, and AI-assisted contribution policy, -see `CONTRIBUTING.md`. - ---- - -## Agent Operating Rules - -Before analyzing or modifying the repository: - -1. Read this file. -2. Use `rg` / `rg --files` or directory listing commands to confirm the current - tree before relying on a path. -3. Start from `src/` and relevant `docs/` for runtime behavior. -4. Read the relevant code before editing. -5. Prefer the smallest change that fits the existing architecture. -6. Report focused verification and mention any tests not run. - -Agents must not: - -* Run `git push`, create PRs, or submit issue/PR comments on the user's behalf. -* Create commits unless the user explicitly requests that specific commit. -* Modify `ggml/`, `thirdparty/`, or `examples/server/frontend/` unless - explicitly requested and necessary. -* Read large local model files or tokenizer vocabulary files. -* Rewrite unrelated code for style-only reasons. -* Add secrets, model weights, generated binaries, local absolute paths, or - machine-specific output. - -When a change is large, architectural, or likely to affect public behavior, -pause and present a short plan before editing. - ---- - -## Repository Map and Editing Boundaries - -This is a routing map for agents, not a full architecture document. The layout -can change, so verify paths before using them. Do not inspect excluded -large-data directories while checking the tree. - -### Primary Project Code - -Core implementation lives under `src/`. - -Current source layout includes: - -* `src/core/` - shared tensor, ggml integration, backend, graph, RNG, and utility - code. -* `src/model/` - model families and model components. -* `src/model_io/` - model file loading, GGUF, safetensors, pickle, and related - serialization helpers. -* `src/runtime/` - sampling, denoising, guidance, caching, preprocessing, and - runtime execution helpers. -* `src/tokenizers/` - tokenizer implementations. -* `src/conditioning/` - conditioning and prompt-related implementation. -* `src/extensions/` - optional feature extensions. -* top-level `src/*.cpp` and `src/*.h` files - public implementation entry - points, model loading, conversion, versioning, and shared managers. - -`src/tokenizers/vocab/` contains large tokenizer vocabulary data. Do not read or -parse files in this directory; reference the path only when necessary. - -### Public API - -`include/` contains the C API exposed by the project. Currently the primary -public header is `include/stable-diffusion.h`. - -Treat public headers as stable API. Avoid breaking compatibility unless the user -explicitly requests it. If public behavior changes, update relevant examples or -documentation. - -### Examples - -`examples/` contains programs demonstrating library usage. - -* `examples/cli/` - command line program for running models, testing features, - and debugging. -* `examples/common/` - shared example support code. -* `examples/server/` - server application built on top of the library. -* `examples/server/frontend/` - git submodule containing independent frontend - code. Avoid modifying it unless explicitly requested. - -### Documentation and Tooling - -* `docs/` - documentation for supported models, build options, behavior, and - workflows. -* `scripts/` - development, model processing, build automation, formatting, and - tooling scripts. -* `cmake/` - CMake support modules. -* `docker/` - Docker-related project files. -* `assets/` - documentation assets; not runtime code. - -### External, Local, and Generated State - -* `ggml/` - git submodule for the ggml dependency. -* `thirdparty/` - vendored third-party dependencies. -* `models/` - local model storage. Ignore this directory and do not read model - files. -* `test/` - local testing scripts. Use only when relevant to the task. -* `build/`, `build_*`, and similar directories - generated build outputs. - Inspect them only when debugging a build result. - ---- - -## Agent Workflow for Code Changes - -1. Identify the relevant modules under `src/`. -2. Check whether the change touches the public API in `include/`. -3. Consult relevant `docs/` and examples before changing user-facing behavior. -4. Follow existing local patterns before adding new abstractions. -5. Keep edits scoped to the requested behavior. -6. Run the narrowest useful build, test, or inspection command available. - -Follow `CONTRIBUTING.md` for formatting, naming, PR expectations, dependency -update policy, and security rules. - ---- - -## Code Comments - -Keep comments rare and useful. - -Do not add comments that only describe what the code does. Add comments only -when the code cannot fully express the logic, the logic is unusually complex, or -there are historical reasons, invariants, constraints, compatibility concerns, -or known pitfalls that future maintainers need to understand. - -Do not add task-specific comments that will be meaningless after review. - -Examples from the current codebase: - -```cpp -// GOOD: explains a safety constraint that is not obvious from the assignment. -// From src/model_io/pickle_io.cpp. -// Non-tensor checkpoint metadata can use REDUCE for arbitrary -// Python objects. Do not execute it; keep stack shape only. -stack.push_back(make_none_value()); - -// BAD: describes only what the next line does. -// Set the token count to zero. -token_count = 0; -``` - ---- - -## Text File Encoding - -When reading or editing repository text files: - -* Prefer UTF-8 with LF for Markdown, frontend source, JSON, and other text-first - project files unless the file already clearly uses a different encoding. -* Do not assume terminal output encoding matches file encoding on Windows. -* A file that looks garbled in PowerShell output may still be valid UTF-8. -* When inspecting UTF-8 files in PowerShell, prefer explicit UTF-8 reads such as: - * `[Console]::OutputEncoding = [System.Text.Encoding]::UTF8` - * `Get-Content -Encoding utf8 ` -* Avoid rewriting a file purely because console output looked garbled; verify - the actual file encoding first. - ---- - -## Tensor and Layout Notes - -Additional tensor/layout rules for this codebase: - -* `sd::Tensor` shape order is not PyTorch/NumPy-style. `shape()[0]` is the - lowest and most contiguous dimension, and higher indices are higher - dimensions. -* Broadcasting for `sd::Tensor` must align dimensions from low to high dimension - indices. If one tensor has fewer dimensions, append implicit `1`s at the - higher-dimension end. -* `ggml_n_dims` / `ggml_n_dims(tensor)` can drop trailing singleton high - dimensions. Do not assume a logical trailing dimension of `1` will still be - counted in ggml metadata. -* Internal tensor-returning interfaces use an empty `sd::Tensor` to represent - null, absent, or failure states. Do not add `std::optional>` - for internal APIs unless a distinct semantic state is truly required. diff --git a/README.md b/README.md index 6ba81d89a..c061f0d6b 100644 --- a/README.md +++ b/README.md @@ -1,189 +1,158 @@

- + stable-diffusion.cpp

-# stable-diffusion.cpp - -
-leejet%2Fstable-diffusion.cpp | Trendshift -
- -Diffusion model(SD,Flux,Wan,...) inference in pure C/C++ - -***Note that this project is under active development. \ -API and command-line option may change frequently.*** - -## 🔥Important News - -* **2026/08/20** 🚀 stable-diffusion.cpp now supports **LTX-2.5** -* **2026/08/04** 🚀 stable-diffusion.cpp adds **Day-1 support for MiniMax-H3** -* **2026/06/25** 🚀 stable-diffusion.cpp now supports **Krea2** -* **2026/06/04** 🚀 stable-diffusion.cpp now supports **Ideogram4** -* **2026/05/31** 🚀 stable-diffusion.cpp now supports **PiD** -* **2026/05/27** 🚀 stable-diffusion.cpp now supports **Lens** -* **2026/05/17** 🚀 stable-diffusion.cpp now supports **LTX-2.3** -* **2026/04/11** 🚀 stable-diffusion.cpp now uses a brand-new embedded web UI. -* **2026/01/18** 🚀 stable-diffusion.cpp now supports **FLUX.2-klein** -* **2025/12/01** 🚀 stable-diffusion.cpp now supports **Z-Image** -* **2025/11/30** 🚀 stable-diffusion.cpp now supports **FLUX.2-dev** -* **2025/10/13** 🚀 stable-diffusion.cpp now supports **Qwen-Image-Edit / Qwen-Image-Edit 2509** -* **2025/10/12** 🚀 stable-diffusion.cpp now supports **Qwen-Image** -* **2025/09/14** 🚀 stable-diffusion.cpp now supports **Wan2.1 Vace** -* **2025/09/06** 🚀 stable-diffusion.cpp now supports **Wan2.1 / Wan2.2** - -## Features - -- Plain C/C++ implementation based on [ggml](https://github.com/ggml-org/ggml), working in the same way as [llama.cpp](https://github.com/ggml-org/llama.cpp) -- Super lightweight and without external dependencies -- Supported models - - Image Models - - [SD1.x, SD2.x, SD-Turbo](./docs/sd.md) - - [SDXL, SDXL-Turbo](./docs/sd.md) - - [Some SD1.x and SDXL distilled models](./docs/distilled_sd.md) - - [SD3/SD3.5](./docs/sd3.md) - - [FLUX.1-dev/FLUX.1-schnell](./docs/flux.md) - - [FLUX.2-dev/FLUX.2-klein](./docs/flux2.md) - - [Lens](./docs/lens.md) - - [Chroma](./docs/chroma.md) - - [Chroma1-Radiance](./docs/chroma_radiance.md) - - [Qwen Image](./docs/qwen_image.md) - - [PiD](./docs/pid.md) - - [LongCat Image](./docs/longcat_image.md) - - [Z-Image](./docs/z_image.md) - - [MiniT2I](./docs/minit2i.md) - - [Ovis-Image](./docs/ovis_image.md) - - [Anima](./docs/anima.md) - - [ERNIE-Image](./docs/ernie_image.md) - - [Boogu Image](./docs/boogu_image.md) - - [Krea2](./docs/krea2.md) - - [Mage-Flow](./docs/mage_flow.md) - - [SeFi-Image](./docs/sefi_image.md) - - [HiDream-O1-Image](./docs/hidream_o1_image.md) - - [Ideogram4](./docs/ideogram4.md) - - [Image Edit Models](./docs/edit.md) - - [FLUX.1-Kontext-dev](./docs/kontext.md) - - [Qwen Image Edit series](./docs/qwen_image_edit.md) - - [LongCat Image Edit](./docs/longcat_image.md) - - [Boogu Image Edit](./docs/boogu_image.md) - - [Mage-Flow-Edit](./docs/mage_flow.md#image-editing) - - Video Models - - [Wan2.1/Wan2.2](./docs/wan.md) - - [MiniMax-H3](./docs/minimax_h3.md) - - [LTX-2.3/LTX-2.5](./docs/ltx2.md) - - [HunyuanVideo 1.5](./docs/hunyuan_video.md) - - [LingBot-Video](./docs/lingbot_video.md) - - [PhotoMaker](./docs/photo_maker.md) support. - - [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL, including Plus) - - Control Net support with SD 1.5 - - [ADetailer](./docs/adetailer.md) - - LoRA support, same as [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora) - - Latent Consistency Models support (LCM/LCM-LoRA) - - Faster and memory efficient latent decoding with [TAESD](./docs/taesd.md) - - Upscale images generated with [ESRGAN](./docs/esrgan.md) -- Supported backends - - CPU (AVX, AVX2 and AVX512 support for x86 architectures) - - CUDA - - Vulkan - - Metal - - OpenCL - - SYCL -- Supported weight formats - - Pytorch checkpoint (`.ckpt` or `.pth` or `.pt`) - - Safetensors (`.safetensors`) - - GGUF (`.gguf`) -- Convert mode supports converting model weights to `.gguf` or `.safetensors` -- Supported platforms - - Linux - - Mac OS - - Windows - - Android (via Termux, [Local Diffusion](https://github.com/rmatif/Local-Diffusion)) -- Flash Attention for memory usage optimization -- Negative prompt -- [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) style tokenizer (not all the features, only token weighting for now) -- VAE tiling processing for reduce memory usage -- Sampling method - - `Euler A` - - `Euler` - - `Heun` - - `DPM2` - - `DPM++ 2M` - - [`DPM++ 2M v2`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/discussions/8457) - - `DPM++ 2S a` - - `ER-SDE` - - [`LCM`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/13952) -- Cross-platform reproducibility - - `--rng cuda`, default, consistent with the `stable-diffusion-webui GPU RNG` - - `--rng cpu`, consistent with the `comfyui RNG` -- Embedds generation parameters into png output as webui-compatible text string - -## Quick Start - -### Get the sd executable - -- Download pre-built binaries from the [releases page](https://github.com/leejet/stable-diffusion.cpp/releases) -- Or build from source by following the [build guide](./docs/build.md) - -### Download model weights - -- download weights(.ckpt or .safetensors or .gguf). For example - - Stable Diffusion v1.5 from https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5 - - ```sh - curl -L -O https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensors - ``` - -### Generate an image with just one command +# stable-diffusion.cpp for Qualcomm devices + +This branch provides optimized stable-diffusion.cpp and GGML paths for Qualcomm Hexagon NPUs and Adreno GPUs. It is used by [Local Dream](https://github.com/xororz/local-dream) for on-device DiT inference. + +- GGML tracking: [llama.cpp issue #28904](https://github.com/ggml-org/llama.cpp/issues/28904) and [PR #28952](https://github.com/ggml-org/llama.cpp/pull/28952) +- SD.cpp integration: [stable-diffusion.cpp PR #1970](https://github.com/leejet/stable-diffusion.cpp/pull/1970) +- Upstream project: [leejet/stable-diffusion.cpp](https://github.com/leejet/stable-diffusion.cpp) + +## Hexagon NPU + +### Weights + +- Text encoder: export Qwen3-4B as Q4_0 with stable-diffusion.cpp, or use [`llm.gguf`](https://huggingface.co/zhiyuanasad/z_image_turbo_adreno/blob/main/llm.gguf) from [zhiyuanasad/z_image_turbo_adreno](https://huggingface.co/zhiyuanasad/z_image_turbo_adreno). +- Z-Image Turbo FP8: [`z-image-turbo_fp8_scaled_e4m3fn_KJ.safetensors`](https://huggingface.co/Kijai/Z-Image_comfy_fp8_scaled/blob/main/z-image-turbo_fp8_scaled_e4m3fn_KJ.safetensors). +- FLUX.2/Klein 4B FP8: [`flux-2-klein-4b-fp8.safetensors`](https://huggingface.co/black-forest-labs/FLUX.2-klein-4b-fp8/blob/main/flux-2-klein-4b-fp8.safetensors). + +### Performance + +Device: Snapdragon 8 Elite, SM8750, HTP v79. Text encoder, DiT, and VAE run on HTP. Prompt: `a lovely cat`. Sampler: Euler. CFG: 1. Seed: 42. E2E includes text encoding, all sampling steps, and VAE decoding. + +| Model | Resolution | Steps | Upstream Q4_0 DiT | FP8 DiT | VAE | E2E | +|---|---:|---:|---:|---:|---:|---:| +| Z-Image Turbo | 1024x1024 | 8 | **91.03 s/it** | **10.26 s/it** | 2.47 s | 100.54 s | +| FLUX.2/Klein 4B | 1024x1024 | 4 | **79.42 s/it** | **8.54 s/it** | 2.14 s | 49.89 s | +| Z-Image Turbo | 1536x1536 | 8 | **OOM** | **32.91 s/it** | 9.08 s | 306.03 s | +| FLUX.2/Klein 4B | 1536x1536 | 4 | **OOM** | **22.42 s/it** | 5.44 s | 111.68 s | +| Z-Image Turbo | 2048x2048 | 4 | **OOM** | **72.51 s/it** | 14.24 s | 307.07 s | +| FLUX.2/Klein 4B | 2048x2048 | 4 | **OOM** | **46.24 s/it** | 19.28 s | 210.77 s | + +The 1024 and 1536 runs use direct VAE decode. The 2048 runs use 64x64 VAE tiles. At 1K, FP8 is 8.87x faster for Z-Image and 9.30x faster for FLUX.2/Klein than the current upstream Hexagon Q4_0/Q8_0 path. + +### Images + +| Z-Image Turbo | FLUX.2/Klein 4B | +|---|---| +| **1024x1024, 8 steps**
Z-Image 1024x1024, 8 steps | **1024x1024, 4 steps**
FLUX.2 Klein 1024x1024, 4 steps | +| **1536x1536, 8 steps**
Z-Image 1536x1536, 8 steps | **1536x1536, 4 steps**
FLUX.2 Klein 1536x1536, 4 steps | +| **2048x2048, 4 steps**
Z-Image 2048x2048, 4 steps | **2048x2048, 4 steps**
FLUX.2 Klein 2048x2048, 4 steps | + +### Commands + +Place `sd-cli`, `libggml-htp-v79.so`, the model files, and the VAE files in the current directory, then run: ```sh -./bin/sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat" +export LD_LIBRARY_PATH="$PWD" ADSP_LIBRARY_PATH="$PWD" ``` -***For detailed command-line arguments, check out [cli doc](./examples/cli/README.md).*** +#### Z-Image Turbo, 1024x1024, 8 steps -## Performance +```sh +./sd-cli \ + --diffusion-model z-image-turbo_fp8_scaled_e4m3fn_KJ.safetensors \ + --llm llm.gguf \ + --vae ae.safetensors \ + --backend diffusion=HTP0,te=HTP0,vae=HTP0 \ + --fa --vae-conv-direct \ + -t 4 -p "a lovely cat" --cfg-scale 1 \ + --steps 8 --sampling-method euler \ + -W 1024 -H 1024 --seed 42 \ + -o zimage_1024_s8.png +``` -If you want to improve performance or reduce VRAM/RAM usage, please refer to [performance guide](./docs/performance.md). -For runtime and parameter backend placement, see the [backend selection guide](./docs/backend.md). +#### FLUX.2/Klein 4B, 1024x1024, 4 steps -## More Guides +```sh +./sd-cli \ + --diffusion-model flux-2-klein-4b-fp8.safetensors \ + --llm llm.gguf \ + --vae flux2-vae.safetensors \ + --backend diffusion=HTP0,te=HTP0,vae=HTP0 \ + --fa --vae-conv-direct \ + -t 8 -p "a lovely cat" --cfg-scale 1 \ + --steps 4 --sampling-method euler \ + -W 1024 -H 1024 --seed 42 \ + -o klein_1024_s4.png +``` + +#### Z-Image Turbo, 1536x1536, 8 steps + +```sh +./sd-cli \ + --diffusion-model z-image-turbo_fp8_scaled_e4m3fn_KJ.safetensors \ + --llm llm.gguf \ + --vae ae.safetensors \ + --backend diffusion=HTP0,te=HTP0,vae=HTP0 \ + --fa --vae-conv-direct \ + -t 8 -p "a lovely cat" --cfg-scale 1 \ + --steps 8 --sampling-method euler \ + -W 1536 -H 1536 --seed 42 \ + -o zimage_1536_s8.png +``` + +#### FLUX.2/Klein 4B, 1536x1536, 4 steps -- [Backend selection](./docs/backend.md) -- [RPC](./docs/rpc.md) -- [LoRA](./docs/lora.md) -- [LCM/LCM-LoRA](./docs/lcm.md) -- [Docker](./docs/docker.md) -- [Quantization and GGUF](./docs/quantization_and_gguf.md) -- [INT8 convrot safetensors](./docs/int8_convrot.md) -- [Inference acceleration via caching](./docs/caching.md) +```sh +./sd-cli \ + --diffusion-model flux-2-klein-4b-fp8.safetensors \ + --llm llm.gguf \ + --vae flux2-vae.safetensors \ + --backend diffusion=HTP0,te=HTP0,vae=HTP0 \ + --fa --vae-conv-direct \ + -t 8 -p "a lovely cat" --cfg-scale 1 \ + --steps 4 --sampling-method euler \ + -W 1536 -H 1536 --seed 42 \ + -o klein_1536_s4.png +``` -## Bindings +#### Z-Image Turbo, 2048x2048, 4 steps -These projects wrap `stable-diffusion.cpp` for easier use in other languages/frameworks. +```sh +./sd-cli \ + --diffusion-model z-image-turbo_fp8_scaled_e4m3fn_KJ.safetensors \ + --llm llm.gguf \ + --vae ae.safetensors \ + --backend diffusion=HTP0,te=HTP0,vae=HTP0 \ + --params-backend te=disk \ + --fa --vae-conv-direct \ + --vae-tiling --vae-tile-size 64x64 --vae-tile-overlap 0.25 \ + -t 4 -p "a lovely cat" --cfg-scale 1 \ + --steps 4 --sampling-method euler \ + -W 2048 -H 2048 --seed 42 \ + -o zimage_2048_s4.png +``` -* Golang (non-cgo): [seasonjs/stable-diffusion](https://github.com/seasonjs/stable-diffusion) -* Golang (cgo): [Binozo/GoStableDiffusion](https://github.com/Binozo/GoStableDiffusion) -* Golang (non-cgo): [l8bloom/gosd](https://github.com/l8bloom/gosd) -* C#: [DarthAffe/StableDiffusion.NET](https://github.com/DarthAffe/StableDiffusion.NET) -* Python: [william-murray1204/stable-diffusion-cpp-python](https://github.com/william-murray1204/stable-diffusion-cpp-python) -* Rust: [newfla/diffusion-rs](https://github.com/newfla/diffusion-rs) -* Flutter/Dart: [rmatif/Local-Diffusion](https://github.com/rmatif/Local-Diffusion) +#### FLUX.2/Klein 4B, 2048x2048, 4 steps + +```sh +./sd-cli \ + --diffusion-model flux-2-klein-4b-fp8.safetensors \ + --llm llm.gguf \ + --vae flux2-vae.safetensors \ + --backend diffusion=HTP0,te=HTP0,vae=HTP0 \ + --params-backend te=disk \ + --fa --vae-conv-direct \ + --vae-tiling --vae-tile-size 64x64 --vae-tile-overlap 0.25 \ + -t 4 -p "a lovely cat" --cfg-scale 1 \ + --steps 4 --sampling-method euler \ + -W 2048 -H 2048 --seed 42 \ + -o klein_2048_s4.png +``` -## UIs +### FP8 versus upstream Q4_0/Q8_0 -These projects use `stable-diffusion.cpp` as a backend for their image generation. +Resolution: 1024x1024. Sampling steps: 8. -- [GIMP Plugins](https://github.com/themanyone/gimp-plugins) -- [Jellybox](https://jellybox.com) -- [Stable Diffusion GUI](https://github.com/fszontagh/sd.cpp.gui.wx) -- [Stable Diffusion CLI-GUI](https://github.com/piallai/stable-diffusion.cpp) -- [Local Diffusion](https://github.com/rmatif/Local-Diffusion) -- [sd.cpp-webui](https://github.com/daniandtheweb/sd.cpp-webui) -- [LocalAI](https://github.com/mudler/LocalAI) -- [Neural-Pixel](https://github.com/Luiz-Alcantara/Neural-Pixel) -- [KoboldCpp](https://github.com/LostRuins/koboldcpp) +> 雨夜的未来上海外滩，镜头前是一辆旧式有轨电车穿过积水街道，街边霓虹牌同时写着“欢迎光临”“火锅”“Open 24 Hours”，远处玻璃摩天楼与石库门老建筑并列，空中漂浮无人机广告屏，屏幕上有清晰汉字“春风得意”，画面里有穿风衣的人群、红色雨伞、湿漉漉的柏油路反射青蓝与橙红灯光，构图复杂、层次深、电影感、超细节 -## Contributors +| Upstream Q4_0 + Q8_0 | F8_E4M3 | +|---|---| +| Z-Image Q4_0 plus Q8_0 | Z-Image F8_E4M3 | -Thank you to all the people who have already contributed to stable-diffusion.cpp! +## Adreno GPU -[![Contributors](https://contrib.rocks/image?repo=leejet/stable-diffusion.cpp)](https://github.com/leejet/stable-diffusion.cpp/graphs/contributors) +Adreno OpenCL benchmarks, images, and commands will be added here. From 6fe523faafcdb550acf8dd194c1b4d53a092fdba Mon Sep 17 00:00:00 2001 From: yzyyzyhhh <96101183+happyyzy@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:19:29 +0800 Subject: [PATCH 3/6] feat: wire Z-Image QKNorm-RoPE global op --- README.md | 53 ++++++++++++++++++++++++++++++++- ggml | 2 +- src/model/common/ggml_block.hpp | 12 ++++++++ src/model/diffusion/z_image.hpp | 26 +++++++++++++--- 4 files changed, 87 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c061f0d6b..bc1effea2 100644 --- a/README.md +++ b/README.md @@ -155,4 +155,55 @@ Resolution: 1024x1024. Sampling steps: 8. ## Adreno GPU -Adreno OpenCL benchmarks, images, and commands will be added here. +| Model | Size / steps | Before s/it | After s/it | Before sampling (s) | After sampling (s) | +|---|---|---:|---:|---:|---:| +| Klein 4B | 512 / 4 | 11.81 | 5.45 | 52.76 | 25.54 | +| Klein 4B | 1024 / 4 | 37.65 | 21.15 | 154.09 | 90.84 | +| Z-Image Turbo | 512 / 8 | 12.65 | 6.88 | 107.48 | 60.59 | +| Z-Image Turbo | 1024 / 8 | 65.03 | 36.27 | 505.22 | 292.91 | + +### Images + +| Case | Before | After | +|---|---|---| +| Klein 512, 4 steps | ![Klein 512 before](https://raw.githubusercontent.com/happyyzy/ggml/a12c1c4bef30676c421626b23b27e1ffafb98698/adreno-qkv-preprocess-20260905/klein-before-512.png) | ![Klein 512 after](https://raw.githubusercontent.com/happyyzy/ggml/a12c1c4bef30676c421626b23b27e1ffafb98698/adreno-qkv-preprocess-20260905/klein-after-512.png) | +| Klein 1024, 4 steps | ![Klein 1024 before](https://raw.githubusercontent.com/happyyzy/ggml/a12c1c4bef30676c421626b23b27e1ffafb98698/adreno-qkv-preprocess-20260905/klein-before-1024.png) | ![Klein 1024 after](https://raw.githubusercontent.com/happyyzy/ggml/a12c1c4bef30676c421626b23b27e1ffafb98698/adreno-qkv-preprocess-20260905/klein-after-1024.png) | +| Z-Image 512, 8 steps | ![Z-Image 512 before](https://raw.githubusercontent.com/happyyzy/ggml/a12c1c4bef30676c421626b23b27e1ffafb98698/adreno-qkv-preprocess-20260905/zimage-before-512.png) | ![Z-Image 512 after](https://raw.githubusercontent.com/happyyzy/ggml/a12c1c4bef30676c421626b23b27e1ffafb98698/adreno-qkv-preprocess-20260905/zimage-after-512.png) | +| Z-Image 1024, 8 steps | ![Z-Image 1024 before](https://raw.githubusercontent.com/happyyzy/ggml/a12c1c4bef30676c421626b23b27e1ffafb98698/adreno-qkv-preprocess-20260905/zimage-before-1024.png) | ![Z-Image 1024 after](https://raw.githubusercontent.com/happyyzy/ggml/a12c1c4bef30676c421626b23b27e1ffafb98698/adreno-qkv-preprocess-20260905/zimage-after-1024.png) | + +### Model weights + +The GGUF weights can be converted with stable-diffusion.cpp or downloaded directly from [Flux.2 Klein Adreno](https://huggingface.co/zhiyuanasad/flux2_klein_adreno) and [Z-Image Turbo Adreno](https://huggingface.co/zhiyuanasad/z_image_turbo_adreno). + +### Commands + +Build with `GGML_OPENCL_USE_ADRENO_KERNELS=ON`. + +```sh +export GGML_OPENCL_Q4_0_DENSE_DP4A=1 +export GGML_OPENCL_XMEM_SDPA=1 +``` + +#### Klein 512 + +```sh +./sd-cli --diffusion-model models/flux-2-klein-4b-Q4_0.gguf --llm models/qwen_3_4b-Q4_0.gguf --vae models/flux2-vae.safetensors -p 'a lovely cat' --cfg-scale 1 --guidance 3.5 --steps 4 --seed 42 -W 512 -H 512 --diffusion-fa --vae-conv-direct -t 4 -v -o klein_512.png +``` + +#### Klein 1024 + +```sh +./sd-cli --diffusion-model models/flux-2-klein-4b-Q4_0.gguf --llm models/qwen_3_4b-Q4_0.gguf --vae models/flux2-vae.safetensors -p 'a lovely cat' --cfg-scale 1 --guidance 3.5 --steps 4 --seed 42 -W 1024 -H 1024 --diffusion-fa --vae-conv-direct -t 4 -v -o klein_1024.png +``` + +#### Z-Image 512 + +```sh +./sd-cli --diffusion-model models/z_image_turbo-Q4_0-nobf16.gguf --llm models/qwen_3_4b-Q4_0.gguf --vae models/ae_old.safetensors -p 'a lovely cat wearing black sunglasses, studio photo' --cfg-scale 1 --guidance 3.5 --steps 8 --seed 42 -W 512 -H 512 --diffusion-fa --vae-conv-direct -t 4 -v -o zimage_512.png +``` + +#### Z-Image 1024 + +```sh +./sd-cli --diffusion-model models/z_image_turbo-Q4_0-nobf16.gguf --llm models/qwen_3_4b-Q4_0.gguf --vae models/ae_old.safetensors -p 'a lovely cat wearing black sunglasses, studio photo' --cfg-scale 1 --guidance 3.5 --steps 8 --seed 42 -W 1024 -H 1024 --diffusion-fa --vae-conv-direct -t 4 -v -o zimage_1024.png +``` diff --git a/ggml b/ggml index 3f916ebb5..e4999c395 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit 3f916ebb52dfa5c84276a832d2e3f28f4d2dc595 +Subproject commit e4999c39598352b28657306cef56b7bdbcc42624 diff --git a/src/model/common/ggml_block.hpp b/src/model/common/ggml_block.hpp index 5a0c9be97..185a9e10d 100644 --- a/src/model/common/ggml_block.hpp +++ b/src/model/common/ggml_block.hpp @@ -901,6 +901,18 @@ class RMSNorm : public UnaryBlock { x = ggml_mul_inplace(ctx->ggml_ctx, x, w); return x; } + + ggml_tensor* try_forward_rope(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* theta) { + if (ctx->backend == nullptr) { + return nullptr; + } + ggml_tensor* w = params["weight"]; + if (ctx->weight_adapter) { + w = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, w, prefix + "weight"); + } + ggml_tensor* out = ggml_qknorm_rope(ctx->ggml_ctx, x, w, theta, eps); + return ggml_backend_supports_op(ctx->backend, out) ? out : nullptr; + } }; class MultiheadAttention : public GGMLBlock { diff --git a/src/model/diffusion/z_image.hpp b/src/model/diffusion/z_image.hpp index 4ae47e268..698267aaa 100644 --- a/src/model/diffusion/z_image.hpp +++ b/src/model/diffusion/z_image.hpp @@ -194,15 +194,33 @@ namespace ZImage { qkv->nb[3], (num_heads + num_kv_heads) * qkv->nb[1]); // [N, n_token, num_kv_heads, head_dim] + bool qk_norm_rope = false; if (qk_norm) { auto q_norm = std::dynamic_pointer_cast(blocks["q_norm"]); auto k_norm = std::dynamic_pointer_cast(blocks["k_norm"]); - - q = q_norm->forward(ctx, q); - k = k_norm->forward(ctx, k); + ggml_tensor* q_rope = q_norm->try_forward_rope(ctx, q, pe); + ggml_tensor* k_rope = k_norm->try_forward_rope(ctx, k, pe); + if (q_rope != nullptr && k_rope != nullptr) { + x = ggml_ext_attention_ext(ctx->ggml_ctx, + ctx->backend, + q_rope, + k_rope, + v, + num_heads, + mask, + true, + ctx->flash_attn_enabled, + 1.f / 128.f); + qk_norm_rope = true; + } else { + q = q_norm->forward(ctx, q); + k = k_norm->forward(ctx, k); + } } - x = Rope::attention(ctx, q, k, v, pe, mask, 1.f / 128.f); // [N, n_token, num_heads * head_dim] + if (!qk_norm_rope) { + x = Rope::attention(ctx, q, k, v, pe, mask, 1.f / 128.f); // [N, n_token, num_heads * head_dim] + } x = out_proj->forward(ctx, x); // [N, n_token, hidden_size] return x; From 7cb83187bf500cb34d494185c00e45a0d535bcd2 Mon Sep 17 00:00:00 2001 From: yzyyzyhhh <96101183+happyyzy@users.noreply.github.com> Date: Sat, 19 Sep 2026 21:39:09 +0800 Subject: [PATCH 4/6] feat: update Hexagon VAE path and editing example --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ ggml | 2 +- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bc1effea2..cf9475cc3 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,48 @@ Resolution: 1024x1024. Sampling steps: 8. |---|---| | Z-Image Q4_0 plus Q8_0 | Z-Image F8_E4M3 | +### Image editing + +Z-Image Turbo generates the reference image, then FLUX.2/Klein removes the Einstein field equation while preserving the rest of the scene. + +| Z-Image Turbo reference | FLUX.2/Klein edit | +|---|---| +| **1024x1024, 8 steps**
Einstein teaching in front of a blackboard | **1024x1024, 4 steps**
Einstein field equation removed from the blackboard | + +#### Generate the reference with Z-Image Turbo + +```sh +./sd-cli \ + --diffusion-model z-image-turbo_fp8_scaled_e4m3fn_KJ.safetensors \ + --llm llm.gguf \ + --vae ae.safetensors \ + --backend diffusion=HTP0,te=HTP0,vae=HTP0 \ + --fa --vae-conv-direct \ + -t 4 \ + -p "爱因斯坦站在黑板前教学，身前是有SJTU标志的讲台桌，手持粉笔。黑板上清晰写着爱因斯坦场方程：G_μν + Λg_μν = 8πG T_μν；以及麦克斯韦方程微分形式：dF = 0，d*F = *J。写实风格，大学课堂，学术氛围。" \ + --cfg-scale 1 --steps 8 --sampling-method euler \ + -W 1024 -H 1024 --seed 42 \ + -o zimage_einstein_1024_s8.png +``` + +#### Edit with FLUX.2/Klein + +```sh +./sd-cli \ + --diffusion-model flux-2-klein-4b-fp8.safetensors \ + --llm llm.gguf \ + --vae flux2-vae.safetensors \ + --backend diffusion=HTP0,te=HTP0,vae=HTP0 \ + --params-backend te=disk \ + --fa --vae-conv-direct \ + -t 4 \ + -p "删除黑板上的爱因斯坦场方程‘G_μν + Λg_μν = 8πG T_μν’，将该公式擦除干净并自然补全黑板背景。保留麦克斯韦方程‘dF = 0，d*F = *J’、爱因斯坦、带SJTU标志的讲台桌、粉笔、大学课堂和其他画面内容不变，保持写实风格。" \ + --ref-image zimage_einstein_1024_s8.png \ + --cfg-scale 1 --steps 4 --sampling-method euler \ + -W 1024 -H 1024 --seed 42 \ + -o klein_edit_remove_equation_1024_s4.png +``` + ## Adreno GPU | Model | Size / steps | Before s/it | After s/it | Before sampling (s) | After sampling (s) | diff --git a/ggml b/ggml index e4999c395..9a6aee1e3 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit e4999c39598352b28657306cef56b7bdbcc42624 +Subproject commit 9a6aee1e3d4617109b38081063b6695b15d24ce3 From 0d7d2b845e567c4cd138eb2586a55febfddf2388 Mon Sep 17 00:00:00 2001 From: yzyyzyhhh <96101183+happyyzy@users.noreply.github.com> Date: Sun, 20 Sep 2026 17:41:51 +0800 Subject: [PATCH 5/6] fix: support unaligned Hexagon VAE output rows --- README.md | 1 - ggml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index cf9475cc3..dd1d0edf0 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,6 @@ export LD_LIBRARY_PATH="$PWD" ADSP_LIBRARY_PATH="$PWD" --llm llm.gguf \ --vae flux2-vae.safetensors \ --backend diffusion=HTP0,te=HTP0,vae=HTP0 \ - --params-backend te=disk \ --fa --vae-conv-direct \ --vae-tiling --vae-tile-size 64x64 --vae-tile-overlap 0.25 \ -t 4 -p "a lovely cat" --cfg-scale 1 \ diff --git a/ggml b/ggml index 9a6aee1e3..b47b2107f 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit 9a6aee1e3d4617109b38081063b6695b15d24ce3 +Subproject commit b47b2107f369a20aa5a6294856b1b0543e904c22 From 7841da23e0ff65c9bb6587f404b7a99856a6a301 Mon Sep 17 00:00:00 2001 From: yzyyzyhhh <96101183+happyyzy@users.noreply.github.com> Date: Sun, 20 Sep 2026 21:22:42 +0800 Subject: [PATCH 6/6] feat: accelerate Klein 9B on Hexagon --- README.md | 37 ++++++++++++++++++++++++++++++++- ggml | 2 +- src/model/common/ggml_block.hpp | 5 ++++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dd1d0edf0..de35bbb7e 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,13 @@ This branch provides optimized stable-diffusion.cpp and GGML paths for Qualcomm - SD.cpp integration: [stable-diffusion.cpp PR #1970](https://github.com/leejet/stable-diffusion.cpp/pull/1970) - Upstream project: [leejet/stable-diffusion.cpp](https://github.com/leejet/stable-diffusion.cpp) +## Important News + +| Date | Update | +|---|---| +| 2026-09-17 | Added Hexagon NPU support for Z-Image Turbo and FLUX.2/Klein 4B. | +| 2026-09-20 | Added Hexagon NPU support for FLUX.2/Klein 9B Q4_0. | + ## Hexagon NPU ### Weights @@ -41,6 +48,35 @@ The 1024 and 1536 runs use direct VAE decode. The 2048 runs use 64x64 VAE tiles. | **1536x1536, 8 steps**
Z-Image 1536x1536, 8 steps | **1536x1536, 4 steps**
FLUX.2 Klein 1536x1536, 4 steps | | **2048x2048, 4 steps**
Z-Image 2048x2048, 4 steps | **2048x2048, 4 steps**
FLUX.2 Klein 2048x2048, 4 steps | +### FLUX.2/Klein 9B + +Klein 9B uses Q4_0 DiT and Q4_0 Qwen3-8B weights. Text encoder parameters are released after conditioning with `te=disk`; DiT, text encoding, and VAE execution all run on HTP. + +- DiT: [`flux-2-klein-9b-Q4_0.gguf`](https://huggingface.co/leejet/FLUX.2-klein-9B-GGUF/blob/main/flux-2-klein-9b-Q4_0.gguf) +- Text encoder: [`Qwen_Qwen3-8B-Q4_0.gguf`](https://huggingface.co/bartowski/Qwen_Qwen3-8B-GGUF/blob/main/Qwen_Qwen3-8B-Q4_0.gguf) +- VAE: [`flux2-vae.safetensors`](https://huggingface.co/unsloth/FLUX.2-VAE/blob/main/split_files/vae/flux2-vae.safetensors) + +| Resolution | Steps | Warm DiT | VAE decode | E2E | +|---|---:|---:|---:|---:| +| 1024x1024 | 4 | **15.52 s/it** | 2.25 s | **77.76 s** | + +FLUX.2 Klein 9B Q4_0, 1024x1024, 4 steps + +```sh +./sd-cli \ + --diffusion-model flux-2-klein-9b-Q4_0.gguf \ + --llm Qwen3-8B-Q4_0.gguf \ + --vae flux2-vae.safetensors \ + --backend diffusion=HTP0,te=HTP0,vae=HTP0 \ + --params-backend te=disk \ + --fa --vae-conv-direct \ + -t 4 \ + -p 'A cinematic photograph of a red fox standing on a moss-covered stone bridge in an autumn forest, golden morning light, mist between the trees, highly detailed fur, natural colors' \ + --cfg-scale 1 --steps 4 --sampling-method euler \ + -W 1024 -H 1024 --seed 42 \ + -o klein9b_q40_segmented_1024_s4.png +``` + ### Commands Place `sd-cli`, `libggml-htp-v79.so`, the model files, and the VAE files in the current directory, then run: @@ -184,7 +220,6 @@ Z-Image Turbo generates the reference image, then FLUX.2/Klein removes the Einst --llm llm.gguf \ --vae flux2-vae.safetensors \ --backend diffusion=HTP0,te=HTP0,vae=HTP0 \ - --params-backend te=disk \ --fa --vae-conv-direct \ -t 4 \ -p "删除黑板上的爱因斯坦场方程‘G_μν + Λg_μν = 8πG T_μν’，将该公式擦除干净并自然补全黑板背景。保留麦克斯韦方程‘dF = 0，d*F = *J’、爱因斯坦、带SJTU标志的讲台桌、粉笔、大学课堂和其他画面内容不变，保持写实风格。" \ diff --git a/ggml b/ggml index b47b2107f..821382026 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit b47b2107f369a20aa5a6294856b1b0543e904c22 +Subproject commit 821382026d8b015191526865a2667c188f9a1e8b diff --git a/src/model/common/ggml_block.hpp b/src/model/common/ggml_block.hpp index 185a9e10d..b1da635ac 100644 --- a/src/model/common/ggml_block.hpp +++ b/src/model/common/ggml_block.hpp @@ -309,8 +309,11 @@ class Linear : public UnaryBlock { ggml_tensor* x0, ggml_tensor* x1) { ggml_tensor* w = params["weight"]; + const bool segmented_weight = w->type == GGML_TYPE_F8_E4M3 || + w->type == GGML_TYPE_Q4_0 || + w->type == GGML_TYPE_MXFP4; if (ctx->weight_adapter == nullptr && scale == 1.f && - w->type == GGML_TYPE_F8_E4M3) { + segmented_weight) { ggml_tensor* out = ggml_mul_mat_segmented(ctx->ggml_ctx, w, x0, x1); if (force_prec_f32) { ggml_mul_mat_set_prec(out, GGML_PREC_F32);