From 31acb3fdd5b269ddc42cf3087054e4f28881e52b Mon Sep 17 00:00:00 2001 From: xindongliu594 Date: Thu, 3 Sep 2026 10:26:08 +0800 Subject: [PATCH 1/5] feat: add reusable GGUF Route B support for Qwen3.5 --- GGUF_ROUTE_B_QWEN38.md | 664 ++++++++++++++++ csrc/config/quant_config.cpp | 5 + csrc/engine/rank_worker.cpp | 70 ++ csrc/engine/rank_worker.hpp | 4 + .../layers/causal_lm_templates/text_model.hpp | 45 +- csrc/layers/linear/base_linear.cpp | 47 +- csrc/layers/linear/base_linear.hpp | 26 +- csrc/layers/linear/fused_linear.cpp | 127 ++- csrc/layers/linear/fused_linear.hpp | 26 +- csrc/layers/linear/linear.cpp | 13 +- csrc/layers/linear/linear.hpp | 9 +- csrc/layers/mlp/mlp.cpp | 8 +- csrc/layers/mlp/mlp.hpp | 5 +- .../layers/quantization/base_quantization.hpp | 61 ++ csrc/layers/quantization/fp8.cpp | 184 +++++ csrc/layers/quantization/fp8.hpp | 44 ++ csrc/layers/quantization/gguf.cpp | 606 +++++++++++++++ csrc/layers/quantization/gguf.hpp | 157 ++++ csrc/layers/quantization/quantization.hpp | 2 + .../quantization/quantization_scheme.hpp | 3 + csrc/models/qwen3_5/qwen3_5_attention.cpp | 62 +- csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp | 132 +++- csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp | 42 +- csrc/models/qwen3_5/qwen3_5_for_causal_lm.hpp | 1 + .../qwen3_5/qwen3_5_fused_qkv_linear.cpp | 51 +- .../qwen3_5/qwen3_5_fused_qkv_linear.hpp | 8 +- .../qwen3_next/qwen3_next_gated_deltanet.cpp | 53 +- csrc/pybind11/engine/engine.hpp | 5 + python/infinilm/infer_engine.py | 20 +- python/infinilm/llm/llm.py | 6 + .../infinilm/llm/model_runner/model_runner.py | 6 + python/infinilm/modeling_utils.py | 73 +- scripts/gguf_mapping.py | 424 ++++++++++ scripts/gguf_routeb_audit.py | 555 +++++++++++++ scripts/gguf_routeb_blocks_probe.cpp | 82 ++ scripts/gguf_routeb_blocks_probe.cu | 125 +++ scripts/gguf_routeb_blocks_ref.py | 616 +++++++++++++++ scripts/gguf_routeb_compare.py | 91 +++ scripts/gguf_routeb_env.sh | 21 + scripts/gguf_routeb_first_diff.py | 189 +++++ scripts/gguf_routeb_first_diff_batch.py | 161 ++++ scripts/gguf_routeb_gemv_check.py | 293 +++++++ scripts/gguf_routeb_gemv_probe.cu | 158 ++++ scripts/gguf_routeb_head_precision.py | 95 +++ scripts/gguf_routeb_infinilm_ref.py | 114 +++ scripts/gguf_routeb_infinilm_trace.py | 227 ++++++ scripts/gguf_routeb_llama_probe.py | 50 ++ scripts/gguf_routeb_llama_ref.py | 114 +++ scripts/gguf_routeb_llama_trace.py | 61 ++ scripts/gguf_routeb_probe_params.py | 70 ++ scripts/gguf_routeb_prompts.jsonl | 32 + scripts/gguf_routeb_shape_contract.py | 257 ++++++ scripts/gguf_routeb_stage2_check.py | 297 +++++++ scripts/gguf_routeb_stage3_check.py | 299 +++++++ scripts/gguf_routeb_tokenizer_check.py | 124 +++ scripts/gguf_routeb_typecensus.py | 55 ++ scripts/gguf_to_infinilm.py | 733 ++++++++++++++++++ scripts/gguf_transforms.py | 121 +++ 58 files changed, 7834 insertions(+), 95 deletions(-) create mode 100644 GGUF_ROUTE_B_QWEN38.md create mode 100644 csrc/layers/quantization/fp8.cpp create mode 100644 csrc/layers/quantization/fp8.hpp create mode 100644 csrc/layers/quantization/gguf.cpp create mode 100644 csrc/layers/quantization/gguf.hpp create mode 100644 scripts/gguf_mapping.py create mode 100644 scripts/gguf_routeb_audit.py create mode 100644 scripts/gguf_routeb_blocks_probe.cpp create mode 100644 scripts/gguf_routeb_blocks_probe.cu create mode 100644 scripts/gguf_routeb_blocks_ref.py create mode 100755 scripts/gguf_routeb_compare.py create mode 100644 scripts/gguf_routeb_env.sh create mode 100755 scripts/gguf_routeb_first_diff.py create mode 100644 scripts/gguf_routeb_first_diff_batch.py create mode 100644 scripts/gguf_routeb_gemv_check.py create mode 100644 scripts/gguf_routeb_gemv_probe.cu create mode 100644 scripts/gguf_routeb_head_precision.py create mode 100755 scripts/gguf_routeb_infinilm_ref.py create mode 100644 scripts/gguf_routeb_infinilm_trace.py create mode 100644 scripts/gguf_routeb_llama_probe.py create mode 100755 scripts/gguf_routeb_llama_ref.py create mode 100644 scripts/gguf_routeb_llama_trace.py create mode 100644 scripts/gguf_routeb_probe_params.py create mode 100644 scripts/gguf_routeb_prompts.jsonl create mode 100644 scripts/gguf_routeb_shape_contract.py create mode 100644 scripts/gguf_routeb_stage2_check.py create mode 100644 scripts/gguf_routeb_stage3_check.py create mode 100755 scripts/gguf_routeb_tokenizer_check.py create mode 100644 scripts/gguf_routeb_typecensus.py create mode 100644 scripts/gguf_to_infinilm.py create mode 100644 scripts/gguf_transforms.py diff --git a/GGUF_ROUTE_B_QWEN38.md b/GGUF_ROUTE_B_QWEN38.md new file mode 100644 index 000000000..f48929d0c --- /dev/null +++ b/GGUF_ROUTE_B_QWEN38.md @@ -0,0 +1,664 @@ +# InfiniLM 适配 Qwen3.8-27B GGUF 技术报告 + +> 路线：GGUF 原生块量化（Route B） +> 目标模型：Qwen3.8-27B-UD-Q6_K +> 目标框架：InfiniLM + InfiniCore +> 文档日期：2026-09-03 +> 状态：核心适配已完成并可运行；严格逐 token 一致性优化仍有可选提升空间 + +## 1. 摘要 + +本工作完成了 Qwen3.8-27B GGUF 模型到 InfiniLM 的原生块量化适配。这里的“原生”是指： + +- GGUF 中的 Q8_0、Q4_K、Q5_K、Q6_K 权重块不先完整反量化为 BF16； +- 打包时直接保留 GGUF block bytes，并以 `torch.uint8` 张量写入 safetensors； +- 推理时由新增的 `linear_gguf` 算子在 GPU kernel 内按块解码并参与矩阵乘； +- 小 batch/decode 使用寄存器 GEMV，大 batch/prefill 使用分块解码加 GEMM； +- 暂不支持原生执行的少量权重在打包阶段显式转为 BF16，不允许静默回退。 + +最终产物能够完成全量 27B 模型加载、prefill、逐 token decode 和确定性生成。打包模型包含 +947 个张量、6 个 safetensors 分片，总权重体积 23.264 GiB；其中 491 个张量保持 GGUF +block bytes，456 个张量为 BF16。 + +功能层面，GGUF 适配已经跑通。以 llama.cpp 为参照进行 32 个样例、每例 32 个 token 的严格 +比较，当前接受的严格基线达到 **27/32 个样例完全一致、920/1024 个 token 一致**。这个指标 +衡量的是两个不同推理后端的逐 token 数值复现程度，不等同于模型能否正确运行。原计划中的 +`>=29/32` 属于额外的严格一致性优化目标，目前尚未达到，也不是 GGUF 适配可用性的必要条件。 + +## 2. 背景、目标与非目标 + +### 2.1 输入和输出 + +源模型： + +```text +/home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf +``` + +当前正式打包产物： + +```text +/home/liuxd/models/Qwen3.8-27B-GGUF-native-v2 +``` + +源 GGUF 文件约 21.97 GB。打包后的 v1/native-v2 产物采用 InfiniLM 可装载的 safetensors +目录结构，同时在 `config.json` 中保存每个权重的 GGML 类型和 GGUF 量化配置。 + +### 2.2 主要目标 + +1. 在 InfiniLM 中加载并运行 Qwen3.8-27B GGUF 模型。 +2. 尽可能保留 GGUF 原生量化块，避免将全部权重展开为 BF16。 +3. 支持 Q8_0、Q4_K、Q5_K、Q6_K 四种主要 GGUF block 类型。 +4. 同时覆盖 prompt prefill 和 autoregressive decode。 +5. 建立可复用的 GGUF 打包、类型路由、算子和验证框架，以便后续适配其他模型。 +6. 用独立门禁证明没有错映射、错 shape、错字节布局或静默稠密回退。 + +### 2.3 当前非目标 + +- 不要求 InfiniLM 与 llama.cpp 在所有输入上逐 bit 或逐 token 完全相同； +- 不在本阶段实现所有 IQ 系列 GGUF 量化格式； +- 不在本阶段实现 GGUF blob 的 tensor parallel 切分； +- 不把实验性 Q8 激活量化或局部 F32 路径默认启用； +- 不以 llama.cpp 的速度数据替代 InfiniLM 自身性能测试。 + +## 3. 模型特点与适配难点 + +Qwen3.8-27B 不是只包含标准全注意力层的简单 Transformer。模型共有 64 个 decoder layer， +其中包含全注意力层和 Gated DeltaNet/线性注意力层，并维护额外的 GDN/SSM state。适配难点主要 +来自以下方面： + +1. GGUF 张量命名与 InfiniLM 参数命名不一致；部分 GGUF 融合权重需要拆成多个运行时权重。 +2. 不同 GGUF 类型具有不同 block size 和字节布局，U8 张量的第二维不是逻辑输入维度。 +3. GGUF 某些二维权重的物理取向与运行时线性层的逻辑取向不同。 +4. 线性注意力 `out_proj` 的 V head 布局需要额外的 grouped-to-tiled 置换。 +5. GGUF 与原始模型在 RMSNorm gain 表达约定上存在差异，错误处理会造成系统性数值偏差。 +6. prefill 的 M 较大，不能只实现单 token GEMV；同时又不能把完整权重永久展开为 BF16。 +7. BF16 舍入、归约顺序、采样语义会使接近的 logits 在两个后端产生不同 top-1 token。 + +## 4. 总体技术路线 + +完整数据流如下： + +```text +Qwen3.8-27B GGUF + | + v +gguf_mapping.py 生成唯一映射计划 + | + v +gguf_to_infinilm.py + |-- 支持类型：原始 block bytes -> U8 weight_bytes + |-- 例外类型：显式解码 -> BF16 weight + |-- 写入 ggml_types / quantization_config + | + v +InfiniLM safetensors 目录（6 shards） + | + v +GGUFBlockQuantization 按完整权重键解析类型和 shard + | + v +BaseLinear / Qwen3.5 模型层调用 linear_gguf + | + +-- decode：寄存器内 block decode + GEMV + | + +-- prefill：tile decode 到 workspace + GEMM + v +BF16/F32 hidden -> 后续 attention、GDN、MLP、norm、lm_head +``` + +这条路线的关键原则是：量化格式信息从打包到运行时始终显式存在；若类型、shape 或映射不满足 +约束，程序直接报错，而不是悄悄改走稠密权重。 + +## 5. GGUF 打包与权重映射 + +### 5.1 单一映射事实源 + +`/home/liuxd/InfiniLM/scripts/gguf_mapping.py` 是张量映射的单一事实源。每个映射条目描述： + +- InfiniLM 参数名； +- GGUF tensor 名； +- 逻辑 shape； +- 是否保存为 blob； +- 支持的 GGML 类型； +- 融合张量的 slice 范围； +- 是否执行转置或 V head 置换； +- checkpoint 键和类型表键。 + +打包器、shape contract、内存预算和 C++ 运行时检查都基于同一映射计划，避免 Python 打包 +规则和 C++ 加载规则分别维护后逐渐漂移。 + +### 5.2 支持的原生量化类型 + +| GGML 类型 | 类型 ID | 典型 block | 当前处理方式 | +|---|---:|---|---| +| Q8_0 | 8 | 32 个权重 / 34 B | 原生 U8 blob + GPU 解码 | +| Q4_K | 12 | K-quant block | 原生 U8 blob + GPU 解码 | +| Q5_K | 13 | K-quant block | 原生 U8 blob + GPU 解码 | +| Q6_K | 14 | 256 个权重 / 210 B | 原生 U8 blob + GPU 解码 | + +例如： + +- Q6_K，`K=5120` 时每行 `5120 / 256 * 210 = 4200 B`； +- Q6_K，`K=6144` 时每行 5040 B； +- Q6_K，`K=10240` 时每行 8400 B； +- Q6_K，`K=17408` 时每行 14280 B； +- Q8_0，`K=5120` 时每行 `5120 / 32 * 34 = 5440 B`。 + +因此 blob 的物理 shape 是 `[N, row_bytes]`，而不是普通线性权重的 `[N, K]`。运行时从 +descriptor 中同时获得逻辑 K、GGML type 和 row bytes，并检查三者是否一致。 + +### 5.3 BF16 例外路径 + +当前模型中不属于四种原生类型的少量 IQ4_XS/IQ4_NL 权重在打包阶段显式解码为 BF16。 +embedding 和 lm_head 在当前 v1 也采用 BF16 例外路径，以降低首次集成的复杂度。例外是映射 +计划的一部分，不是运行时静默 fallback。 + +后续若补充 embedding gather-dequant 和量化 lm_head，可预计再节省约 2.51 GiB 权重显存。 + +### 5.4 融合权重、取向与置换 + +模型包含 GGUF 融合张量到多个 InfiniLM 参数的拆分。947 个产物条目多于 851 个实际消费 +GGUF 权重，主要来自 48 个 GDN 层的融合 `attn_qkv` 一分为三。打包器按映射表定义的 slice +拆分，不能仅依靠名称替换。 + +对于线性注意力输出投影，还需要对 V head 执行 grouped-to-tiled 置换。当前实现按 +`16 x 3 x 128` 的语义布局转换，使 GGUF 权重布局与 InfiniLM GDN 计算布局一致。 + +### 5.5 RMSNorm 约定修正 + +排查中发现 GGUF/RMSNorm gain 与原模型权重的表达约定不同。若把已经 baked `+1` 的 norm +权重再次加 1，会产生明显的逐层漂移。打包器新增 `_is_baked_plus1_norm()`，对所有 norm +权重统一判断，并排除不应套用该规则的张量。 + +### 5.6 打包器 + +主要脚本： + +```text +/home/liuxd/InfiniLM/scripts/gguf_to_infinilm.py +/home/liuxd/InfiniLM/scripts/gguf_mapping.py +/home/liuxd/InfiniLM/scripts/gguf_transforms.py +``` + +打包器完成以下工作： + +1. 读取 GGUF metadata 和 tensor directory； +2. 根据模型维度生成完整映射计划； +3. 对原生支持类型逐行复制 block bytes； +4. 对明确列出的例外解码为 BF16； +5. 执行 slice、转置、V permutation 和 norm convention 修正； +6. 写入带 index 的 safetensors shards； +7. 在 `config.json` 写入 `quantization_config` 和 947 项 `ggml_types`； +8. 复制 tokenizer/config 所需文件； +9. 对 shape、dtype、字节数和抽样原始 bytes 做自检。 + +脚本支持 `--dry-run` 和 `--skip-pack`，可以在不重复生成 23 GiB 产物的情况下审计映射或复用 +已有分片。 + +### 5.7 最终产物组成 + +| 项目 | 数量/大小 | +|---|---:| +| safetensors 分片 | 6 | +| 总张量数 | 947 | +| 原生 U8 blob | 491 | +| BF16 张量 | 456 | +| U8 blob 体积 | 17.648 GiB | +| BF16 体积 | 5.615 GiB | +| 合计 | 23.264 GiB | + +`ggml_types` 的类型直方图为：`dense_bf16=456`、`Q6_K=304`、`Q5_K=124`、 +`Q8_0=59`、`Q4_K=4`。 + +## 6. InfiniLM 模型和量化框架接线 + +### 6.1 GGUFBlockQuantization + +新增： + +```text +/home/liuxd/InfiniLM/csrc/layers/quantization/gguf.hpp +/home/liuxd/InfiniLM/csrc/layers/quantization/gguf.cpp +``` + +`GGUFBlockQuantization` 的职责包括： + +- 从模型配置读取 `ggml_types`； +- 按完整 checkpoint tensor key 查找具体 GGML 类型； +- 区分 `.weight_bytes` blob 和 BF16 `.weight`； +- 处理 fused shard 对应关系； +- 对需要的 shard 应用 V permutation 语义； +- 创建并调用 InfiniCore `linear_gguf_`； +- 对未知类型、缺失类型、shape 不符和不支持的 tensor parallel 显式报错。 + +### 6.2 Linear 层传递完整权重身份 + +普通量化框架只知道当前 Linear 的逻辑维度，但 GGUF 路由还必须知道它对应哪个 checkpoint +tensor。为此扩展了 `BaseLinear` 及相关线性层，使其保存 checkpoint stem 或 `shard_stems_`。 +量化对象据此解析每个 fused shard 的类型，而不是按当前 C++ 对象名进行模糊匹配。 + +### 6.3 Qwen3.5/Qwen3.8 模型结构 + +完成了 Qwen3.5 风格模型在配置、注册、权重映射和运行时模块上的接入，包括： + +- decoder layer； +- full attention； +- Gated DeltaNet/linear attention； +- MLP； +- GDN/SSM cache state； +- final norm 和 causal LM 输出； +- tokenizer/chat template 相关配置。 + +模型结构适配与 GGUF block 算子相互独立：前者决定“哪些张量放到哪里”，后者决定“某个 +量化 Linear 怎样执行”。这种拆分是后续复用到其他架构的基础。 + +### 6.4 `ignore_eos` 语义修正 + +严格比较时发现，llama.cpp 的 `ignore_eos=true` 是在采样前屏蔽 EOS，而 InfiniLM 原有的 +`stop_on_eos=false` 仅表示采到 EOS 后不停止，并不会阻止 EOS 被选中。这是采样语义差异， +不是算子误差。 + +为此扩展： + +- C++ RankWorker Input 的 `suppressed_token_ids`； +- pybind 和 InferEngine 的字段传递； +- 低层 `GenerationConfig.ignore_eos`； +- 高层 SamplingParams 到每请求屏蔽列表的转换。 + +修正后 `ctx_03` 从第 27 token 分叉变为 32/32 完全一致，同时保留其他 stopping criteria。 + +## 7. 新增和扩展的算子 + +### 7.1 算子总表 + +| 算子/模块 | 类型 | 作用 | 默认状态 | +|---|---|---|---| +| `linear_gguf` | 新增 | 直接消费 GGUF U8 block 权重 | 启用 | +| GGML block decoders | 新增 | 解码 Q8_0/Q4_K/Q5_K/Q6_K | 启用 | +| register GGUF GEMV | 新增 | 小 M 的 decode/small-prefill | 启用 | +| tile dequant + GEMM | 新增 | 大 M prefill | 启用 | +| mixed add-RMSNorm | 扩展 | 承接实验性 F32 hidden 边界 | 普通路径不触发 | +| mixed GEMM fallback | 扩展 | BF16 权重乘 F32 hidden | 仅实验路径触发 | +| Q8A activation path | 实验新增 | Q8 激活量化后与 GGUF 权重计算 | 默认关闭 | +| F32 GGUF output | 实验扩展 | 指定 Linear 保留 F32 输出 | 默认关闭 | + +### 7.2 `linear_gguf` 完整注册链 + +新增文件： + +```text +/home/liuxd/InfiniCore/include/infiniop/ops/linear_gguf.h +/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/linear_gguf.h +/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/info.h +/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/operator.cc +/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/nvidia/linear_gguf_nvidia.cuh +/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/nvidia/linear_gguf_nvidia.cu +/home/liuxd/InfiniCore/include/infinicore/ops/linear_gguf.hpp +/home/liuxd/InfiniCore/src/infinicore/ops/linear_gguf/linear_gguf.cc +/home/liuxd/InfiniCore/src/infinicore/ops/linear_gguf/linear_gguf_infiniop.cc +``` + +这条链覆盖 C API descriptor、InfiniCore C++ dispatcher、workspace 计算、设备 dispatch、 +plan/run/cleanup。`info.h` 是 shape、dtype、GGML type 和 row bytes 契约的集中校验点。 + +### 7.3 GGML block 解码器 + +文件： + +```text +/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/ggml_blocks.h +``` + +这里实现 Q8_0、Q4_K、Q5_K、Q6_K 的共享 host/device block decoder。decoder 按 GGUF 的 +原始位布局读取 scale、高位掩码和量化值。因为某些 row stride 只保证 2 字节对齐，不能假设 +每个 block 都满足 4/16 字节对齐；实现使用安全的 byte load/拷贝方式，避免未对齐访问错误。 + +### 7.4 小 M 寄存器 GEMV + +文件： + +```text +/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/nvidia/linear_gguf_gemv.cuh +``` + +执行方式： + +1. 一个 warp 负责一个输出行； +2. warp lanes 沿 K 方向处理多个 GGUF block； +3. block 权重在寄存器中即时解码； +4. 与输入激活做 FP32 累加； +5. warp reduction 得到输出； +6. 正式路径把结果写为 BF16。 + +kernel 编译容量支持 `M<=16`。当前严格基线通过 +`INFINI_GGUF_STRICT_SMALL_PREFILL_MAX_M=10` 选择 `M<=10` 使用该路径，因为它在当前 +32x32 对拍中比更早切换到 prefill 路径更接近 llama.cpp 的归约结果。 + +### 7.5 大 M prefill + +文件： + +```text +/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/nvidia/linear_gguf_dequant.cuh +``` + +当 M 超过小 M 路由阈值时，算子按 tile 把量化权重解码到临时 workspace，再调用 GEMM。 +这种方式没有把整套模型权重永久还原为 BF16，只为当前 Linear 分配必要的临时 scratch，因而 +仍符合 Route B。数值门覆盖到 `M=1024`，端到端验证覆盖 `M=12/64/512`。 + +### 7.6 mixed add-RMSNorm 扩展 + +修改文件： + +```text +/home/liuxd/InfiniCore/src/infiniop/ops/add_rms_norm/info.h +/home/liuxd/InfiniCore/src/infiniop/ops/add_rms_norm/nvidia/add_rms_norm_nvidia.cu +``` + +为研究 BF16 物化边界，新增两类受限组合： + +- F32 `a` + BF16 residual `b` + BF16 weight，FP32 求和和 RMS，输出 BF16； +- BF16 `a` + BF16 `b` + BF16 weight，FP32 求和和 RMS，输出 F32。 + +第一类用于让单个 F32 GGUF Linear 安全跨过 residual+norm 后回到 BF16；第二类用于 final +residual+RMSNorm 全 F32 实验。正常 BF16 模型路径保持不变。 + +### 7.7 GEMM mixed-dtype 修复与 fallback + +修改文件： + +```text +/home/liuxd/InfiniCore/src/infiniop/ops/gemm/nvidia/gemm_nvidia.cu +``` + +完成两项改动： + +1. 修复 row-major 转置执行中交换 A/B 指针却没有同步交换 `a_type/b_type` 的问题； +2. 为 `BF16 large matrix x F32 hidden -> F32` 增加 batch=1 的 tiled register-GEMV fallback。 + +第二项是因为当前 cuBLAS 对该实际 mixed 组合返回 `CUBLAS_STATUS_NOT_SUPPORTED`。fallback +按 16 个 hidden column 分 tile，可覆盖任意 prompt M，但只在实验性 F32 final path 中使用。 + +### 7.8 Q8A 激活量化实验 + +`linear_gguf` 中还加入了受环境变量控制的 Q8A/Q8_1-like 激活量化路径，用于研究 llama.cpp +的激活量化和归约方式。它支持全 GGUF 类型或只命中某个 GGML type。实测该路径能修复个别 +样例，但会使其他样例退化，因此保留代码用于研究，默认关闭。 + +## 8. 数值一致性优化 + +### 8.1 为什么两个后端不会天然完全一致 + +即使权重 block 解码公式正确，以下差异仍可能改变非常接近的 top-1： + +- GEMV/GEMM 的分块和归约顺序； +- 中间结果何时从 FP32 舍入到 BF16； +- llama.cpp 的 Q8 激活量化与 InfiniLM 的 BF16 激活； +- RMSNorm residual sum 的物化 dtype； +- lm_head 累加精度； +- EOS 屏蔽等采样语义。 + +因此严格逐 token 一致性是独立的高标准验证项，不能简单等同于“算子正确性”。 + +### 8.2 当前接受的严格配置 + +当前接受配置包含： + +- GGUF 四类型原生 block kernel； +- FP32 lm_head logits； +- 通用 `ignore_eos` 采样语义； +- small-prefill register path； +- `INFINI_GGUF_STRICT_SMALL_PREFILL=1`； +- `INFINI_GGUF_STRICT_SMALL_PREFILL_MAX_M=10`； +- Q8A、局部 F32 GGUF 输出和 final-FP32 全部关闭。 + +该配置得到： + +```text +27 / 32 cases exact +920 / 1024 tokens match +``` + +剩余首分叉为： + +```text +zh_04 @ token 28 +zh_05 @ token 19 +zh_06 @ token 4 +code_04 @ token 4 +math_04 @ token 1 +``` + +### 8.3 已验证但未采用的实验 + +| 实验 | 结果 | 决策 | +|---|---|---| +| 全类型 Q8A | 能修复 `zh_05`，但五个重点例合计仅 75/160 token | 默认关闭 | +| Q6_K-only Q8A | 27/32、910/1024 | 退化，关闭 | +| Q5_K-only Q8A | 27/32、907/1024 | 退化，关闭 | +| layer0 attention out_proj F32 | `math_04` margin 明显恶化 | 关闭 | +| layer0 MLP down_proj F32 | `math_04` margin 明显恶化 | 关闭 | +| 强制 cuBLAS/寄存器 GEMV切换 | 未稳定修复剩余分叉 | 不采用 | +| final residual+RMSNorm F32 | 27/32、921/1024；修复 `zh_05` 但回归 `math_02` | 默认关闭 | + +final-FP32 的全量结果比基线多匹配 1 个 token，但 exact case 仍为 27/32。它将 `zh_05` +修复为 32/32，同时使原本 exact 的 `math_02` 在 token 18 分叉。`math_02` 的参考 token 只落后 +约 `5.95e-4`，说明这是非常临界的归约/舍入翻转，但在没有消除回归前不能作为默认优化。 + +## 9. 验证方法与结果 + +### 9.1 字节布局和解码公式 + +- 映射/字节布局审计：48 PASS / 0 FAIL； +- 四类型 block decode 交叉验证：47 PASS / 0 FAIL； +- 使用真实 GGUF blocks 做大规模抽样； +- 对 half 的 65536 种 bit pattern 做穷举扫描，并覆盖四种格式相关路径； +- blob 原始字节与源 GGUF 分类抽样逐字节一致。 + +### 9.2 Linear 数值门 + +- decode GEMV：两套独立产物各 56 PASS / 0 FAIL，cosine similarity 均大于 0.999； +- prefill：316 PASS / 0 FAIL； +- 覆盖四种 GGUF 类型、多种 N/K/M 和真实模型行字节； +- 端到端 prefill 覆盖 M=12、64、512。 + +### 9.3 映射、shape 与加载 + +- 映射计划共 947 条； +- 491 个 blob 的行字节均可整除且与 GGUF `n_bytes` 一致； +- 947 个产物 tensor 与引擎消费 tensor 双向集合一致； +- 947/947 shape 一致； +- 491 个 blob 在配置和 safetensors 中均为 U8； +- 6 个分片全量加载成功； +- 首个 blob forward 日志证明进入 `linear_gguf`，不存在稠密静默回退。 + +### 9.4 mini8 端到端 + +构造覆盖全部四种量化类型的 mini8 模型，61 个 blob 的分布为： + +```text +Q6_K: 35 +Q5_K: 12 +Q8_0: 10 +Q4_K: 4 +``` + +模型成功执行 `generate()`，阶段检查 11 PASS / 0 FAIL，确认加载、路由、decode 和状态推进 +形成闭环。 + +### 9.5 全量模型 + +全量 Qwen3.8-27B native-v2 已完成： + +- 配置构造； +- 947 项权重装载； +- prompt prefill； +- autoregressive decode； +- GDN/SSM state 更新； +- 多 prompt 重复确定性； +- 32 x 32 严格 token 对拍。 + +llama.cpp 参考运行记录约为 prompt 29.0 token/s、generation 24.6 token/s。该数字只用于描述 +参考后端，不是 InfiniLM 性能结论。InfiniLM 的正式吞吐、首 token 延迟、显存峰值和不同 +prompt 长度曲线仍需要独立 benchmark 后才能下结论。 + +## 10. 最终结果与完成度判断 + +### 10.1 已完成 + +1. Qwen3.8-27B 模型架构可以在 InfiniLM 中构造和执行。 +2. GGUF 到 InfiniLM 的映射、打包和配置生成已经完成。 +3. Q8_0/Q4_K/Q5_K/Q6_K 四种原生权重算子已经完成。 +4. decode 和 prefill 两条执行路径都已跑通。 +5. 27B 全量模型可加载、生成，且明确走 U8 blob 原生 kernel。 +6. shape、字节、数值、端到端和严格对拍均有报告留档。 +7. 采样侧 `ignore_eos` 语义已与参考设置对齐。 +8. 通用的 F32 边界、mixed add-RMSNorm 和 mixed GEMM 实验能力已经实现。 + +### 10.2 当前最终采用结果 + +```text +功能适配：成功 +全量模型加载：成功 +Prefill：成功 +Decode：成功 +原生量化类型：Q8_0 / Q4_K / Q5_K / Q6_K +严格一致性：27/32 cases，920/1024 tokens +严格目标 >=29/32：未完成，属于可选优化项 +``` + +### 10.3 如何理解“成功” + +如果验收标准是“让 InfiniLM 正确加载并推理 Qwen3.8-27B GGUF，并保留主要 GGUF 量化权重 +不展开”，本工作已经完成。 + +如果验收标准额外要求“InfiniLM 与 llama.cpp 在固定 32 个样例中至少 29 个逐 token 完全 +一致”，当前还差 2 个 exact case。后者是跨后端数值复现目标，不影响模型基本可用性；是否 +继续投入，应由比赛规则、评测规则或业务需求决定。 + +## 11. 对其他 GGUF 模型的复用方式 + +### 11.1 可直接复用的通用部分 + +- safetensors U8 `weight_bytes` 存储约定； +- `config.json` 中的 `ggml_types` 和 `quantization_config`； +- Q8_0/Q4_K/Q5_K/Q6_K block decoder； +- `linear_gguf` C API、C++ API 和 NVIDIA backend； +- small-M register GEMV 与 large-M prefill dispatch； +- row bytes、dtype、shape 和无 silent fallback 契约； +- GGUF 原字节抽样、解码交叉验证和 Linear 数值门； +- mixed-dtype 边界诊断能力； +- 32x32 token 对拍、首分叉和 logits margin 工具。 + +### 11.2 每个新模型仍需适配的部分 + +- GGUF tensor name 到模型参数名的映射； +- fused tensor 的拆分/拼接规则； +- transpose、head permutation 或专家布局； +- norm gain 等模型特有权重约定； +- attention、MoE、SSM/GDN 等模型结构； +- cache/state 形状和生命周期； +- tokenizer、chat template、EOS 和 stop semantics； +- 模型实际包含但当前 kernel 未支持的 GGML 类型。 + +### 11.3 推荐的新模型适配流程 + +1. 读取 GGUF metadata，列出架构、tensor names、types、shapes 和 block bytes。 +2. 新增模型维度类和 `build_plan()` 映射，不先写运行时特例。 +3. 运行 dry-run，做源 GGUF 与目标模型参数的双向集合/shape 审计。 +4. 将已支持的四种类型标记为 blob；其他类型明确列为 BF16 例外或新增 decoder。 +5. 实现模型特有的 fused slice、transpose、permutation 和 norm convention。 +6. 打包并执行全量字节/shape/dtype 自检。 +7. 给各类型建立独立 block decode 和 Linear 数值门。 +8. 用 mini 模型覆盖所有类型，完成 prefill+decode 闭环。 +9. 加载全量模型，确认日志中首个和代表性权重进入 `linear_gguf`。 +10. 最后做生成质量、确定性、性能和参考后端一致性测试。 + +这种流程下，新增一个结构相近且量化类型相同的模型，主要工作会集中在映射和模型结构层； +底层 GGUF 算子无需重复实现。 + +## 12. 当前限制与后续建议 + +### 12.1 当前限制 + +1. embedding 和 lm_head 尚未采用原生 GGUF kernel。 +2. IQ4_XS/IQ4_NL 等 IQ 类型尚未原生支持。 +3. GGUF blob 当前未实现 tensor parallel 切分。 +4. prefill 已正确运行，但 tile 解码 workspace 和 GEMM 路由仍有性能优化空间。 +5. 当前 32x32 与 llama.cpp 不是完全一致，剩余 5 个样例存在首分叉。 +6. InfiniLM 正式性能数据尚未形成完整 benchmark 报告。 +7. Q8A、F32 GGUF output、final-FP32 等研究路径均默认关闭。 + +### 12.2 优先级建议 + +如果目标是工程交付，建议按以下顺序继续： + +1. 固化默认环境、构建说明和一键回归； +2. 测 InfiniLM 吞吐、TTFT、decode latency 和显存峰值； +3. 增加 native lm_head 和 embedding，降低约 2.51 GiB 权重占用； +4. 根据目标模型分布决定是否实现 IQ4； +5. 若有多卡需求，再设计 blob tensor parallel； +6. 只有评测明确要求时，再继续追求 `>=29/32` 的严格一致性。 + +若继续严格一致性，下一步应拆分 final-FP32 变量，分别测试：仅 F32 norm output、F32 +residual sum + BF16 norm output、以及不同 lm_head reduction order。候选必须同时保留 +`math_02` exact 并修复 `zh_05`，再允许跑完整 32x32，避免无方向地枚举局部精度开关。 + +## 13. 运行与复现要点 + +环境脚本： + +```bash +source /home/liuxd/InfiniLM/scripts/gguf_routeb_env.sh +``` + +严格基线的关键环境变量： + +```bash +export INFINI_GGUF_STRICT_SMALL_PREFILL=1 +export INFINI_GGUF_STRICT_SMALL_PREFILL_MAX_M=10 +``` + +实验变量默认不应设置： + +```text +INFINI_GGUF_DECODE_Q8A +INFINI_GGUF_DECODE_Q8A_TYPE +INFINI_GGUF_F32_DECODE_OUT +INFINI_GGUF_F32_DECODE_OUT_MATCH +INFINILM_FINAL_NORM_FP32_FUSED +``` + +关键报告： + +```text +/home/liuxd/tmp_routeb/reports/R3_strict_small_prefill_maxm10_32x32.json +/home/liuxd/tmp_routeb/reports/R3_compare_strict_small_prefill_maxm10_32x32.json +/home/liuxd/tmp_routeb/reports/R3_final_fp32_32x32.json +/home/liuxd/tmp_routeb/reports/R3_compare_final_fp32_32x32.json +``` + +构建时需要特别注意：只执行 `xmake build/install infiniop` 不足以保证 Python runtime 使用 +最新库。运行时实际优先加载： + +```text +/home/liuxd/InfiniCore/python/infinicore/lib/libinfiniop.so +``` + +因此 InfiniCore 改动后还必须执行 `xmake install _infinicore`，并核对安装目录与构建目录的 +动态库哈希一致。此前多次“代码改了但结果不变”的根因就是只更新了 `/home/liuxd/.infini/lib` +而没有更新 Python 实际加载的副本。 + +## 14. 结论 + +本次工作已经建立了一条完整、可验证、可复用的 GGUF Route B：从 GGUF tensor 映射、原始 +block bytes 打包，到 InfiniLM 类型路由、InfiniCore 原生 GPU 解码、prefill/decode，再到 +全量 27B 模型生成和跨后端对拍，整个链路已经打通。 + +新增的核心能力不是只针对某一个 Qwen 权重文件的临时代码，而是一个可承载多模型的 GGUF +块量化线性算子框架。适配其他 GGUF 大模型时，可以复用存储协议、四类 block decoder、 +`linear_gguf` 执行后端和验证体系，只需重点补充模型映射、结构接线和新的量化类型。 + +当前应将项目状态定义为：**GGUF 功能适配完成，主要量化权重原生执行成功；严格一致性达到 +27/32，但 29/32 目标尚未完成且不是基本可用性的必要条件。** diff --git a/csrc/config/quant_config.cpp b/csrc/config/quant_config.cpp index e58966d89..195c70d44 100644 --- a/csrc/config/quant_config.cpp +++ b/csrc/config/quant_config.cpp @@ -20,6 +20,11 @@ QuantConfig::get_quantization_method() const { return std::make_shared(quantization_config); } else if (quant_method == "gptq") { return std::make_shared(quantization_config); + } else if (quant_method == "fp8") { + return std::make_shared(quantization_config); + } else if (quant_method == "gguf") { + // 路线 B：GGUF block 字节原样进显存，kernel 在 InfiniCore（阶段 3） + return std::make_shared(quantization_config); } else if (quant_method == "quark") { return std::make_shared(quantization_config); } else { diff --git a/csrc/engine/rank_worker.cpp b/csrc/engine/rank_worker.cpp index 0fa0a84cf..7936f88d7 100644 --- a/csrc/engine/rank_worker.cpp +++ b/csrc/engine/rank_worker.cpp @@ -2,11 +2,48 @@ #include "../models/model_factory.hpp" #include "infinicore/ops.hpp" #include "infinicore/ops/distributed/send_recv.hpp" +#include +#include #include #include namespace infinilm::engine { +namespace { + +infinicore::Tensor negative_infinity_cpu(infinicore::DataType dtype) { + auto scalar = infinicore::Tensor::empty( + {1}, dtype, infinicore::Device::cpu()); + switch (dtype) { + case infinicore::DataType::F16: { + const uint16_t value = 0xfc00U; + std::memcpy(scalar->data(), &value, sizeof(value)); + break; + } + case infinicore::DataType::BF16: { + const uint16_t value = 0xff80U; + std::memcpy(scalar->data(), &value, sizeof(value)); + break; + } + case infinicore::DataType::F32: { + const uint32_t value = 0xff800000U; + std::memcpy(scalar->data(), &value, sizeof(value)); + break; + } + case infinicore::DataType::F64: { + const uint64_t value = 0xfff0000000000000ULL; + std::memcpy(scalar->data(), &value, sizeof(value)); + break; + } + default: + throw std::runtime_error( + "suppressed_token_ids requires floating-point logits"); + } + return scalar; +} + +} // namespace + RankWorker::RankWorker( std::shared_ptr infinilm_config, const distributed::RankInfo &rank_info, @@ -484,12 +521,45 @@ void RankWorker::thread_loop() { const size_t n_out = sample_all_positions ? static_cast(input_offsets[n_req]) : n_req; auto output_ids{infinicore::Tensor::empty({n_out}, infinicore::DataType::I64, rank_info_.device)}; + const auto &suppressed = local_args.suppressed_token_ids; + if (!suppressed.empty() && suppressed.size() != n_req) { + throw std::runtime_error( + "suppressed_token_ids must contain one list per request"); + } + infinicore::Tensor neg_inf_cpu; + infinicore::Tensor neg_inf_device; + if (!suppressed.empty()) { + neg_inf_cpu = negative_infinity_cpu(logits->dtype()); + neg_inf_device = neg_inf_cpu->to(rank_info_.device); + } + + size_t req_idx = 0; + for (size_t i{0}; i < n_out; ++i) { size_t score_idx = i; if (!sample_all_positions && !logits_are_last_token_only) { score_idx = static_cast(input_offsets[i + 1] - 1); } auto score{logits->view({logits_positions, vocab_size})->narrow({{0, score_idx, 1}})->view({vocab_size})}; + if (sample_all_positions) { + while (req_idx + 1 < n_req + && i >= static_cast(input_offsets[req_idx + 1])) { + ++req_idx; + } + } else { + req_idx = i; + } + if (!suppressed.empty()) { + for (const int64_t token_id : suppressed[req_idx]) { + if (token_id < 0 + || static_cast(token_id) >= vocab_size) { + throw std::runtime_error( + "suppressed token ID is outside the vocabulary"); + } + score->narrow({{0, static_cast(token_id), 1}}) + ->copy_from(neg_inf_device); + } + } auto out{output_ids->narrow({{0, i, 1}})->view({})}; float random_val = std::uniform_real_distribution(0, 1)(rng_); infinicore::op::random_sample_( diff --git a/csrc/engine/rank_worker.hpp b/csrc/engine/rank_worker.hpp index d396ef6f1..d000331c9 100644 --- a/csrc/engine/rank_worker.hpp +++ b/csrc/engine/rank_worker.hpp @@ -10,6 +10,7 @@ #include "rank_barrier.hpp" #include +#include #include #include #include @@ -73,6 +74,9 @@ class RankWorker { /// Sample logits at every packed input position instead of one token per request. bool sample_all_positions{false}; + /// Token IDs excluded from sampling for each request in the batch. + std::vector> suppressed_token_ids{}; + float temperature{1}; int top_k{50}; diff --git a/csrc/layers/causal_lm_templates/text_model.hpp b/csrc/layers/causal_lm_templates/text_model.hpp index 49b60d7f4..ffa0b70ac 100644 --- a/csrc/layers/causal_lm_templates/text_model.hpp +++ b/csrc/layers/causal_lm_templates/text_model.hpp @@ -6,11 +6,15 @@ #include "infinicore/nn/embedding.hpp" #include "infinicore/nn/rmsnorm.hpp" #include "infinicore/ops.hpp" +#include "infinicore/ops/add_rms_norm.hpp" +#include "infinicore/ops/cast.hpp" #include "infinicore/ops/distributed/allgather.hpp" #include "infinicore/ops/distributed/send_recv.hpp" #include "infinicore/tensor.hpp" +#include #include #include +#include namespace infinilm::layers::causal_lm_templates { @@ -79,7 +83,8 @@ class TextModel : public infinicore::nn::Module { return hidden_states; } - norm_->forward_inplace(hidden_states, residual); + dump_pre_final_norm_if_requested(hidden_states, residual); + final_norm_inplace(hidden_states, residual); return hidden_states; } @@ -119,7 +124,8 @@ class TextModel : public infinicore::nn::Module { return hidden_states; } - norm_->forward_inplace(hidden_states, residual); + dump_pre_final_norm_if_requested(hidden_states, residual); + final_norm_inplace(hidden_states, residual); return hidden_states; } @@ -136,6 +142,22 @@ class TextModel : public infinicore::nn::Module { INFINICORE_NN_MODULE(infinicore::nn::RMSNorm, norm); private: + void final_norm_inplace(infinicore::Tensor &hidden_states, + infinicore::Tensor &residual) const { + const char *env = std::getenv("INFINILM_FINAL_NORM_FP32_FUSED"); + const bool enabled = env != nullptr && env[0] != '\0' && std::string(env) != "0"; + if (!enabled) { + norm_->forward_inplace(hidden_states, residual); + return; + } + auto y32 = infinicore::Tensor::empty(hidden_states->shape(), infinicore::DataType::F32, hidden_states->device()); + auto sum32 = infinicore::Tensor::empty(residual->shape(), infinicore::DataType::F32, residual->device()); + infinicore::op::add_rms_norm_(y32, sum32, hidden_states, residual, norm_->weight(), + static_cast(norm_->eps())); + hidden_states = y32; + residual = sum32; + } + bool is_first_pp_stage() const { return pp_stage_ == 0; } bool is_last_pp_stage() const { return pp_stage_ + 1 == pp_size_; } @@ -206,6 +228,25 @@ class TextModel : public infinicore::nn::Module { return infinicore::op::add(residual, hidden_states); } + void dump_pre_final_norm_if_requested( + infinicore::Tensor &hidden_states, + infinicore::Tensor &residual) const { + const char *dump_dir = std::getenv("INFINILM_FINAL_PRENORM_DUMP_DIR"); + if (dump_dir == nullptr || dump_dir[0] == '\0') { + return; + } + const char *dump_numel = + std::getenv("INFINILM_FINAL_PRENORM_DUMP_NUMEL"); + if (dump_numel != nullptr && dump_numel[0] != '\0' + && hidden_states->numel() + != std::strtoull(dump_numel, nullptr, 10)) { + return; + } + auto pre_norm = materialize_hidden_states(hidden_states, residual); + pre_norm->debug( + std::string(dump_dir) + "/infini_pre_final_norm.bin"); + } + infinicore::DataType dtype_{infinicore::DataType::F32}; size_t hidden_size_{0}; size_t pp_size_{1}; diff --git a/csrc/layers/linear/base_linear.cpp b/csrc/layers/linear/base_linear.cpp index dc4c77f62..5c0ce0a8a 100644 --- a/csrc/layers/linear/base_linear.cpp +++ b/csrc/layers/linear/base_linear.cpp @@ -9,19 +9,24 @@ BaseLinear::BaseLinear(size_t in_features, size_t out_features, bool bias, const infinicore::DataType &dtype, const infinicore::Device &device, int split_dim, int tp_rank, int tp_size, - int tp_num_heads) + int tp_num_heads, const std::string &stem) : in_features_(in_features), out_features_(out_features), has_bias_(bias), dtype_(dtype), split_dim_(split_dim), + stem_(stem), quantization_(quantization) { device_ = device; auto layout = quantization_->get_param_layout( in_features, out_features, split_dim, tp_rank, tp_size, - tp_num_heads, dtype, bias); + tp_num_heads, dtype, bias, stem); + + // 空布局 = 量化方案声明“这个 Linear 的参数不是一整块”（GGUF 的融合组）， + // 具体 shard 由派生类在构造体内用 init_fused_shards() 申请。 + sharded_ = layout.empty(); for (const auto &desc : layout) { infinicore::nn::Parameter param( @@ -33,6 +38,10 @@ BaseLinear::BaseLinear(size_t in_features, size_t out_features, } infinicore::Tensor BaseLinear::compute_linear(infinicore::Tensor &input) const { + if (sharded_ && parameters_.empty()) { + throw std::runtime_error( + "BaseLinear::compute_linear: 融合量化布局的 shard 还没注册（内部错误）"); + } // Build params map from direct parameters only (not state_dict which uses a // static local and is not thread-safe across RankWorker threads). infinilm::quantization::ParamsMap params; @@ -40,7 +49,7 @@ infinicore::Tensor BaseLinear::compute_linear(infinicore::Tensor &input) const { params[name] = static_cast(param); } - return quantization_->forward(params, input, has_bias_, alpha_); + return quantization_->forward(params, input, has_bias_, alpha_, stem_, shard_stems_); } infinicore::Tensor BaseLinear::compute_linear_allreduce( @@ -161,4 +170,36 @@ std::vector BaseLinear::split_params( parameters_, splits, split_dim_, tp_rank, tp_size, tp_num_heads); } +std::vector BaseLinear::init_fused_shards( + const std::vector &shards) { + std::vector registered; + shard_stems_.clear(); + shard_stems_.reserve(shards.size()); + for (size_t i = 0; i < shards.size(); ++i) { + const auto &sh = shards[i]; + // 下标 i 同时是参数 key 里的 shard 和 shard_stems_ 的位置：两者在同一行里产生 + shard_stems_.push_back(sh.stem); + // 各 shard 自己是一块完整的列并行参数，不做 TP 切分（GGUF 路径 tp_size 恒为 1， + // 量化类里会对 tp_size > 1 直接抛错，见方案 §6.2） + auto layout = quantization_->get_param_layout( + in_features_, sh.out_features, split_dim_, 0, 1, -1, dtype_, false, sh.stem); + if (layout.empty()) { + throw std::runtime_error( + "BaseLinear::init_fused_shards: shard '" + sh.stem + "' 又返回了空布局"); + } + for (const auto &desc : layout) { + infinicore::nn::Parameter param( + desc.shape, desc.dtype, device_, desc.split_dim, 0, 1, 0); + // key 里的 "shard." 前缀是量化类在 forward() 里还原拼接顺序的依据 + this->register_parameter( + std::string(infinilm::quantization::GGUFBlockQuantization::SHARD_PREFIX) + + std::to_string(i) + "." + desc.name, + param); + registered.push_back({sh.name + "." + desc.name, std::move(param)}); + } + } + sharded_ = true; + return registered; +} + } // namespace infinilm::nn diff --git a/csrc/layers/linear/base_linear.hpp b/csrc/layers/linear/base_linear.hpp index 8b452544e..ad76110d2 100644 --- a/csrc/layers/linear/base_linear.hpp +++ b/csrc/layers/linear/base_linear.hpp @@ -18,7 +18,8 @@ class BaseLinear : public infinicore::nn::Module { const infinicore::DataType &dtype = infinicore::DataType::F32, const infinicore::Device &device = infinicore::Device(), int split_dim = -1, int tp_rank = 0, int tp_size = 1, - int tp_num_heads = -1); + int tp_num_heads = -1, + const std::string &stem = ""); // Forward pass: output = input @ weight.T + bias infinicore::Tensor forward(infinicore::Tensor &input) const; @@ -53,6 +54,23 @@ class BaseLinear : public infinicore::nn::Module { const std::vector &splits, int tp_rank, int tp_size, int tp_num_heads) const; + // One shard of a fused linear, for schemes that cannot share a single fused + // buffer (GGUF block quantization: row_bytes differs per shard type). + struct FusedShard { + std::string name; // "q_proj" / "gate_proj" ... 注册到父模块时用 + size_t out_features; // 本 shard 的逻辑输出行数 + std::string stem; // "layers.0.self_attn.q_proj." 类型表查询用 + }; + + // 为融合 Linear 逐 shard 各分配一块独立 buffer：本对象 parameters_ 里的 key 是 + // "shard."（i 即输出 dim(-1) 上的顺序），返回值里的 full_name 是 + // "."，交给调用方的 register_fn 注册到父模块（与 split_params 同路）。 + // 只有 get_param_layout(带 stem) 返回空布局（融合组）的方案才走这里。 + // 顺带把每个 shard 的 checkpoint stem 记进 shard_stems_（下标 = 上面的 i）： + // 组 stem 查不出各 shard 的格式，forward 必须把它们交还给量化方案。 + std::vector init_fused_shards( + const std::vector &shards); + // Allow subclasses to access the raw parameters map const infinicore::nn::Parameter &get_parameter_ref(const std::string &name) const; @@ -67,6 +85,12 @@ class BaseLinear : public infinicore::nn::Module { infinicore::DataType dtype_; int split_dim_ = -1; float alpha_ = 1.0f; + std::string stem_; // checkpoint 张量名路径（只给按名字查表的量化方案用，见 §6.0 纠正 2） + // init_fused_shards 记下的逐 shard checkpoint stem，下标 == parameters_ key 里的 i。 + // 与 key 在同一个循环里产生、forward 里消费，因此只是个局部不变量（不是跨阶段约定）； + // 非融合路径为空。语义见 BaseQuantization::forward 的 shard_stems 重载。 + std::vector shard_stems_; + bool sharded_ = false; // 融合量化布局：本对象不持有融合 buffer，参数在 shard.* 里 std::shared_ptr quantization_; }; diff --git a/csrc/layers/linear/fused_linear.cpp b/csrc/layers/linear/fused_linear.cpp index 1bcb96c94..b8c30ddaa 100644 --- a/csrc/layers/linear/fused_linear.cpp +++ b/csrc/layers/linear/fused_linear.cpp @@ -20,7 +20,7 @@ QKVParallelLinear::QKVParallelLinear(size_t hidden_size, num_q_head, num_kv_head, num_kv_head, bias, bias, bias, quantization, - dtype, device, rank_info) {} + dtype, device, rank_info, "") {} QKVParallelLinear::QKVParallelLinear(size_t hidden_size, size_t q_dim, size_t k_dim, size_t v_dim, @@ -29,7 +29,8 @@ QKVParallelLinear::QKVParallelLinear(size_t hidden_size, std::shared_ptr quantization, const infinicore::DataType &dtype, const infinicore::Device &device, - engine::distributed::RankInfo rank_info) + engine::distributed::RankInfo rank_info, + const std::string &stem) : infinilm::nn::ColumnParallelLinear( hidden_size, calculate_out_feature_size(num_q_head, q_dim, num_k_head, k_dim, num_v_head, v_dim, rank_info), @@ -38,7 +39,9 @@ QKVParallelLinear::QKVParallelLinear(size_t hidden_size, dtype, device, rank_info.tp_rank, - rank_info.tp_size), + rank_info.tp_size, + -1, + stem), q_dim_(q_dim), k_dim_(k_dim), v_dim_(v_dim), @@ -83,8 +86,9 @@ QKVParallelLinear::QKVParallelLinear(size_t hidden_size, bool bias, const infinicore::DataType &dtype, const infinicore::Device &device, - engine::distributed::RankInfo rank_info) - : QKVParallelLinear(hidden_size, head_dim, head_dim, head_dim, num_q_head, num_kv_head, num_kv_head, bias, bias, bias, q_name, k_name, v_name, register_fn, quantization, dtype, device, rank_info) { + engine::distributed::RankInfo rank_info, + const std::string &prefix) + : QKVParallelLinear(hidden_size, head_dim, head_dim, head_dim, num_q_head, num_kv_head, num_kv_head, bias, bias, bias, q_name, k_name, v_name, register_fn, quantization, dtype, device, rank_info, prefix) { } QKVParallelLinear::QKVParallelLinear(size_t hidden_size, @@ -96,15 +100,39 @@ QKVParallelLinear::QKVParallelLinear(size_t hidden_size, std::shared_ptr quantization, const infinicore::DataType &dtype, const infinicore::Device &device, - engine::distributed::RankInfo rank_info) - : QKVParallelLinear(hidden_size, q_dim, k_dim, v_dim, num_q_head, num_k_head, num_v_head, q_bias, k_bias, v_bias, quantization, dtype, device, rank_info) { + engine::distributed::RankInfo rank_info, + const std::string &prefix) + : QKVParallelLinear(hidden_size, q_dim, k_dim, v_dim, num_q_head, num_k_head, num_v_head, q_bias, k_bias, v_bias, quantization, dtype, device, rank_info, prefix) { register_fn_ = register_fn; - split_infos_ = { - {q_name, 0, q_out_size_, 0}, - {k_name, q_out_size_, k_out_size_, num_k_head_}, - {v_name, q_out_size_ + k_out_size_, v_out_size_, num_v_head_}, - }; - auto params = this->split_params(split_infos_, tp_rank_, tp_size_, num_k_head_); + if (this->sharded_) { + // GGUF：q/k/v 在本文件里 ggml 类型全不相同（§6.0 纠正 1），没有可 narrow 的 + // 融合 buffer —— 每 shard 各自一块，stem 指向各自的 checkpoint 张量。 + if (prefix.empty()) { + throw std::runtime_error( + "QKVParallelLinear: 按 checkpoint 张量名查表的量化方案（GGUF）必须传 prefix"); + } + shard_specs_ = { + {q_name, q_out_size_, prefix + "." + q_name + "."}, + {k_name, k_out_size_, prefix + "." + k_name + "."}, + {v_name, v_out_size_, prefix + "." + v_name + "."}, + }; + } else { + split_infos_ = { + {q_name, 0, q_out_size_, 0}, + {k_name, q_out_size_, k_out_size_, num_k_head_}, + {v_name, q_out_size_ + k_out_size_, v_out_size_, num_v_head_}, + }; + } + register_fused_params(); +} + +void QKVParallelLinear::register_fused_params() { + if (!register_fn_) { + return; + } + auto params = this->sharded_ + ? this->init_fused_shards(shard_specs_) + : this->split_params(split_infos_, tp_rank_, tp_size_, num_k_head_); for (auto &sp : params) { register_fn_(sp.full_name, std::move(sp.param)); } @@ -112,11 +140,10 @@ QKVParallelLinear::QKVParallelLinear(size_t hidden_size, void QKVParallelLinear::process_weights_after_loading() { BaseLinear::process_weights_after_loading(); + // 融合量化布局（sharded_）下 split_infos_ 为空，不会重跑：那些 shard 参数就是 + // 加载目标，重新分配会把已读进来的字节丢掉 if (register_fn_ && !split_infos_.empty()) { - auto params = this->split_params(split_infos_, tp_rank_, tp_size_, num_k_head_); - for (auto &sp : params) { - register_fn_(sp.full_name, std::move(sp.param)); - } + register_fused_params(); } } @@ -125,14 +152,16 @@ void QKVParallelLinear::process_weights_after_loading() { // --------------------------------------------------------- GateUpParallelLinear::GateUpParallelLinear(size_t hidden_size, size_t intermediate_size, std::shared_ptr quantization, bool bias, const infinicore::DataType &dtype, const infinicore::Device &device, - engine::distributed::RankInfo rank_info) - : GateUpParallelLinear(hidden_size, intermediate_size, bias, bias, quantization, dtype, device, rank_info) { + engine::distributed::RankInfo rank_info, + const std::string &stem) + : GateUpParallelLinear(hidden_size, intermediate_size, bias, bias, quantization, dtype, device, rank_info, stem) { } GateUpParallelLinear::GateUpParallelLinear(size_t hidden_size, size_t intermediate_size, bool gate_bias, bool up_bias, std::shared_ptr quantization, const infinicore::DataType &dtype, const infinicore::Device &device, - engine::distributed::RankInfo rank_info) + engine::distributed::RankInfo rank_info, + const std::string &stem) : infinilm::nn::ColumnParallelLinear( hidden_size, intermediate_size * 2, @@ -141,7 +170,9 @@ GateUpParallelLinear::GateUpParallelLinear(size_t hidden_size, size_t intermedia dtype, device, rank_info.tp_rank, - rank_info.tp_size), + rank_info.tp_size, + -1, + stem), gate_bias_(gate_bias), up_bias_(up_bias) { if (gate_bias_ != up_bias_) { @@ -166,19 +197,43 @@ GateUpParallelLinear::GateUpParallelLinear(size_t hidden_size, size_t intermedia std::shared_ptr quantization, bool bias, const infinicore::DataType &dtype, const infinicore::Device &device, - engine::distributed::RankInfo rank_info) - : GateUpParallelLinear(hidden_size, intermediate_size, quantization, bias, dtype, device, rank_info) { - const std::string &key_name = parameters_.count("qweight") ? "qweight" : "weight"; - const auto &key_param = get_parameter_ref(key_name); - int fused_dim = this->get_quantization()->get_fused_split_dim(); - size_t logical_output = this->get_quantization()->get_logical_dim_size(key_param->size(fused_dim)); - size_t half_size = logical_output / 2; + engine::distributed::RankInfo rank_info, + const std::string &prefix) + : GateUpParallelLinear(hidden_size, intermediate_size, quantization, bias, dtype, device, rank_info, prefix) { register_fn_ = register_fn; - split_infos_ = { - {gate_name, 0, half_size}, - {up_name, half_size, half_size}, - }; - auto params = this->split_params(split_infos_, tp_rank_, tp_size_, -1); + if (this->sharded_) { + // GGUF：gate/up 在本文件 28/64 层类型不同（§6.0 纠正 1），两者 row_bytes 不同， + // 装不进同一块融合 buffer，所以各自一块、各自查自己是 blob 还是稠密。 + if (prefix.empty()) { + throw std::runtime_error( + "GateUpParallelLinear: 按 checkpoint 张量名查表的量化方案（GGUF）必须传 prefix"); + } + const size_t half = intermediate_size / tp_size_; + shard_specs_ = { + {gate_name, half, prefix + "." + gate_name + "."}, + {up_name, half, prefix + "." + up_name + "."}, + }; + } else { + const std::string &key_name = parameters_.count("qweight") ? "qweight" : "weight"; + const auto &key_param = get_parameter_ref(key_name); + int fused_dim = this->get_quantization()->get_fused_split_dim(); + size_t logical_output = this->get_quantization()->get_logical_dim_size(key_param->size(fused_dim)); + size_t half_size = logical_output / 2; + split_infos_ = { + {gate_name, 0, half_size}, + {up_name, half_size, half_size}, + }; + } + register_fused_params(); +} + +void GateUpParallelLinear::register_fused_params() { + if (!register_fn_) { + return; + } + auto params = this->sharded_ + ? this->init_fused_shards(shard_specs_) + : this->split_params(split_infos_, tp_rank_, tp_size_, -1); for (auto &sp : params) { register_fn_(sp.full_name, std::move(sp.param)); } @@ -186,11 +241,9 @@ GateUpParallelLinear::GateUpParallelLinear(size_t hidden_size, size_t intermedia void GateUpParallelLinear::process_weights_after_loading() { BaseLinear::process_weights_after_loading(); + // 同 QKVParallelLinear：sharded_ 时 split_infos_ 为空，不重跑切分 if (register_fn_ && !split_infos_.empty()) { - auto params = this->split_params(split_infos_, tp_rank_, tp_size_, -1); - for (auto &sp : params) { - register_fn_(sp.full_name, std::move(sp.param)); - } + register_fused_params(); } } diff --git a/csrc/layers/linear/fused_linear.hpp b/csrc/layers/linear/fused_linear.hpp index 8773a081c..0a9d6c987 100644 --- a/csrc/layers/linear/fused_linear.hpp +++ b/csrc/layers/linear/fused_linear.hpp @@ -16,7 +16,8 @@ class QKVParallelLinear : public infinilm::nn::ColumnParallelLinear { std::shared_ptr quantization = nullptr, const infinicore::DataType &dtype = infinicore::DataType::F32, const infinicore::Device &device = infinicore::Device(), - engine::distributed::RankInfo rank_info = engine::distributed::RankInfo()); + engine::distributed::RankInfo rank_info = engine::distributed::RankInfo(), + const std::string &stem = ""); explicit QKVParallelLinear(size_t hidden_size, size_t head_dim, @@ -36,7 +37,8 @@ class QKVParallelLinear : public infinilm::nn::ColumnParallelLinear { std::shared_ptr quantization = nullptr, const infinicore::DataType &dtype = infinicore::DataType::F32, const infinicore::Device &device = infinicore::Device(), - engine::distributed::RankInfo rank_info = engine::distributed::RankInfo()); + engine::distributed::RankInfo rank_info = engine::distributed::RankInfo(), + const std::string &prefix = ""); QKVParallelLinear(size_t hidden_size, size_t head_dim, @@ -47,7 +49,8 @@ class QKVParallelLinear : public infinilm::nn::ColumnParallelLinear { bool bias = false, const infinicore::DataType &dtype = infinicore::DataType::F32, const infinicore::Device &device = infinicore::Device(), - engine::distributed::RankInfo rank_info = engine::distributed::RankInfo()); + engine::distributed::RankInfo rank_info = engine::distributed::RankInfo(), + const std::string &prefix = ""); void process_weights_after_loading() override; @@ -91,6 +94,11 @@ class QKVParallelLinear : public infinilm::nn::ColumnParallelLinear { size_t num_kv_head_replicas_ = 1; RegisterParamFn register_fn_; std::vector split_infos_; + // GGUF 等「每 shard 一块 buffer」的方案用（与 split_infos_ 二选一，见 sharded_） + std::vector shard_specs_; + + // 把各 shard 参数交给 register_fn（narrow 视图或独立 buffer，两条路同一入口） + void register_fused_params(); }; class GateUpParallelLinear : public infinilm::nn::ColumnParallelLinear { @@ -100,12 +108,14 @@ class GateUpParallelLinear : public infinilm::nn::ColumnParallelLinear { bool bias = false, const infinicore::DataType &dtype = infinicore::DataType::F32, const infinicore::Device &device = infinicore::Device(), - engine::distributed::RankInfo rank_info = engine::distributed::RankInfo()); + engine::distributed::RankInfo rank_info = engine::distributed::RankInfo(), + const std::string &stem = ""); GateUpParallelLinear(size_t hidden_size, size_t intermediate_size, bool gate_bias, bool up_bias, std::shared_ptr quantization = nullptr, const infinicore::DataType &dtype = infinicore::DataType::F32, const infinicore::Device &device = infinicore::Device(), - engine::distributed::RankInfo rank_info = engine::distributed::RankInfo()); + engine::distributed::RankInfo rank_info = engine::distributed::RankInfo(), + const std::string &stem = ""); GateUpParallelLinear(size_t hidden_size, size_t intermediate_size, const std::string &gate_name, const std::string &up_name, @@ -114,7 +124,8 @@ class GateUpParallelLinear : public infinilm::nn::ColumnParallelLinear { bool bias = false, const infinicore::DataType &dtype = infinicore::DataType::F32, const infinicore::Device &device = infinicore::Device(), - engine::distributed::RankInfo rank_info = engine::distributed::RankInfo()); + engine::distributed::RankInfo rank_info = engine::distributed::RankInfo(), + const std::string &prefix = ""); void process_weights_after_loading() override; @@ -128,6 +139,9 @@ class GateUpParallelLinear : public infinilm::nn::ColumnParallelLinear { bool up_bias_; RegisterParamFn register_fn_; std::vector split_infos_; + std::vector shard_specs_; + + void register_fused_params(); }; } // namespace infinilm::layers::linear diff --git a/csrc/layers/linear/linear.cpp b/csrc/layers/linear/linear.cpp index f24496700..eacf0a187 100644 --- a/csrc/layers/linear/linear.cpp +++ b/csrc/layers/linear/linear.cpp @@ -13,8 +13,9 @@ Linear::Linear(size_t in_features, size_t out_features, bool bias, Linear::Linear(size_t in_features, size_t out_features, std::shared_ptr quantization, - bool bias, const infinicore::DataType &dtype, const infinicore::Device &device) - : BaseLinear(in_features, out_features, quantization, bias, dtype, device, -1, 0, 1) { + bool bias, const infinicore::DataType &dtype, const infinicore::Device &device, + const std::string &stem) + : BaseLinear(in_features, out_features, quantization, bias, dtype, device, -1, 0, 1, -1, stem) { } infinicore::Tensor Linear::forward(infinicore::Tensor &input) const { @@ -42,9 +43,9 @@ ColumnParallelLinear::ColumnParallelLinear(size_t in_features, size_t out_featur std::shared_ptr quantization, bool bias, const infinicore::DataType &dtype, const infinicore::Device &device, infinicore::Size tp_rank, infinicore::Size tp_size, - int tp_num_heads) + int tp_num_heads, const std::string &stem) : BaseLinear(in_features, out_features, quantization, bias, dtype, device, - 0, tp_rank, tp_size, tp_num_heads), + 0, tp_rank, tp_size, tp_num_heads, stem), tp_rank_(tp_rank), tp_size_(tp_size) { } @@ -74,9 +75,9 @@ RowParallelLinear::RowParallelLinear(size_t in_features, size_t out_features, std::shared_ptr quantization, bool bias, const infinicore::DataType &dtype, const infinicore::Device &device, infinicore::Size tp_rank, infinicore::Size tp_size, - infinicclComm_t communicator) + infinicclComm_t communicator, const std::string &stem) : BaseLinear(in_features, out_features, quantization, bias, dtype, device, - 1, tp_rank, tp_size), + 1, tp_rank, tp_size, -1, stem), tp_rank_(tp_rank), tp_size_(tp_size), communicator_(communicator) { } diff --git a/csrc/layers/linear/linear.hpp b/csrc/layers/linear/linear.hpp index 566cee77c..9f1046097 100644 --- a/csrc/layers/linear/linear.hpp +++ b/csrc/layers/linear/linear.hpp @@ -21,7 +21,8 @@ class Linear : public BaseLinear { std::shared_ptr quantization, bool bias = true, const infinicore::DataType &dtype = infinicore::DataType::F32, - const infinicore::Device &device = infinicore::Device()); + const infinicore::Device &device = infinicore::Device(), + const std::string &stem = ""); infinicore::Tensor forward(infinicore::Tensor &input) const; std::string extra_repr() const; @@ -42,7 +43,8 @@ class ColumnParallelLinear : public BaseLinear { const infinicore::DataType &dtype = infinicore::DataType::F32, const infinicore::Device &device = infinicore::Device(), infinicore::Size tp_rank = 0, infinicore::Size tp_size = 1, - int tp_num_heads = -1); + int tp_num_heads = -1, + const std::string &stem = ""); infinicore::Tensor forward(infinicore::Tensor &input) const; std::string extra_repr() const; @@ -67,7 +69,8 @@ class RowParallelLinear : public BaseLinear { const infinicore::DataType &dtype = infinicore::DataType::F32, const infinicore::Device &device = infinicore::Device(), infinicore::Size tp_rank = 0, infinicore::Size tp_size = 1, - infinicclComm_t communicator = nullptr); + infinicclComm_t communicator = nullptr, + const std::string &stem = ""); infinicore::Tensor forward(infinicore::Tensor &input) const; std::string extra_repr() const; diff --git a/csrc/layers/mlp/mlp.cpp b/csrc/layers/mlp/mlp.cpp index f7604c505..3a8d5f255 100644 --- a/csrc/layers/mlp/mlp.cpp +++ b/csrc/layers/mlp/mlp.cpp @@ -5,7 +5,8 @@ namespace infinilm::layers::mlp { MLP::MLP(std::shared_ptr model_config, - const infinicore::Device &device) { + const infinicore::Device &device, + const std::string &prefix) { const auto &dtype{model_config->get_dtype()}; hidden_size_ = model_config->get("hidden_size"); @@ -20,10 +21,11 @@ MLP::MLP(std::shared_ptr model_config, auto register_fn = [this](const std::string &n, infinicore::nn::Parameter p) { this->register_parameter(n, std::move(p)); }; gate_up_proj_ = std::make_shared( hidden_size_, intermediate_size_, "gate_proj", "up_proj", register_fn, - quantization_method, use_bias_, dtype, device, rank_info); + quantization_method, use_bias_, dtype, device, rank_info, prefix); down_proj_ = this->register_module( "down_proj", intermediate_size_, hidden_size_, quantization_method, - use_bias_, dtype, device, tp_rank, tp_size, rank_info.comm); + use_bias_, dtype, device, tp_rank, tp_size, rank_info.comm, + prefix.empty() ? std::string() : prefix + ".down_proj."); } infinicore::Tensor MLP::forward(const infinicore::Tensor &hidden_states) const { diff --git a/csrc/layers/mlp/mlp.hpp b/csrc/layers/mlp/mlp.hpp index abd81bd88..7e19436e7 100644 --- a/csrc/layers/mlp/mlp.hpp +++ b/csrc/layers/mlp/mlp.hpp @@ -24,9 +24,12 @@ class MLP : public infinicore::nn::Module { * * @param model_config: Model configuration. * @param device Device to create tensors on + * @param prefix 本层在 checkpoint 里的路径（形如 "layers.0.mlp"）。只给需要 + * 按张量名查表的量化方案用（GGUF）；其他方案留空即可。 */ MLP(std::shared_ptr model_config, - const infinicore::Device &device); + const infinicore::Device &device, + const std::string &prefix = ""); /** * @brief Forward pass: compute MLP output diff --git a/csrc/layers/quantization/base_quantization.hpp b/csrc/layers/quantization/base_quantization.hpp index 4b17cc949..8dc94823b 100644 --- a/csrc/layers/quantization/base_quantization.hpp +++ b/csrc/layers/quantization/base_quantization.hpp @@ -62,6 +62,67 @@ class BaseQuantization : public std::enable_shared_from_this { float alpha = 1.0f) const = 0; + // ---- Name-aware variants ------------------------------------------------- + // Some schemes decide a parameter's layout from the *checkpoint tensor name* + // instead of from the module's (in_features, out_features) pair: GGUF block + // quantization has one ggml type per tensor, and `row_bytes` is a function of + // that type, so nothing can be derived from the logical shape alone. + // + // `stem` is the checkpoint path of the weight *without* the final tensor-name + // component, relative to quantization_config.key_prefix, and it always keeps + // the trailing separator: + // "layers.0.mlp.gate_proj." -> that one weight + // "layers.0.self_attn." -> ditto (probes weight / weight_bytes) + // "layers.0.self_attn" (no trailing '.') -> a fused linear: this scheme owns + // no buffer, each shard has its own checkpoint entry and is registered + // separately (see BaseLinear::init_fused_shards). + // An empty stem is always an error for such schemes. + // + // The default implementations forward to the name-less versions, so the + // existing quantization classes need no change. + virtual std::vector get_param_layout( + size_t in_features, size_t out_features, + int split_dim, int tp_rank, int tp_size, + int tp_num_heads, + const infinicore::DataType &dtype, + bool bias, + const std::string &stem) const { + return get_param_layout(in_features, out_features, split_dim, tp_rank, + tp_size, tp_num_heads, dtype, bias); + } + + virtual infinicore::Tensor forward( + const ParamsMap ¶ms, + const infinicore::Tensor &input, + bool has_bias, + float alpha, + const std::string &stem) const { + return forward(params, input, has_bias, alpha); + } + + // A fused linear's `stem` is only the *group* name (no trailing '.'), which is + // not enough for name-driven schemes: each shard has its own checkpoint entry + // and its own format (measured on Qwen3.8: 0 of 16 full-attn groups share one + // ggml type across q/k/v, 4 of 32 FFN groups across gate/up). + // + // `shard_stems[i]` is the checkpoint stem of the shard behind parameter key + // "shard." — both are produced by the same loop in + // BaseLinear::init_fused_shards, so index correspondence is a local invariant, + // not a cross-phase assumption. Empty vector = not a fused linear. + // + // Default forwards to the stem-only version, so quantization classes without + // per-shard formats need no change. + virtual infinicore::Tensor forward( + const ParamsMap ¶ms, + const infinicore::Tensor &input, + bool has_bias, + float alpha, + const std::string &stem, + const std::vector &shard_stems) const { + (void)shard_stems; + return forward(params, input, has_bias, alpha, stem); + } + virtual infinicore::Tensor forward_allreduce( const ParamsMap ¶ms, const infinicore::Tensor &input, diff --git a/csrc/layers/quantization/fp8.cpp b/csrc/layers/quantization/fp8.cpp new file mode 100644 index 000000000..7d96513c6 --- /dev/null +++ b/csrc/layers/quantization/fp8.cpp @@ -0,0 +1,184 @@ +#include "fp8.hpp" +#include "none_quantization.hpp" + +#include +#include +#include + +#include +#include + +namespace infinilm::quantization { + +std::vector FP8Quantization::get_param_layout( + size_t in_features, size_t out_features, + int split_dim, int tp_rank, int tp_size, + int /*tp_num_heads*/, + const infinicore::DataType &dtype, + bool bias) const { + + std::vector descs; + + // Weight: FP8 (E4M3) format - keep as F8, do NOT convert to BF16 + descs.push_back({"weight", {out_features, in_features}, + infinicore::DataType::F8, split_dim, tp_rank, tp_size}); + + // Per-block weight scale (inverse): BF16, shape = [ceil(N/128), ceil(K/128)] + size_t num_out_blocks = (out_features + BLOCK_SIZE - 1) / BLOCK_SIZE; + size_t num_in_blocks = (in_features + BLOCK_SIZE - 1) / BLOCK_SIZE; + descs.push_back({"weight_scale_inv", {num_out_blocks, num_in_blocks}, + infinicore::DataType::F32, split_dim, tp_rank, tp_size}); + + if (bias) { + descs.push_back({"bias", {out_features}, dtype, + split_dim >= 0 ? 0 : -1, + split_dim >= 0 ? tp_rank : 0, + split_dim >= 0 ? tp_size : 1}); + } + return descs; +} + +infinicore::Tensor FP8Quantization::forward( + const ParamsMap ¶ms, + const infinicore::Tensor &input, + bool has_bias, + float /*alpha*/) const { + + auto weight_it = params.find("weight"); + auto scale_it = params.find("weight_scale_inv"); + auto bias_it = params.find("bias"); + + if (weight_it == params.end()) { + throw std::runtime_error("FP8Quantization::forward: weight not found"); + } + if (scale_it == params.end()) { + throw std::runtime_error("FP8Quantization::forward: weight_scale_inv not found"); + } + + auto weight = weight_it->second; + auto scale = scale_it->second; + + // Ensure input, weight, and scale are contiguous + // (split_params creates narrow views that may not be contiguous) + auto x = input->is_contiguous() ? input : input->contiguous(); + auto w = weight->is_contiguous() ? weight : weight->contiguous(); + auto s = scale->is_contiguous() ? scale : scale->contiguous(); + + // Get dimensions + auto x_shape = x->shape(); + size_t ndim = x_shape.size(); + size_t K = x_shape[ndim - 1]; // last dim is always feature dim + // M = product of all leading dims + size_t M = 1; + for (size_t i = 0; i < ndim - 1; i++) { + M *= x_shape[i]; + } + auto w_shape = w->shape(); + size_t N = w_shape[0]; + + // Flatten input to 2D [M, K] and ensure contiguous + auto flat = x->view({M, K}); + flat = flat->is_contiguous() ? flat : flat->contiguous(); + + // Allocate output [M, N] + auto out = infinicore::Tensor::empty( + {M, N}, input->dtype(), input->device()); + + // Call block-FP8 linear: BF16 input x F8 weight + block scale -> BF16 output + infinicore::op::block_fp8_linear_( + out, flat, w, s); + + if (has_bias && bias_it != params.end()) { + auto bias = bias_it->second; + auto bias_broadcast = bias->view({1, N}); + infinicore::op::add_(out, out, bias_broadcast); + } + + // Reshape output to match input's leading dims with N + std::vector out_shape(x_shape.begin(), x_shape.end() - 1); + out_shape.push_back(N); + return out->view(out_shape); +} + +std::vector FP8Quantization::split_params( + const std::unordered_map ¶ms, + const std::vector &splits, + int narrow_dim, + int tp_rank, int tp_size, int /*tp_num_heads*/) const { + + std::vector result; + auto weight_it = params.find("weight"); + auto scale_it = params.find("weight_scale_inv"); + auto bias_it = params.find("bias"); + + for (const auto &s : splits) { + result.push_back({s.prefix + ".weight", + infinicore::nn::Parameter( + weight_it->second->narrow({{static_cast(narrow_dim), s.start, s.size}}), + narrow_dim, tp_rank, tp_size, s.num_shards)}); + + if (scale_it != params.end()) { + size_t scale_start = s.start / BLOCK_SIZE; + size_t scale_size = (s.size + BLOCK_SIZE - 1) / BLOCK_SIZE; + result.push_back({s.prefix + ".weight_scale_inv", + infinicore::nn::Parameter( + scale_it->second->narrow({{static_cast(narrow_dim), scale_start, scale_size}}), + narrow_dim, tp_rank, tp_size, s.num_shards)}); + } + + if (bias_it != params.end()) { + result.push_back({s.prefix + ".bias", + infinicore::nn::Parameter( + bias_it->second->narrow({{0, s.start, s.size}}), + 0, tp_rank, tp_size, s.num_shards)}); + } + } + return result; +} + +std::shared_ptr FP8Quantization::process_weights_after_loading( + ParamsMap ¶ms, + const infinicore::Device &device, + int /*split_dim*/) const { + + auto weight_it = params.find("weight"); + auto scale_it = params.find("weight_scale_inv"); + + if (weight_it == params.end()) { + return nullptr; + } + if (scale_it == params.end()) { + spdlog::debug("FP8: no weight_scale_inv found, skipping"); + return nullptr; + } + + auto weight = weight_it->second; + auto scale = scale_it->second; + + size_t out_features = weight->shape()[0]; + size_t in_features = weight->shape()[1]; + + size_t num_out_blocks = (out_features + BLOCK_SIZE - 1) / BLOCK_SIZE; + size_t num_in_blocks = (in_features + BLOCK_SIZE - 1) / BLOCK_SIZE; + + auto scale_shape = scale->shape(); + if (scale_shape.size() != 2 || + scale_shape[0] != num_out_blocks || + scale_shape[1] != num_in_blocks) { + throw std::runtime_error("FP8Quantization: weight_scale_inv shape mismatch"); + } + + // Keep weight as FP8 (1 byte/element), just ensure contiguous + params["weight"] = weight->contiguous(); + + // Scale is already FP32 (converted on Python side during loading) + params["weight_scale_inv"] = scale->contiguous(); + + spdlog::debug("FP8: kept weight as F8, scale cast to F32, shape=[{}, {}]", + out_features, in_features); + + // Return nullptr to continue using FP8Quantization (not NoneQuantization) + return nullptr; +} + +} // namespace infinilm::quantization diff --git a/csrc/layers/quantization/fp8.hpp b/csrc/layers/quantization/fp8.hpp new file mode 100644 index 000000000..7897852ba --- /dev/null +++ b/csrc/layers/quantization/fp8.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include "base_quantization.hpp" + +namespace infinilm::quantization { + +class FP8Quantization : public BaseQuantization { +public: + explicit FP8Quantization(const nlohmann::json &quant_config) + : BaseQuantization(quant_config) {} + + QuantScheme get_quant_scheme() const override { + return QuantScheme::FP8_W8A8; + } + + std::vector get_param_layout( + size_t in_features, size_t out_features, + int split_dim, int tp_rank, int tp_size, + int tp_num_heads, + const infinicore::DataType &dtype, + bool bias) const override; + + infinicore::Tensor forward( + const ParamsMap ¶ms, + const infinicore::Tensor &input, + bool has_bias, + float alpha = 1.0f) const override; + + std::vector split_params( + const std::unordered_map ¶ms, + const std::vector &splits, + int narrow_dim, + int tp_rank, int tp_size, int tp_num_heads) const override; + + std::shared_ptr process_weights_after_loading( + ParamsMap ¶ms, + const infinicore::Device &device, + int split_dim = -1) const override; + +private: + static constexpr size_t BLOCK_SIZE = 128; +}; + +} // namespace infinilm::quantization diff --git a/csrc/layers/quantization/gguf.cpp b/csrc/layers/quantization/gguf.cpp new file mode 100644 index 000000000..803593075 --- /dev/null +++ b/csrc/layers/quantization/gguf.cpp @@ -0,0 +1,606 @@ +#include "gguf.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace infinilm::quantization { + +namespace { + +// ggml 块的 (block_size, type_size)：一行 row_bytes = in / block_size * type_size。 +// 数值取自 §2.3 的实测（与 gguf-py 的 GGML_QUANT_SIZES、llama.cpp 的 ggml.h 一致）。 +// 只列本路线 kernel 计划支持的类型；表外的 id 一律抛错，逼着打包期把它稠密化， +// 而不是运行期猜一个 stride（猜错 = 读越界 = 结果错）。 +struct GgmlBlock { + int64_t id; + const char *name; + size_t block_size; + size_t type_size; +}; + +constexpr GgmlBlock GGML_BLOCKS[] = { + {8, "Q8_0", 32, 34}, + {12, "Q4_K", 256, 144}, + {13, "Q5_K", 256, 176}, + {14, "Q6_K", 256, 210}, +}; + +const GgmlBlock *ggml_block(int64_t id) { + for (const auto &b : GGML_BLOCKS) { + if (b.id == id) { + return &b; + } + } + return nullptr; +} + +std::string supported_types() { + std::string s; + for (const auto &b : GGML_BLOCKS) { + if (!s.empty()) { + s += "/"; + } + s += b.name; + } + return s; +} + +constexpr const char *DENSE_MARK = "dense_bf16"; + +bool env_enabled(const char *name) { + const char *value = std::getenv(name); + return value != nullptr && value[0] != '\0' && std::strcmp(value, "0") != 0; +} + +bool use_f32_decode_output(const std::string &table_key, size_t m_count) { + if (!env_enabled("INFINI_GGUF_F32_DECODE_OUT") || m_count > 16) { + return false; + } + const char *match = std::getenv("INFINI_GGUF_F32_DECODE_OUT_MATCH"); + return match == nullptr || match[0] == '\0' + || table_key.find(match) != std::string::npos; +} + +} // namespace + +GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) + : BaseQuantization(quant_config) { + if (!quant_config_.is_object() || !quant_config_.contains("ggml_types")) { + throw std::runtime_error( + "GGUFBlockQuantization: quantization_config 缺 ggml_types（阶段 1 打包器写入）"); + } + key_prefix_ = get_or("key_prefix", ""); + + const auto &table = quant_config_.at("ggml_types"); + if (!table.is_object() || table.empty()) { + throw std::runtime_error("GGUFBlockQuantization: ggml_types 表为空"); + } + + size_t n_blob = 0; + size_t n_dense = 0; + size_t n_outside = 0; + for (const auto &kv : table.items()) { + const std::string &name = kv.key(); + // 实测：产物 121 张量里只有 lm_head.weight 不在 model.language_model. 子树下 + // （lm_head 在 C++ 模块树里是根节点的兄弟），这类键原样保留、不裁前缀。 + // 它们永远不会被 stem 查到（lm_head 走非量化 ctor），留着是为了让类型表 + // 与产物张量名保持双向逐字相等（阶段 1 自检的判据）。 + std::string key = name; + if (!key_prefix_.empty() && name.compare(0, key_prefix_.size(), key_prefix_) == 0) { + key = name.substr(key_prefix_.size()); + } else { + ++n_outside; + } + + int64_t id = DENSE_BF16; + if (kv.value().is_string()) { + const std::string v = kv.value().get(); + if (v != DENSE_MARK) { + throw std::runtime_error( + "GGUFBlockQuantization: '" + name + "' 的取值 '" + v + "' 既不是整数 type id 也不是 \"" + + DENSE_MARK + "\""); + } + ++n_dense; + } else { + if (!kv.value().is_number_integer()) { + throw std::runtime_error( + "GGUFBlockQuantization: '" + name + "' 的取值不是整数 ggml type id"); + } + id = kv.value().get(); + if (id == DENSE_BF16) { + throw std::runtime_error( + "GGUFBlockQuantization: '" + name + "' 的 type id 与稠密标记 -1 冲突"); + } + if (!ggml_block(id)) { + throw std::runtime_error( + "GGUFBlockQuantization: '" + name + "' 是不支持的 ggml type id=" + + std::to_string(id) + "（当前支持 " + supported_types() + + "；其余类型必须在打包期稠密化，不能留到运行期猜）"); + } + ++n_blob; + } + + if (!types_.emplace(std::move(key), TypeEntry{id, name}).second) { + throw std::runtime_error( + "GGUFBlockQuantization: 裁掉 key_prefix 后键重复：'" + name + "'"); + } + } + + // 激活 V 头置换规则（out_proj 一类「要置换的是权重列」的条目）。缺这个键 = 拒启， + // 不静默不置换：漏一次置换 = 48 个 value head 与权重列整体错位 + // =「能加载、能跑、输出错」，正是阶段 4 §8.5 要排除的那一类错。 + if (!quant_config_.contains("activation_vperm")) { + throw std::runtime_error( + "GGUFBlockQuantization: quantization_config 缺 activation_vperm（out_proj 的 V 头" + "列序置换规则）——旧产物用打包器 --skip-pack 刷新 config.json 即可，不必重打包权重"); + } + { + const auto &rules = quant_config_.at("activation_vperm"); + if (!rules.is_array()) { + throw std::runtime_error("GGUFBlockQuantization: activation_vperm 必须是数组，实际是 " + + std::string(rules.type_name())); + } + for (const auto &j : rules) { + if (!j.is_object()) { + throw std::runtime_error("GGUFBlockQuantization: activation_vperm 条目不是对象"); + } + ActVPerm r; + for (const char *key : {"suffix", "num_k_heads", "num_v_per_k", "head_dim"}) { + if (!j.contains(key)) { + throw std::runtime_error("GGUFBlockQuantization: activation_vperm 条目缺 '" + + std::string(key) + "'"); + } + } + r.suffix = j.at("suffix").get(); + r.n_k = j.at("num_k_heads").get(); + r.r = j.at("num_v_per_k").get(); + r.hd = j.at("head_dim").get(); + if (r.suffix.empty() || r.suffix.back() != '.' || !r.n_k || !r.r || !r.hd) { + throw std::runtime_error( + "GGUFBlockQuantization: activation_vperm 条目不合法：suffix='" + r.suffix + + "' 需以 '.' 结尾，三个维度需为正（实际 " + std::to_string(r.n_k) + "/" + + std::to_string(r.r) + "/" + std::to_string(r.hd) + "）"); + } + if (std::any_of(vperm_.begin(), vperm_.end(), + [&r](const ActVPerm &e) { return e.suffix == r.suffix; })) { + throw std::runtime_error("GGUFBlockQuantization: activation_vperm 里 '" + r.suffix + + "' 出现多次（同一条规则只能有一份）"); + } + vperm_.push_back(std::move(r)); + } + } + + // n_outside 有两种成因，得分开说：早先全量产物未声明 key_prefix，整张表都被计入 + // 「前缀外」（实测日志里印成「前缀外 947」），很容易被读成「947 条都查不到」。 + spdlog::info( + "GGUF block quantization: 类型表 {} 条（blob {} / 稠密 {} / 未裁前缀 {}），key_prefix='{}'{}", + types_.size(), n_blob, n_dense, n_outside, key_prefix_, + key_prefix_.empty() + ? "（未声明：表键即 safetensors 张量名的相对形态，整表不裁前缀）" + : "（在 prefix 之外，如 lm_head）"); + + // 与下一行一起构成「本次加载到底有没有在做置换」的唯一可 grep 证据（A/B 靠它） + std::string vs; + for (const auto &r : vperm_) { + if (!vs.empty()) { + vs += ", "; + } + vs += r.suffix + "=" + std::to_string(r.n_k) + "x" + std::to_string(r.r) + "x" + + std::to_string(r.hd); + } + spdlog::info("GGUF block quantization: 激活 V 头置换规则 {} 条（grouped->tiled）：{}", + vperm_.size(), vs.empty() ? "无" : vs); +} + +GGUFBlockQuantization::~GGUFBlockQuantization() { + if (n_blob_ + n_dense_ + n_group_ > 0) { + spdlog::info("GGUF block quantization: 布局查表命中 blob {} / 稠密 {} / 融合组 {}", + n_blob_, n_dense_, n_group_); + } +} + +bool GGUFBlockQuantization::is_known_type(int64_t type_id) { + return ggml_block(type_id) != nullptr; +} + +std::string GGUFBlockQuantization::describe(const std::string &stem) const { + // 报错信息里拼回绝对名，方便直接在产物 / pack_report.json 里 grep + return (stem.empty() ? std::string("<空 stem>") : key_prefix_ + stem); +} + +int64_t GGUFBlockQuantization::resolve(const std::string &stem, std::string *matched_key) const { + const std::string blob_key = stem + BLOB_SUFFIX; + const std::string dense_key = stem + DENSE_SUFFIX; + const auto blob_it = types_.find(blob_key); + const auto dense_it = types_.find(dense_key); + const int hits = (blob_it != types_.end()) + (dense_it != types_.end()); + + // 命中 0 个 = 拼错或产物缺张量；命中 2 个 = 打包器同时写了 blob 与稠密版本。 + // 两种都必须是异常：任何「查不到就走稠密」的回落都会变成能加载、显存暴涨、结果错。 + if (hits != 1) { + throw std::runtime_error( + "GGUFBlockQuantization: stem '" + describe(stem) + "' 在类型表里命中 " + + std::to_string(hits) + " 个候选（期望恰好 1 个：'" + blob_key + "' 或 '" + dense_key + + "'）；表共 " + std::to_string(types_.size()) + " 条"); + } + const auto &hit = blob_it != types_.end() ? *blob_it : *dense_it; + if (matched_key) { + *matched_key = hit.second.name; + } + return hit.second.id; +} + +bool GGUFBlockQuantization::has_group(const std::string &group_stem) const { + const std::string head = group_stem + "."; + return std::any_of(types_.begin(), types_.end(), [&head](const auto &kv) { + return kv.first.compare(0, head.size(), head) == 0; + }); +} + +size_t GGUFBlockQuantization::row_bytes(size_t in_features, int64_t type_id) const { + const GgmlBlock *b = ggml_block(type_id); + if (!b) { + throw std::runtime_error( + "GGUFBlockQuantization: 不支持的 ggml type id=" + std::to_string(type_id) + + "（当前支持 " + supported_types() + "）"); + } + if (in_features % b->block_size != 0) { + throw std::runtime_error( + "GGUFBlockQuantization: in_features=" + std::to_string(in_features) + + " 不能被 " + b->name + " 的块大小 " + std::to_string(b->block_size) + " 整除"); + } + return in_features / b->block_size * b->type_size; +} + +const GGUFBlockQuantization::ActVPerm *GGUFBlockQuantization::vperm_rule( + const std::string &stem) const { + for (const auto &r : vperm_) { + if (stem.size() >= r.suffix.size() && + stem.compare(stem.size() - r.suffix.size(), r.suffix.size(), r.suffix) == 0) { + return &r; + } + } + return nullptr; +} + +infinicore::Tensor GGUFBlockQuantization::gather_grouped_to_tiled( + const ActVPerm &rule, const infinicore::Tensor &input, const std::string &name) { + const auto shape = input->shape(); + const size_t ndim = shape.size(); + if (ndim < 2) { + throw std::runtime_error( + "GGUFBlockQuantization: " + name + " 的激活 rank=" + std::to_string(ndim) + + "，至少要是 [..., in_features]"); + } + const size_t K = shape[ndim - 1]; + const size_t want = rule.n_k * rule.r * rule.hd; + if (K != want) { + // TP 会把 in 维切成没关头数不等的分片，套上整头置换就是静默错位； + // 与 get_param_layout 里「暂不支持 tensor parallel」的护栏保持同一口径。 + throw std::runtime_error( + "GGUFBlockQuantization: " + name + " 的激活末维 " + std::to_string(K) + + " != activation_vperm 的 num_k_heads*num_v_per_k*head_dim = " + std::to_string(want) + + "（切分后的分片不能套整头置换）"); + } + // [..., n_k, r, hd] -> [..., r, n_k, hd]：把 grouped（k-major）的激活置换为 tiled（v-major）。 + const size_t k_axis = ndim - 1; + infinicore::Shape grouped(shape.begin(), shape.end() - 1); + grouped.insert(grouped.end(), {rule.n_k, rule.r, rule.hd}); + infinicore::Shape order; + order.reserve(grouped.size()); + for (size_t a = 0; a + 1 < ndim; ++a) { + order.push_back(a); + } + order.insert(order.end(), {k_axis + 1, k_axis, k_axis + 2}); + + auto x = input->is_contiguous() ? input : input->contiguous(); + return x->view(grouped)->permute(order)->contiguous()->view(shape); +} + +std::vector GGUFBlockQuantization::get_param_layout( + size_t, size_t, int, int, int, int, + const infinicore::DataType &, bool) const { + throw std::runtime_error( + "GGUFBlockQuantization: 不接受无名字的 get_param_layout 调用（每个权重的 ggml " + "类型只能由 checkpoint 张量名决定）"); +} + +std::vector GGUFBlockQuantization::get_param_layout( + size_t in_features, size_t out_features, + int split_dim, int tp_rank, int tp_size, + int tp_num_heads, + const infinicore::DataType &dtype, + bool bias, + const std::string &stem) const { + (void)tp_num_heads; + + if (stem.empty()) { + throw std::runtime_error( + "GGUFBlockQuantization: 构造 Linear 时没有传 checkpoint stem（in=" + + std::to_string(in_features) + ", out=" + std::to_string(out_features) + + "）——方案 §6.1 列出的构造点必须全部补上 prefix/stem"); + } + if (tp_size != 1 || tp_rank != 0) { + throw std::runtime_error( + "GGUFBlockQuantization: 暂不支持 tensor parallel（blob 的 TP 切分留待多卡阶段）：" + + describe(stem)); + } + if (bias) { + throw std::runtime_error( + "GGUFBlockQuantization: GGUF 产物里没有 bias 张量：" + describe(stem)); + } + + // 不带结尾 '.' 的 stem 表示「融合 Linear」：本类不为它分配任何 buffer， + // 各 shard 由 BaseLinear::init_fused_shards 用各自的 stem 单独申请。 + if (stem.back() != '.') { + if (!has_group(stem)) { + throw std::runtime_error( + "GGUFBlockQuantization: 融合组 stem '" + stem + + "' 在类型表里没有任何 '" + stem + "..*' 条目"); + } + ++n_group_; + return {}; + } + + const int64_t id = resolve(stem); + if (id == DENSE_BF16) { + ++n_dense_; + // 与 NoneQuantization 同形：打包期已反量化成 BF16，正常 GEMM + return {{"weight", {out_features, in_features}, dtype, split_dim, tp_rank, tp_size}}; + } + + ++n_blob_; + const size_t rb = row_bytes(in_features, id); + return {{{BLOB_SUFFIX}, {out_features, rb}, infinicore::DataType::U8, split_dim, tp_rank, tp_size}}; +} + +infinicore::Tensor GGUFBlockQuantization::forward( + const ParamsMap &, const infinicore::Tensor &, bool, float) const { + throw std::runtime_error( + "GGUFBlockQuantization: 不接受无名字的 forward 调用（每个权重的 ggml 类型只能由 " + "checkpoint 名字决定，融合 Linear 还需要 shard_stems）"); +} + +infinicore::Tensor GGUFBlockQuantization::forward_shard( + const std::string &suffix, + const infinicore::Tensor &weight, + const infinicore::Tensor &input, + float alpha, + int64_t type_id, + const std::string &table_key) const { + if (suffix == DENSE_SUFFIX) { + // 参数后缀是 get_param_layout 按 resolve() 结果选的，两者不一致 = 有地方改坏了 + //（blob 被当成 BF16 读就是「能加载、结果错」），宁可抛。 + if (type_id != DENSE_BF16) { + throw std::runtime_error( + "GGUFBlockQuantization: " + table_key + " 的参数后缀是 " + DENSE_SUFFIX + + "，但类型表给出的 ggml type id=" + std::to_string(type_id) + "（不一致）"); + } + auto x = input->is_contiguous() ? input : input->contiguous(); + auto w = weight->is_contiguous() ? weight : weight->contiguous(); + return infinicore::op::linear(x, w, std::nullopt, alpha); + } + if (suffix == BLOB_SUFFIX) { + // 权重保持量化形态：块字节直接喂 kernel。这里绝不静默回落稠密 GEMM—— + // 那等于把块字节当成 BF16 读，能跑完但结果是错的，宁可抛。 + if (alpha != 1.0F) { + throw std::runtime_error( + "linear_gguf: 不支持 alpha=" + std::to_string(alpha) + + "（GGUF blob 路径没有缩放权重，alpha!=1 说明上层期望与实现不符）：" + + table_key); + } + auto x = input->is_contiguous() ? input : input->contiguous(); + auto w = weight->is_contiguous() ? weight : weight->contiguous(); + + const auto x_shape = x->shape(); + const size_t ndim = x_shape.size(); + const size_t K = x_shape[ndim - 1]; + size_t M = 1; + for (size_t i = 0; i + 1 < ndim; ++i) { + M *= x_shape[i]; + } + const size_t N = static_cast(w->size(0)); + // 这里不再设 M 上限：gemv（小 M）与 prefill（大 M）两条路径在 + // linear_gguf 算子内部按同一个 kMaxDecodeM 谓词选。上层再留一份数字， + // 两边一旦不同步就只剩一条过时的门（阶段 3.3 之前正是这种情形）。 + + auto flat = x->view({M, K}); + flat = flat->is_contiguous() ? flat : flat->contiguous(); + const bool f32_decode_out = use_f32_decode_output(table_key, M); + const auto out_dtype = f32_decode_out + ? infinicore::DataType::F32 + : input->dtype(); + auto out = infinicore::Tensor::empty({M, N}, out_dtype, input->device()); + // 只报第一个 blob 调用：端到端排障时区分「死在 blob 路径之前」与 + //「已在 kernel 里」，两者处置完全不同（前者是接线问题，后者是下游算子）。 + static std::atomic blob_calls{0}; + if (blob_calls.fetch_add(1) == 0) { + spdlog::info( + "linear_gguf: 首个 blob 前向 {} — M={} N={} K={} ggml_type={} row_bytes={}", + table_key, M, N, K, type_id, w->size(1)); + } + if (f32_decode_out) { + static std::atomic f32_calls{0}; + if (f32_calls.fetch_add(1) == 0) { + spdlog::warn( + "linear_gguf: 实验性 F32 decode 输出已启用，首个命中 {} — M={} N={} K={}", + table_key, M, N, K); + } + } + infinicore::op::linear_gguf_(out, flat, w, type_id); + + std::vector out_shape(x_shape.begin(), x_shape.end() - 1); + out_shape.push_back(N); + return out->view(out_shape); + } + throw std::runtime_error( + "GGUFBlockQuantization: " + table_key + " 的参数后缀 '" + suffix + + "' 既不是 " + DENSE_SUFFIX + " 也不是 " + BLOB_SUFFIX); +} + +infinicore::Tensor GGUFBlockQuantization::forward( + const ParamsMap ¶ms, + const infinicore::Tensor &input, + bool has_bias, + float alpha, + const std::string &stem) const { + // 没有 shard stems 就只能服务非融合布局；融合 Linear 走下面那个重载。 + return forward(params, input, has_bias, alpha, stem, {}); +} + +infinicore::Tensor GGUFBlockQuantization::forward( + const ParamsMap ¶ms, + const infinicore::Tensor &input, + bool has_bias, + float alpha, + const std::string &stem, + const std::vector &shard_stems) const { + if (has_bias) { + throw std::runtime_error( + "GGUFBlockQuantization: 不支持 bias（" + describe(stem) + "）"); + } + + // 先按规则置换激活，再进 blob / 稠密两条路：两条路的权重列序都直接来自同一个 GGUF + // 张量（稠密化只换 dtype 不动列序），需要置换的语义完全一致。 + infinicore::Tensor x = input; + const ActVPerm *rule = vperm_rule(stem); + if (!shard_stems.empty()) { + for (const auto &s : shard_stems) { + if (vperm_rule(s)) { + throw std::runtime_error( + "GGUFBlockQuantization: 融合组 '" + describe(stem) + "' 的 shard '" + + describe(s) + "' 命中激活置换规则，但一根 input 同时服务于所有 shard，" + "无法按 shard 分别置换（实际产物里 out_proj 不是融合 Linear，走到这里=接线错）"); + } + } + } else if (rule) { + x = gather_grouped_to_tiled(*rule, input, describe(stem)); + // 只报第一次：端到端排障时它是「gather 真的在跑」的唯一证据，不靠日志量堆 + static std::atomic vperm_applied{0}; + if (vperm_applied.fetch_add(1) == 0) { + spdlog::info( + "linear_gguf: 首个激活 V 头置换 {} — grouped->tiled {}x{}x{}", + describe(stem), rule->n_k, rule->r, rule->hd); + } + } + + // 非融合：一个参数（weight 或 weight_bytes），stem 就是它自己的完整 checkpoint 路径 + if (shard_stems.empty()) { + if (params.size() != 1) { + throw std::runtime_error( + "GGUFBlockQuantization: " + describe(stem) + " 有 " + + std::to_string(params.size()) + " 个参数却没收到 shard_stems" + "（内部错误：BaseLinear::compute_linear 没有把 shard_stems_ 传下来）"); + } + const auto &kv = *params.begin(); + std::string table_key; + const int64_t id = resolve(stem, &table_key); + return forward_shard(kv.first, kv.second, x, alpha, id, table_key); + } + + // 融合：parameters_ 里是 shard.，i 就是它们在输出 dim(-1) 上的顺序， + // 与融合 Linear 的 SplitInfo 顺序一致 —— 所以输出拼回一根连续的 [.., sum(out_i)]， + // 上层的 narrow 逻辑完全不用改（方案 §6.0 纠正 1）。 + // 每个 shard 的 ggml 类型由 shard_stems[i] 查表（实测 q/k/v 不同类型，见 §7.2）。 + if (shard_stems.size() != params.size()) { + throw std::runtime_error( + "GGUFBlockQuantization: " + describe(stem) + " 有 " + + std::to_string(params.size()) + " 个 shard 参数但收到 " + + std::to_string(shard_stems.size()) + " 个 shard stem（内部错误：两者应在 " + "BaseLinear::init_fused_shards 的同一个循环里产生）"); + } + std::vector> parts; + for (const auto &kv : params) { + if (kv.first.compare(0, std::string(SHARD_PREFIX).size(), SHARD_PREFIX) != 0) { + throw std::runtime_error( + "GGUFBlockQuantization: 融合 Linear 的参数名 '" + kv.first + + "' 不是 " + SHARD_PREFIX + ". 形式（" + describe(stem) + "）"); + } + const size_t dot = kv.first.find('.'); + if (dot == std::string::npos) { + throw std::runtime_error( + "GGUFBlockQuantization: 融合 Linear 的参数名 '" + kv.first + "' 缺 '.'"); + } + const size_t idx = std::stoul(kv.first.substr(std::string(SHARD_PREFIX).size(), + dot - std::string(SHARD_PREFIX).size())); + if (idx >= shard_stems.size()) { + throw std::runtime_error( + "GGUFBlockQuantization: 参数名 '" + kv.first + "' 的 shard 下标越出 shard_stems（" + + describe(stem) + "）"); + } + std::string table_key; + const int64_t id = resolve(shard_stems[idx], &table_key); + parts.emplace_back(idx, forward_shard(kv.first.substr(dot + 1), kv.second, x, alpha, + id, table_key)); + } + std::sort(parts.begin(), parts.end(), + [](const auto &a, const auto &b) { return a.first < b.first; }); + + std::vector outs; + outs.reserve(parts.size()); + for (auto &p : parts) { + outs.push_back(p.second); + } + const auto shape = input->shape(); + return infinicore::op::cat(outs, static_cast(shape.size()) - 1); +} + +std::vector GGUFBlockQuantization::split_params( + const std::unordered_map ¶ms, + const std::vector &splits, + int, int, int, int) const { + // 恒等映射：GGUF 的融合 Linear 已经按 shard 分配了独立 buffer（没有可 narrow 的父 + // buffer），这里只把 shard. 换成 . 交给 register_fn。 + std::vector result; + for (size_t i = 0; i < splits.size(); ++i) { + const std::string head = std::string(SHARD_PREFIX) + std::to_string(i) + "."; + for (const auto &kv : params) { + if (kv.first.compare(0, head.size(), head) != 0) { + continue; + } + result.push_back({splits[i].prefix + "." + kv.first.substr(head.size()), + infinicore::nn::Parameter(kv.second)}); + } + } + if (result.size() != splits.size()) { + throw std::runtime_error( + "GGUFBlockQuantization::split_params: " + std::to_string(splits.size()) + + " 个 shard 只匹配到 " + std::to_string(result.size()) + + " 个参数（GGUF 融合 Linear 应走 BaseLinear::init_fused_shards）"); + } + return result; +} + +std::shared_ptr GGUFBlockQuantization::process_weights_after_loading( + ParamsMap ¶ms, + const infinicore::Device &, + int) const { + for (auto &kv : params) { + const bool is_blob = kv.first.size() >= strlen(BLOB_SUFFIX) && + kv.first.compare(kv.first.size() - strlen(BLOB_SUFFIX), + strlen(BLOB_SUFFIX), BLOB_SUFFIX) == 0; + if (!is_blob) { + continue; + } + if (kv.second->dtype() != infinicore::DataType::U8) { + throw std::runtime_error( + "GGUFBlockQuantization: blob 参数 '" + kv.first + "' 的 dtype 不是 U8"); + } + if (!kv.second->is_contiguous()) { + throw std::runtime_error( + "GGUFBlockQuantization: blob 参数 '" + kv.first + "' 不连续（阶段 3 kernel 按行取字节）"); + } + } + // 返回 nullptr：不换方案、不改写字节 + return nullptr; +} + +} // namespace infinilm::quantization diff --git a/csrc/layers/quantization/gguf.hpp b/csrc/layers/quantization/gguf.hpp new file mode 100644 index 000000000..0a9eb475b --- /dev/null +++ b/csrc/layers/quantization/gguf.hpp @@ -0,0 +1,157 @@ +#pragma once + +#include "base_quantization.hpp" + +#include +#include +#include + +namespace infinilm::quantization { + +// GGUF block quantization（路线 B）：打包器把 GGUF 张量的**原始块字节**逐字节搬进 +// safetensors，一行的宽度是 row_bytes(in_features, type)，不是 in_features 个元素。 +// 因此每个权重的布局只能由 checkpoint 张量名查表决定，逻辑形状推不出来。 +// +// 类型表 = config.json:quantization_config.ggml_types，键就是 safetensors 里的张量名 +// 原文（打包器自检保证与产物张量名双向逐字相等），值要么是 ggml type id，要么是 +// 字符串 "dense_bf16"（打包期已反量化成 BF16 的那些：embed / lm_head / norm / +// GDN 标量 / v1 的 IQ4_*）。quantization_config.key_prefix 在这里裁掉一次，因为 +// 挂在 model. 以下的模块不知道自己的绝对路径。详见执行方案 §2.3 / §6.0。 +class GGUFBlockQuantization : public BaseQuantization { +public: + // 稠密化条目在类型表里的取值（与任何 ggml type id 都不冲突：id 从 0 起） + static constexpr int64_t DENSE_BF16 = -1; + // blob 权重在 checkpoint 里的张量名后缀（与 scripts/gguf_mapping.BLOB_SUFFIX 一致） + static constexpr const char *BLOB_SUFFIX = "weight_bytes"; + static constexpr const char *DENSE_SUFFIX = "weight"; + // 融合 Linear 在 parameters_ 里给各 shard 用的 key 前缀，见 BaseLinear::init_fused_shards + static constexpr const char *SHARD_PREFIX = "shard"; + + explicit GGUFBlockQuantization(const nlohmann::json &quant_config); + + ~GGUFBlockQuantization() override; + + QuantScheme get_quant_scheme() const override { + return QuantScheme::GGUF_BLOCK; + } + + // 名称未知的布局无法决定 ggml 类型，GGUF 只能通过带 stem 的重载被调用 + std::vector get_param_layout( + size_t in_features, size_t out_features, + int split_dim, int tp_rank, int tp_size, + int tp_num_heads, + const infinicore::DataType &dtype, + bool bias) const override; + + std::vector get_param_layout( + size_t in_features, size_t out_features, + int split_dim, int tp_rank, int tp_size, + int tp_num_heads, + const infinicore::DataType &dtype, + bool bias, + const std::string &stem) const override; + + infinicore::Tensor forward( + const ParamsMap ¶ms, + const infinicore::Tensor &input, + bool has_bias, + float alpha = 1.0f) const override; + + infinicore::Tensor forward( + const ParamsMap ¶ms, + const infinicore::Tensor &input, + bool has_bias, + float alpha, + const std::string &stem) const override; + + // 融合 Linear 的唯一入口：各 shard 的 ggml type id 只能由自己的 stem 查出来 + //（实测 q/k/v 同类型的 full-attn 层数 0/16），而组 stem 做不到。见 §7.2 子步骤 0。 + infinicore::Tensor forward( + const ParamsMap ¶ms, + const infinicore::Tensor &input, + bool has_bias, + float alpha, + const std::string &stem, + const std::vector &shard_stems) const override; + + // GGUF 的融合 Linear 不在 base_linear 里走这条路（各 shard 本来就是独立 buffer， + // 没有可 narrow 的父 buffer），这里只做「shard -> .」的名字映射， + // 字节一个不动，供 BaseLinear::split_params 的既有调用点安全通过。 + std::vector split_params( + const std::unordered_map ¶ms, + const std::vector &splits, + int narrow_dim, + int tp_rank, int tp_size, int tp_num_heads) const override; + + // 不改写任何字节：blob 的语义就是「GGUF 原始字节」，一旦被 post-process + // 加工就失去与 llama.cpp 逐 block 对拍的能力（方案 §4 的基准）。 + std::shared_ptr process_weights_after_loading( + ParamsMap ¶ms, + const infinicore::Device &device, + int split_dim = -1) const override; + + // ---- 供自检 / 诊断使用 ---- + // stem -> ggml type id 或 DENSE_BF16。命中 0 个或 2 个候选都抛错：宁可拒启， + // 也不能静默走稠密路径（能加载、显存暴涨、结果错）。 + // matched_key 非空时额外给出表里真正命中的那条键（已裁前缀的形态），报错里用它 + // 才能 grep 到；旧形态产物的 blob 键是归一成 `.weight` 的，不能拿 stem 拼凑。 + int64_t resolve(const std::string &stem, std::string *matched_key = nullptr) const; + size_t row_bytes(size_t in_features, int64_t type_id) const; + bool has_group(const std::string &group_stem) const; + size_t table_size() const { return types_.size(); } + + static bool is_known_type(int64_t type_id); + +private: + // type_id = 本权重在类型表里的 ggml type id（稠密条目为 DENSE_BF16），阶段 3 的 + // kernel 分发靠它；table_key = 命中的表键（报错里给的名字必须能 grep 到）。 + infinicore::Tensor forward_shard( + const std::string &suffix, + const infinicore::Tensor &weight, + const infinicore::Tensor &input, + float alpha, + int64_t type_id, + const std::string &table_key) const; + + // 运行时激活 V 头置换（out_proj 一类「权重列需要重排」的条目）。 + // 为什么必须在运行时做：conversion/qwen.py:607-609 导出 GGUF 时把 ssm_out 的**列** + // 从 grouped 换成了 tiled，而 GDN kernel 的 v 头序是 grouped（InfiniCore + // chunk_gated_delta_rule/cuda/kernel.cuh:112 `key_head_idx = value_head_idx / + // value_heads_per_key_head`）；blob 的块沿 in 维切（Q4_K/Q5_K/Q6_K block_size=256）， + // 打包期置换列 = 跨块重排 = 要重量化，做不到 ⇒ 只能把激活置换过去。 + // 规则不在这里硬编码，由打包器从映射表派生写进 + // config.json:quantization_config.activation_vperm（见 scripts/gguf_mapping.py）。 + struct ActVPerm { + std::string suffix; // 尾匹配用，含结尾 '.'，例如 "linear_attn.out_proj." + size_t n_k; // key 头数 + size_t r; // 每个 key 头带几个 value 头 + size_t hd; // value head_dim + }; + + // stem 命中哪条规则（没有则 nullptr）。按后缀匹配，因为层号在 C++ 侧不可信。 + const ActVPerm *vperm_rule(const std::string &stem) const; + + // [..., n_k*r*hd]（grouped）-> [..., r*n_k*hd]（tiled），纯视图 + 一次 contiguous + static infinicore::Tensor gather_grouped_to_tiled( + const ActVPerm &rule, const infinicore::Tensor &input, const std::string &name); + + std::string describe(const std::string &stem) const; + + // 类型表条目。name = 它在 config.json:ggml_types 里的**原始键**（未裁前缀）， + // 只能靠它把报错写成可在产物里 grep 的名字：裁过前缀的键在旧形态产物里连后缀 + // 都不一样（blob 被归一成了 .weight），拿 stem 拼凑出来的名字两边都 grep 不到。 + struct TypeEntry { + int64_t id; + std::string name; + }; + + std::unordered_map types_; + std::string key_prefix_; + std::vector vperm_; // 见 ActVPerm（空 = config 声明本产物无需置换） + // 命中统计（get_param_layout 是 const，所以 mutable） + mutable size_t n_blob_ = 0; + mutable size_t n_dense_ = 0; + mutable size_t n_group_ = 0; +}; + +} // namespace infinilm::quantization diff --git a/csrc/layers/quantization/quantization.hpp b/csrc/layers/quantization/quantization.hpp index 0cc9cd7e2..1f935d13d 100644 --- a/csrc/layers/quantization/quantization.hpp +++ b/csrc/layers/quantization/quantization.hpp @@ -4,9 +4,11 @@ #include "awq_marlin.hpp" #include "base_quantization.hpp" #include "compressed_tensors.hpp" +#include "gguf.hpp" #include "gptq.hpp" #include "gptq_marlin.hpp" #include "gptq_qy.hpp" #include "mxfp4.hpp" #include "none_quantization.hpp" +#include "fp8.hpp" #include "quantization_scheme.hpp" diff --git a/csrc/layers/quantization/quantization_scheme.hpp b/csrc/layers/quantization/quantization_scheme.hpp index 455968a7a..37f414c6f 100644 --- a/csrc/layers/quantization/quantization_scheme.hpp +++ b/csrc/layers/quantization/quantization_scheme.hpp @@ -11,6 +11,9 @@ enum class QuantScheme { GPTQ_W4A16, GPTQ_MARLIN_W4A16, MXFP4_W4A16, + FP8_W8A16, + FP8_W8A8, + GGUF_BLOCK, }; enum class KVQuantAlgo { diff --git a/csrc/models/qwen3_5/qwen3_5_attention.cpp b/csrc/models/qwen3_5/qwen3_5_attention.cpp index e7a47f4f9..0d3f24a19 100644 --- a/csrc/models/qwen3_5/qwen3_5_attention.cpp +++ b/csrc/models/qwen3_5/qwen3_5_attention.cpp @@ -6,12 +6,33 @@ #include "../../utils.hpp" #include #include +#include #include #include #include +#include #include namespace infinilm::models::qwen3_5 { +namespace { + +bool should_dump_attention(size_t layer_idx) { + const char *dump_dir = std::getenv("INFINILM_ATTENTION_DUMP_DIR"); + const char *target = std::getenv("INFINILM_ATTENTION_DUMP_LAYER"); + return dump_dir != nullptr && dump_dir[0] != '\0' + && target != nullptr && target[0] != '\0' + && layer_idx == std::strtoull(target, nullptr, 10); +} + +void dump_attention_tensor(const infinicore::Tensor &tensor, + const char *name, + size_t layer_idx) { + const char *dump_dir = std::getenv("INFINILM_ATTENTION_DUMP_DIR"); + tensor->debug(std::string(dump_dir) + "/infini_attention_" + name + "_" + + std::to_string(layer_idx) + ".bin"); +} + +} // namespace Qwen35Attention::Qwen35Attention(std::shared_ptr model_config, size_t layer_idx, @@ -44,13 +65,17 @@ Qwen35Attention::Qwen35Attention(std::shared_ptr auto quantization_method = model_config->get_quantization_method(); auto register_fn = [this](const std::string &n, infinicore::nn::Parameter p) { this->register_parameter(n, std::move(p)); }; + // checkpoint 里的本层路径（已去掉 config.json:quantization_config.key_prefix）。 + // 只给按张量名查类型的量化方案（GGUF）用，其他方案不传就是空串，行为不变。 + const std::string prefix = "layers." + std::to_string(layer_idx_) + ".self_attn"; qkv_proj_ = std::make_shared( hidden_size_, head_dim_, total_num_heads, total_num_kv_heads, "q_proj", "k_proj", "v_proj", register_fn, - quantization_method, use_bias, dtype, device, rank_info); + quantization_method, use_bias, dtype, device, rank_info, prefix); o_proj_ = this->register_module( "o_proj", total_num_heads * head_dim_, hidden_size_, quantization_method, - use_output_bias, dtype, device, tp_rank, tp_size, rank_info.comm); + use_output_bias, dtype, device, tp_rank, tp_size, rank_info.comm, + prefix + ".o_proj."); const auto &rope_params = model_config->get_config_json()["rope_parameters"]; const double partial_rotary_factor = rope_params["partial_rotary_factor"].get(); @@ -135,21 +160,50 @@ infinicore::Tensor Qwen35Attention::forward_paged_(const infinicore::Tensor &pos ASSERT_EQ(batch_size, 1); auto [q, gate, k, v] = qkv_proj_->forward_split(hidden_states_mutable); + const bool dump_attention = should_dump_attention(layer_idx_); + if (dump_attention) { + dump_attention_tensor(q, "q_raw", layer_idx_); + dump_attention_tensor(gate, "gate_raw", layer_idx_); + dump_attention_tensor(k, "k_raw", layer_idx_); + dump_attention_tensor(v, "v_raw", layer_idx_); + } auto q_reshaped = q->view({seq_len, num_attention_heads_, head_dim_}); auto k_reshaped = k->view({seq_len, num_key_value_heads_, head_dim_}); auto v_reshaped = v->view({seq_len, num_key_value_heads_, head_dim_}); q_reshaped = q_norm_->forward(q_reshaped); k_reshaped = k_norm_->forward(k_reshaped); + if (dump_attention) { + dump_attention_tensor(q_reshaped, "q_norm", layer_idx_); + dump_attention_tensor(k_reshaped, "k_norm", layer_idx_); + } auto pos_shape = position_ids->shape(); if (pos_shape.size() != 2 && pos_shape.size() != 1) { throw std::runtime_error("Unexpected position_ids shape"); } std::tie(q_reshaped, k_reshaped) = mrope_->forward(q_reshaped, k_reshaped, position_ids); + if (dump_attention) { + dump_attention_tensor(q_reshaped, "q_rope", layer_idx_); + dump_attention_tensor(k_reshaped, "k_rope", layer_idx_); + } auto attn_output = attn_->forward(q_reshaped, k_reshaped, v_reshaped); - attn_output = infinicore::op::mul(attn_output, infinicore::op::sigmoid(gate)->view(attn_output->shape())); - return o_proj_->forward(attn_output); + if (dump_attention) { + dump_attention_tensor(attn_output, "core_output", layer_idx_); + } + auto gate_sigmoid = infinicore::op::sigmoid(gate)->view(attn_output->shape()); + if (dump_attention) { + dump_attention_tensor(gate_sigmoid, "gate_sigmoid", layer_idx_); + } + attn_output = infinicore::op::mul(attn_output, gate_sigmoid); + if (dump_attention) { + dump_attention_tensor(attn_output, "gated_output", layer_idx_); + } + auto projected = o_proj_->forward(attn_output); + if (dump_attention) { + dump_attention_tensor(projected, "projected_output", layer_idx_); + } + return projected; } } // namespace infinilm::models::qwen3_5 diff --git a/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp b/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp index 70964bb69..2df7e2c79 100644 --- a/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp +++ b/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp @@ -1,10 +1,45 @@ #include "qwen3_5_decoderLayer.hpp" #include "infinicore/ops.hpp" +#include "infinicore/ops/add_rms_norm.hpp" +#include "infinicore/ops/cast.hpp" +#include #include #include #include namespace infinilm::models::qwen3_5 { +namespace { + +void dump_prefill_tensor(const infinicore::Tensor &tensor, + const std::string &filename) { + const char *dump_dir = std::getenv("INFINILM_LAYER_DUMP_DIR"); + if (dump_dir == nullptr || dump_dir[0] == '\0' || !tensor) { + return; + } + const char *dump_numel = std::getenv("INFINILM_LAYER_DUMP_NUMEL"); + if (dump_numel == nullptr || dump_numel[0] == '\0' + || tensor->numel() != std::strtoull(dump_numel, nullptr, 10)) { + return; + } + tensor->debug(std::string(dump_dir) + "/" + filename); +} + +bool should_dump_layer(size_t layer_idx) { + const char *first_n = std::getenv("INFINILM_LAYER_DUMP_FIRST_N"); + if (first_n != nullptr && first_n[0] != '\0' + && layer_idx < std::strtoull(first_n, nullptr, 10)) { + return true; + } + return (layer_idx + 1) % 8 == 0; +} + +bool should_dump_operators(size_t layer_idx) { + const char *target = std::getenv("INFINILM_OPERATOR_DUMP_LAYER"); + return target != nullptr && target[0] != '\0' + && layer_idx == std::strtoull(target, nullptr, 10); +} + +} // namespace Qwen35DecoderLayer::Qwen35DecoderLayer(std::shared_ptr model_config, size_t layer_idx, @@ -17,7 +52,8 @@ Qwen35DecoderLayer::Qwen35DecoderLayer(std::shared_ptr layer_types = model_config->get>("layer_types"); layer_type_ = layer_types[layer_idx]; @@ -33,15 +69,105 @@ Qwen35DecoderLayer::Qwen35DecoderLayer(std::shared_ptr Qwen35DecoderLayer::forward(const infinicore::Tensor &positions, infinicore::Tensor &hidden_states, infinicore::Tensor &residual) { - input_layernorm_->forward_inplace(hidden_states, residual); + if (layer_idx_ == 0) { + dump_prefill_tensor(hidden_states, "infini_embed.bin"); + } + if (residual + && hidden_states->dtype() == infinicore::DataType::F32 + && residual->dtype() == infinicore::DataType::BF16) { + auto y = infinicore::Tensor::empty( + hidden_states->shape(), infinicore::DataType::BF16, hidden_states->device()); + auto residual_out = infinicore::Tensor::empty( + residual->shape(), infinicore::DataType::BF16, residual->device()); + infinicore::op::add_rms_norm_( + y, residual_out, hidden_states, residual, + input_layernorm_->weight(), + static_cast(input_layernorm_->eps())); + hidden_states = y; + residual = residual_out; + } else { + input_layernorm_->forward_inplace(hidden_states, residual); + } if ("linear_attention" == layer_type_) { hidden_states = linear_attn_->forward(hidden_states); } else if ("full_attention" == layer_type_) { hidden_states = self_attn_->forward(positions, hidden_states); } - post_attention_layernorm_->forward_inplace(hidden_states, residual); + const char *fp32_fused_env = std::getenv("INFINILM_POST_NORM_FP32_FUSED"); + const bool fp32_fused = fp32_fused_env != nullptr && fp32_fused_env[0] != '\0' + && std::string(fp32_fused_env) != "0"; + const bool mixed_gguf_f32 = residual + && hidden_states->dtype() == infinicore::DataType::F32 + && residual->dtype() == infinicore::DataType::BF16; + if (mixed_gguf_f32) { + auto y = infinicore::Tensor::empty( + hidden_states->shape(), infinicore::DataType::BF16, hidden_states->device()); + auto residual_out = infinicore::Tensor::empty( + residual->shape(), infinicore::DataType::BF16, residual->device()); + infinicore::op::add_rms_norm_( + y, residual_out, hidden_states, residual, + post_attention_layernorm_->weight(), + static_cast(post_attention_layernorm_->eps())); + hidden_states = y; + residual = residual_out; + } else if (fp32_fused) { + auto a32 = infinicore::Tensor::empty(hidden_states->shape(), infinicore::DataType::F32, hidden_states->device()); + auto b32 = infinicore::Tensor::empty(residual->shape(), infinicore::DataType::F32, residual->device()); + infinicore::op::cast_(a32, hidden_states); + infinicore::op::cast_(b32, residual); + auto y32 = infinicore::Tensor::empty(hidden_states->shape(), infinicore::DataType::F32, hidden_states->device()); + auto r32 = infinicore::Tensor::empty(residual->shape(), infinicore::DataType::F32, residual->device()); + infinicore::op::add_rms_norm_(y32, r32, a32, b32, + post_attention_layernorm_->weight(), + static_cast(post_attention_layernorm_->eps())); + hidden_states = y32; + residual = r32; + } else { + post_attention_layernorm_->forward_inplace(hidden_states, residual); + } + if (should_dump_operators(layer_idx_)) { + dump_prefill_tensor(residual, + "infini_attn_residual_" + std::to_string(layer_idx_) + ".bin"); + dump_prefill_tensor(hidden_states, + "infini_attn_post_norm_" + std::to_string(layer_idx_) + ".bin"); + } + const char *fp32_mlp_env = std::getenv("INFINILM_POST_NORM_FP32_MLP"); + const bool fp32_mlp = fp32_mlp_env != nullptr && fp32_mlp_env[0] != '\0' + && std::string(fp32_mlp_env) != "0"; + if (fp32_mlp && !fp32_fused) { + auto fp32_hidden = infinicore::Tensor::empty( + hidden_states->shape(), infinicore::DataType::F32, hidden_states->device()); + infinicore::op::cast_(fp32_hidden, hidden_states); + hidden_states = fp32_hidden; + } hidden_states = mlp_->forward(hidden_states); + if (should_dump_operators(layer_idx_)) { + dump_prefill_tensor(hidden_states, + "infini_ffn_out_" + std::to_string(layer_idx_) + ".bin"); + } + if (fp32_mlp && !fp32_fused) { + auto bf16_hidden = infinicore::Tensor::empty( + hidden_states->shape(), infinicore::DataType::BF16, hidden_states->device()); + infinicore::op::cast_(bf16_hidden, hidden_states); + hidden_states = bf16_hidden; + } + if (should_dump_layer(layer_idx_)) { + auto materialized = residual ? infinicore::op::add(residual, hidden_states) + : hidden_states; + dump_prefill_tensor(materialized, + "infini_layer_" + std::to_string(layer_idx_) + "_post_ffn.bin"); + } + if (fp32_fused) { + auto bf16_hidden = infinicore::Tensor::empty( + hidden_states->shape(), infinicore::DataType::BF16, hidden_states->device()); + infinicore::op::cast_(bf16_hidden, hidden_states); + hidden_states = bf16_hidden; + auto bf16_residual = infinicore::Tensor::empty( + residual->shape(), infinicore::DataType::BF16, residual->device()); + infinicore::op::cast_(bf16_residual, residual); + residual = bf16_residual; + } return std::make_tuple(hidden_states, residual); } diff --git a/csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp b/csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp index 72fe1a87f..c47bc7797 100644 --- a/csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp +++ b/csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp @@ -1,12 +1,16 @@ #include "qwen3_5_for_causal_lm.hpp" #include "../models_registry.hpp" +#include "infinicore/ops/gemm.hpp" +#include #include #include #include namespace infinilm::models::qwen3_5 { +// TextModel diagnostic hooks are compiled into this Qwen3.5 translation unit. + Qwen35ForCausalLM::Qwen35ForCausalLM( std::shared_ptr model_config, const infinicore::Device &device) { @@ -14,6 +18,9 @@ Qwen35ForCausalLM::Qwen35ForCausalLM( const size_t hidden_size = model_config->get("hidden_size"); const size_t vocab_size = model_config->get("vocab_size"); const auto &dtype = model_config->get_dtype(); + fp32_lm_head_output_ = + model_config->get_config_json().value( + "lm_head_output_dtype", std::string()) == "float32"; INFINICORE_NN_MODULE_INIT(model, model_config, device); INFINICORE_NN_MODULE_INIT( @@ -23,7 +30,40 @@ Qwen35ForCausalLM::Qwen35ForCausalLM( InfinilmModel::Output Qwen35ForCausalLM::forward( const InfinilmModel::Input &input) const { auto hidden_states = model_->forward(input); - return {lm_head_->forward(hidden_states)}; + const char *dump_dir = std::getenv("INFINILM_LAYER_DUMP_DIR"); + const char *dump_numel = std::getenv("INFINILM_LAYER_DUMP_NUMEL"); + if (dump_dir != nullptr && dump_dir[0] != '\0' + && dump_numel != nullptr && dump_numel[0] != '\0' + && hidden_states->numel() + == std::strtoull(dump_numel, nullptr, 10)) { + hidden_states->debug( + std::string(dump_dir) + "/infini_result_norm.bin"); + } + infinicore::Tensor logits; + if (fp32_lm_head_output_) { + auto hidden = hidden_states->is_contiguous() + ? hidden_states + : hidden_states->contiguous(); + const size_t ndim = hidden->ndim(); + auto output_shape = hidden->shape(); + output_shape[ndim - 1] = lm_head_->out_features(); + logits = infinicore::Tensor::empty( + output_shape, infinicore::DataType::F32, hidden->device()); + size_t rows = 1; + for (size_t i = 0; i + 1 < ndim; ++i) { + rows *= hidden->shape()[i]; + } + auto weight = lm_head_->weight()->contiguous(); + infinicore::op::gemm_( + logits->view({rows, lm_head_->out_features()}), + hidden->view({rows, lm_head_->in_features()}), + weight->permute({1, 0}), + 1.0f, + 0.0f); + } else { + logits = lm_head_->forward(hidden_states); + } + return {logits, hidden_states}; } void Qwen35ForCausalLM::reset_cache( diff --git a/csrc/models/qwen3_5/qwen3_5_for_causal_lm.hpp b/csrc/models/qwen3_5/qwen3_5_for_causal_lm.hpp index 51211481f..e610f0598 100644 --- a/csrc/models/qwen3_5/qwen3_5_for_causal_lm.hpp +++ b/csrc/models/qwen3_5/qwen3_5_for_causal_lm.hpp @@ -16,6 +16,7 @@ class Qwen35ForCausalLM : public InfinilmModel { void reset_cache(const cache::CacheConfig *cache_config) override; protected: + bool fp32_lm_head_output_{false}; INFINICORE_NN_MODULE(Qwen35Model, model); INFINICORE_NN_MODULE(infinilm::layers::linear::ReplicatedLinear, lm_head); }; diff --git a/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.cpp b/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.cpp index 65409bc18..eba12169f 100644 --- a/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.cpp +++ b/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.cpp @@ -14,7 +14,8 @@ Qwen35FusedQKVLinear::Qwen35FusedQKVLinear(size_t hidden_size, bool bias, const infinicore::DataType &dtype, const infinicore::Device &device, - engine::distributed::RankInfo rank_info) + engine::distributed::RankInfo rank_info, + const std::string &prefix) : infinilm::layers::linear::ColumnParallelLinear( hidden_size, num_q_head * head_dim * 2 + num_kv_head * head_dim * calculate_kv_replicas(num_kv_head, rank_info.tp_size) * 2, @@ -23,7 +24,9 @@ Qwen35FusedQKVLinear::Qwen35FusedQKVLinear(size_t hidden_size, dtype, device, rank_info.tp_rank, - rank_info.tp_size), + rank_info.tp_size, + -1, + prefix), head_dim_(head_dim), local_num_q_heads_(num_q_head / tp_size_), q_proj_out_size_(num_q_head * head_dim * 2 / tp_size_), @@ -32,12 +35,37 @@ Qwen35FusedQKVLinear::Qwen35FusedQKVLinear(size_t hidden_size, v_out_size_(calculate_kv_replicas(num_kv_head, rank_info.tp_size) * num_kv_head * head_dim / tp_size_), num_kv_head_(num_kv_head), register_fn_(register_fn) { - split_infos_ = { - {q_name, 0, q_proj_out_size_, 0}, - {k_name, q_proj_out_size_, k_out_size_, num_kv_head_}, - {v_name, q_proj_out_size_ + k_out_size_, v_out_size_, num_kv_head_}, - }; - auto params = this->split_params(split_infos_, tp_rank_, tp_size_, num_kv_head_); + if (this->sharded_) { + // GGUF：三段各有自己的 checkpoint 张量（q_proj 含交错的 gate），不存在可 narrow + // 的融合 buffer；stem 必须带结尾的 '.'，与类型表里的张量名逐字相等。 + if (prefix.empty()) { + throw std::runtime_error( + "Qwen35FusedQKVLinear: GGUF 量化必须传 layer prefix（形如 layers.3.self_attn）"); + } + shard_specs_ = { + {q_name, q_proj_out_size_, prefix + "." + q_name + "."}, + {k_name, k_out_size_, prefix + "." + k_name + "."}, + {v_name, v_out_size_, prefix + "." + v_name + "."}, + }; + } else { + split_infos_ = { + {q_name, 0, q_proj_out_size_, 0}, + {k_name, q_proj_out_size_, k_out_size_, num_kv_head_}, + {v_name, q_proj_out_size_ + k_out_size_, v_out_size_, num_kv_head_}, + }; + } + register_fused_params(); +} + +void Qwen35FusedQKVLinear::register_fused_params() { + if (!register_fn_) { + return; + } + // GGUF 分支：逐 shard 一次 GEMM，forward() 里拼回同一根 [B,S,q|k|v]， + // 所以下面 forward_split() 的 narrow 偏移量不用改。 + auto params = this->sharded_ + ? this->init_fused_shards(shard_specs_) + : this->split_params(split_infos_, tp_rank_, tp_size_, num_kv_head_); for (auto &sp : params) { register_fn_(sp.full_name, std::move(sp.param)); } @@ -62,11 +90,10 @@ Qwen35FusedQKVLinear::forward_split(infinicore::Tensor &input) { void Qwen35FusedQKVLinear::process_weights_after_loading() { BaseLinear::process_weights_after_loading(); + // sharded_（GGUF）时 split_infos_ 为空：那些 shard 参数就是加载目标， + // 重新分配会把已读进来的块字节丢掉 if (register_fn_ && !split_infos_.empty()) { - auto params = this->split_params(split_infos_, tp_rank_, tp_size_, num_kv_head_); - for (auto &sp : params) { - register_fn_(sp.full_name, std::move(sp.param)); - } + register_fused_params(); } } diff --git a/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.hpp b/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.hpp index 55d2b762e..60596534d 100644 --- a/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.hpp +++ b/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.hpp @@ -18,7 +18,8 @@ class Qwen35FusedQKVLinear : public infinilm::layers::linear::ColumnParallelLine bool bias = false, const infinicore::DataType &dtype = infinicore::DataType::F32, const infinicore::Device &device = infinicore::Device(), - engine::distributed::RankInfo rank_info = engine::distributed::RankInfo()); + engine::distributed::RankInfo rank_info = engine::distributed::RankInfo(), + const std::string &prefix = ""); void process_weights_after_loading() override; @@ -45,6 +46,11 @@ class Qwen35FusedQKVLinear : public infinilm::layers::linear::ColumnParallelLine size_t num_kv_head_; infinilm::layers::linear::RegisterParamFn register_fn_; std::vector split_infos_; + // GGUF：q|gate / k / v 三段的 ggml 类型互不相同（方案 §6.0 纠正 1）， + // 每段各自一块 buffer，与 split_infos_ 二选一 + std::vector shard_specs_; + + void register_fused_params(); }; } // namespace infinilm::models::qwen3_5 diff --git a/csrc/models/qwen3_next/qwen3_next_gated_deltanet.cpp b/csrc/models/qwen3_next/qwen3_next_gated_deltanet.cpp index 022454247..0a1d46f32 100644 --- a/csrc/models/qwen3_next/qwen3_next_gated_deltanet.cpp +++ b/csrc/models/qwen3_next/qwen3_next_gated_deltanet.cpp @@ -10,11 +10,38 @@ #include #include +#include #include #include +#include #include namespace infinilm::models::qwen3_next { +namespace { + +bool should_dump_gdn(size_t layer_idx, size_t seq_len) { + const char *target_layer = std::getenv("INFINILM_GDN_DUMP_LAYER"); + const char *target_seq_len = std::getenv("INFINILM_GDN_DUMP_SEQ_LEN"); + return target_layer != nullptr && target_layer[0] != '\0' + && target_seq_len != nullptr && target_seq_len[0] != '\0' + && layer_idx == std::strtoull(target_layer, nullptr, 10) + && seq_len == std::strtoull(target_seq_len, nullptr, 10); +} + +void dump_gdn_tensor(const infinicore::Tensor &tensor, + const std::string &name, + size_t layer_idx, + size_t seq_len) { + const char *dump_dir = std::getenv("INFINILM_LAYER_DUMP_DIR"); + if (dump_dir == nullptr || dump_dir[0] == '\0' || !tensor + || !should_dump_gdn(layer_idx, seq_len)) { + return; + } + tensor->debug(std::string(dump_dir) + "/infini_gdn_" + name + "_" + + std::to_string(layer_idx) + ".bin"); +} + +} // namespace Qwen3NextCausalConv1D::Qwen3NextCausalConv1D(std::shared_ptr model_config, size_t layer_idx, @@ -125,12 +152,14 @@ Qwen3NextGatedDeltaNet::Qwen3NextGatedDeltaNet(std::shared_ptrget_quantization_method(); auto register_fn = [this](const std::string &n, infinicore::nn::Parameter p) { this->register_parameter(n, std::move(p)); }; + // 本模块在 checkpoint 里的路径（同 Qwen35Attention，只给 GGUF 类查表方案用） + const std::string prefix = "layers." + std::to_string(layer_idx_) + ".linear_attn"; in_proj_qkv_ = std::make_shared( hidden_size, linear_key_head_dim, linear_key_head_dim, linear_value_head_dim, linear_num_key_heads, linear_num_key_heads, linear_num_value_heads, false, false, false, "in_proj_q", "in_proj_k", "in_proj_v", register_fn, - quantization_method, dtype, device, rank_info); - in_proj_z_ = this->register_module("in_proj_z", hidden_size, value_dim, false, dtype, device, tp_rank, tp_size); + quantization_method, dtype, device, rank_info, prefix); + in_proj_z_ = this->register_module("in_proj_z", hidden_size, value_dim, quantization_method, false, dtype, device, tp_rank, tp_size, -1, prefix + ".in_proj_z."); in_proj_a_ = this->register_module("in_proj_a", hidden_size, linear_num_value_heads, false, dtype, device, tp_rank, tp_size); in_proj_b_ = this->register_module("in_proj_b", hidden_size, linear_num_value_heads, false, dtype, device, tp_rank, tp_size); @@ -140,7 +169,8 @@ Qwen3NextGatedDeltaNet::Qwen3NextGatedDeltaNet(std::shared_ptrregister_module( "out_proj", value_dim, hidden_size, quantization_method, - false, dtype, device, rank_info.tp_rank, rank_info.tp_size, rank_info.comm); + false, dtype, device, rank_info.tp_rank, rank_info.tp_size, rank_info.comm, + prefix + ".out_proj."); } infinicore::Tensor Qwen3NextGatedDeltaNet::forward(const infinicore::Tensor &hidden_states) const { @@ -154,11 +184,16 @@ infinicore::Tensor Qwen3NextGatedDeltaNet::forward(const infinicore::Tensor &hid auto z = in_proj_z_->forward(hidden_states_mutable); auto a = in_proj_a_->forward(hidden_states_mutable); auto b = in_proj_b_->forward(hidden_states_mutable); + dump_gdn_tensor(qkv, "qkv_mixed", layer_idx_, seq_len); + dump_gdn_tensor(z, "z", layer_idx_, seq_len); + dump_gdn_tensor(a, "alpha", layer_idx_, seq_len); + dump_gdn_tensor(b, "beta", layer_idx_, seq_len); auto &forward_context = infinilm::global_state::get_forward_context(); auto &mamba_metadata = forward_context.mamba_metadata; auto conv_qkv = this->conv1d_->forward(qkv); + dump_gdn_tensor(conv_qkv, "conv_output_silu", layer_idx_, seq_len); auto q = conv_qkv->narrow({{2, 0, local_key_dim_}}); auto k = conv_qkv->narrow({{2, local_key_dim_, local_key_dim_}}); @@ -184,6 +219,8 @@ infinicore::Tensor Qwen3NextGatedDeltaNet::forward(const infinicore::Tensor &hid {seq_len, 1, local_num_value_heads_}, {b->stride(1), b->stride(0), 1}); auto [g, beta] = infinicore::op::fused_gated_delta_net_gating(A_log_, a_heads, b_heads, dt_bias_); + dump_gdn_tensor(g, "gate", layer_idx_, seq_len); + dump_gdn_tensor(beta, "beta_sigmoid", layer_idx_, seq_len); delta_out = infinicore::op::recurrent_gated_delta_rule_indexed( q_delta, @@ -217,6 +254,8 @@ infinicore::Tensor Qwen3NextGatedDeltaNet::forward(const infinicore::Tensor &hid {1, seq_len, local_num_value_heads_}, {b->stride(0), b->stride(1), 1}); auto [g, beta] = infinicore::op::fused_gated_delta_net_gating(A_log_, a_heads, b_heads, dt_bias_); + dump_gdn_tensor(g, "gate", layer_idx_, seq_len); + dump_gdn_tensor(beta, "beta_sigmoid", layer_idx_, seq_len); delta_out = infinicore::op::chunk_gated_delta_rule( q_delta, @@ -237,12 +276,18 @@ infinicore::Tensor Qwen3NextGatedDeltaNet::forward(const infinicore::Tensor &hid auto delta_out_2d = delta_out->as_strided( {batch_size * seq_len * local_num_value_heads_, value_head_dim_}, {static_cast(value_head_dim_), 1}); + dump_gdn_tensor(delta_out, "delta_out", layer_idx_, seq_len); auto v_norm_2d = norm_->forward(delta_out_2d); auto v_norm = v_norm_2d->as_strided( {batch_size, seq_len, local_value_dim_}, {static_cast(seq_len * local_value_dim_), static_cast(local_value_dim_), 1}); + dump_gdn_tensor(v_norm, "v_norm", layer_idx_, seq_len); auto gated = infinicore::op::mul(v_norm, infinicore::op::silu(z)); - return out_proj_->forward(gated); + dump_gdn_tensor(gated, "gated", layer_idx_, seq_len); + dump_gdn_tensor(gated, "final_output", layer_idx_, seq_len); + auto output = out_proj_->forward(gated); + dump_gdn_tensor(output, "linear_attn_out", layer_idx_, seq_len); + return output; } } // namespace infinilm::models::qwen3_next diff --git a/csrc/pybind11/engine/engine.hpp b/csrc/pybind11/engine/engine.hpp index c5e85577c..38b50ab8a 100644 --- a/csrc/pybind11/engine/engine.hpp +++ b/csrc/pybind11/engine/engine.hpp @@ -193,6 +193,7 @@ inline void bind_infer_engine(py::module &m) { "temperature", "top_p", "top_k", + "suppressed_token_ids", }; for (auto &item : kwargs) { @@ -209,6 +210,9 @@ inline void bind_infer_engine(py::module &m) { input.top_p = py::cast(item.second); } else if (key == "top_k") { input.top_k = py::cast(item.second); + } else if (key == "suppressed_token_ids") { + input.suppressed_token_ids = + py::cast>>(item.second); } } @@ -250,6 +254,7 @@ inline void bind_infer_engine(py::module &m) { .def_readwrite("visual_token_ranges", &InferEngine::Input::visual_token_ranges) .def_readwrite("target_hidden_states", &InferEngine::Input::target_hidden_states) .def_readwrite("sample_all_positions", &InferEngine::Input::sample_all_positions) + .def_readwrite("suppressed_token_ids", &InferEngine::Input::suppressed_token_ids) .def_readwrite("temperature", &InferEngine::Input::temperature) .def_readwrite("top_k", &InferEngine::Input::top_k) .def_readwrite("top_p", &InferEngine::Input::top_p); diff --git a/python/infinilm/infer_engine.py b/python/infinilm/infer_engine.py index 117d82f9a..6db40d8ad 100644 --- a/python/infinilm/infer_engine.py +++ b/python/infinilm/infer_engine.py @@ -124,6 +124,7 @@ class GenerationConfig: eos_token_id: list[int] | None = None stop_on_eos: bool = True + ignore_eos: bool = False def _infer_position_id_axes(hf_config: dict) -> int: @@ -275,6 +276,7 @@ def _build_input( visual_token_ranges=None, target_hidden_states=None, sample_all_positions=False, + suppressed_token_ids=None, temperature=None, top_k=None, top_p=None, @@ -333,6 +335,9 @@ def convert_tensor_list(tensor_list_): visual_token_ranges=visual_token_ranges, target_hidden_states=target_hidden_states, sample_all_positions=sample_all_positions, + suppressed_token_ids=( + [] if suppressed_token_ids is None else suppressed_token_ids + ), temperature=temperature, top_k=top_k, top_p=top_p, @@ -358,6 +363,7 @@ def forward( image_req_ids=None, visual_token_ranges=None, target_hidden_states=None, + suppressed_token_ids=None, temperature=None, top_k=None, top_p=None, @@ -430,6 +436,7 @@ def convert_tensor_list(tensor_list_): image_req_ids=image_req_ids, visual_token_ranges=visual_token_ranges, target_hidden_states=target_hidden_states, + suppressed_token_ids=suppressed_token_ids, temperature=temperature, top_k=top_k, top_p=top_p, @@ -459,6 +466,7 @@ def forward_raw( visual_token_ranges=None, target_hidden_states=None, sample_all_positions=True, + suppressed_token_ids=None, temperature=None, top_k=None, top_p=None, @@ -481,6 +489,7 @@ def forward_raw( visual_token_ranges=visual_token_ranges, target_hidden_states=target_hidden_states, sample_all_positions=sample_all_positions, + suppressed_token_ids=suppressed_token_ids, temperature=temperature, top_k=top_k, top_p=top_p, @@ -509,7 +518,11 @@ def generate( position_id_delta=0, _measure_and_log_time=False, ): - eos_token_id = self.eos_token_id + eos_token_id = generation_config.eos_token_id + if eos_token_id is None: + eos_token_id = self.eos_token_id + elif isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] past_seq_len = 0 output_ids = [] @@ -687,6 +700,11 @@ def generate( tgt_sizes=tgt_sizes if iter == 0 else None, image_grid_thw=image_grid_thw if iter == 0 else None, image_req_ids=image_req_ids if iter == 0 else None, + suppressed_token_ids=( + [list(eos_token_id) for _ in range(batch_size)] + if generation_config.ignore_eos + else [] + ), temperature=generation_config.temperature, top_k=generation_config.top_k, top_p=generation_config.top_p, diff --git a/python/infinilm/llm/llm.py b/python/infinilm/llm/llm.py index 59d5a1eca..ffc90c844 100644 --- a/python/infinilm/llm/llm.py +++ b/python/infinilm/llm/llm.py @@ -67,6 +67,12 @@ def __init__(self, config: EngineConfig): # Initialize KV cache based on cache type if config.cache_type == "static": + if has_mamba_cache: + model_type = hf_config["model_type"] + raise RuntimeError( + "Static KV cache is not supported for Mamba-cache model " + f"{model_type!r} yet. Use --cache-type paged instead." + ) self.scheduler = StaticScheduler( max_cache_len=config.max_cache_len, enable_prefix_caching=config.enable_prefix_caching, diff --git a/python/infinilm/llm/model_runner/model_runner.py b/python/infinilm/llm/model_runner/model_runner.py index a1696f848..a642b71b9 100644 --- a/python/infinilm/llm/model_runner/model_runner.py +++ b/python/infinilm/llm/model_runner/model_runner.py @@ -217,6 +217,12 @@ def _model_forward(self, scheduler_output): self.config.top_p, self.config.top_k, ) + model_input["suppressed_token_ids"] = [ + list(req.eos_token_ids or self.model_engine.eos_token_id or []) + if req.sampling_params.ignore_eos + else [] + for req in scheduler_output.scheduled_requests + ] if self.speculative_runner is not None: return self._model_forward_with_speculative(scheduler_output, model_input) diff --git a/python/infinilm/modeling_utils.py b/python/infinilm/modeling_utils.py index b210b0c56..0e3245ecd 100644 --- a/python/infinilm/modeling_utils.py +++ b/python/infinilm/modeling_utils.py @@ -55,6 +55,13 @@ def parse_dtype(dtype_str: str): "F8_E5M2": torch.float8_e5m2, } +_FP8_DTYPES = tuple( + x for x in ( + getattr(torch, "float8_e4m3fn", None), + getattr(torch, "float8_e5m2", None), + ) if x is not None +) + def _is_internal_moe_packed_weight(key: str) -> bool: # InfiniLM registers packed MoE parameters internally. HF checkpoints @@ -129,7 +136,8 @@ def load_state_dict( for k in f.keys(): tensor = f.get_tensor(k) preserve_fp32 = k.endswith(preserve_fp32_suffixes) - if tensor.is_floating_point() and not preserve_fp32: + preserve_fp8 = tensor.dtype in _FP8_DTYPES + if tensor.is_floating_point() and not preserve_fp32 and not preserve_fp8: tensor = tensor.to(device=device, dtype=dtype) else: tensor = tensor.to(device=device) @@ -204,6 +212,10 @@ def load_model_state_dict_by_file( preserve_fp32_suffixes = (".e_score_correction_bias",) if model_type == "kimi_k3": preserve_fp32_suffixes += (".A_log", ".dt_bias") + if model.hf_config.get("lm_head_output_dtype") == "float32": + # The Qwen3.5 GGUF route keeps BF16 head weights but can request an + # FP32 output accumulator; do not downcast a future FP32 head artifact. + preserve_fp32_suffixes += ("lm_head.weight",) torch_device = "cpu" torch_dtype = infinicore.utils.to_torch_dtype(dtype) @@ -252,6 +264,11 @@ def load_model_state_dict_by_file( if remapper is not None: model_param = remapper(model_param, config=model.hf_config) + # Convert FP8 block scales from BF16 to FP32 for CUTLASS GEMM + for key in list(model_param.keys()): + if key.endswith("weight_scale_inv") and model_param[key].dtype == torch.bfloat16: + model_param[key] = model_param[key].float() + # --------------------------------------------------------- # # Scale embed_tokens on torch side before converting # --------------------------------------------------------- # @@ -324,6 +341,7 @@ def load_model_state_dict_by_file( target_dtype = ( model_params[key].dtype if key.endswith(preserve_fp32_suffixes) + or model_params[key].dtype in _FP8_DTYPES else torch_dtype ) model_param_infini[key] = infinicore.from_torch( @@ -387,7 +405,11 @@ def load_model_state_dict_by_tensor( with safe_open(file_path, "pt", "cpu") as f: for name in f.keys(): - tensor = f.get_tensor(name).to(dtype=torch_dtype) + raw_tensor = f.get_tensor(name) + if raw_tensor.dtype in _FP8_DTYPES: + tensor = raw_tensor.to(device="cpu") + else: + tensor = raw_tensor.to(dtype=torch_dtype) if name == "model.embed_tokens.weight": embed_tokens_torch_unscaled = tensor @@ -407,7 +429,11 @@ def load_model_state_dict_by_tensor( model_params = torch.load(file_path, weights_only=True, map_location="cpu") for key in model_params.keys(): - tensor = model_params[key].to(dtype=torch_dtype) + raw_tensor = model_params[key] + if raw_tensor.dtype in _FP8_DTYPES: + tensor = raw_tensor.to(device="cpu") + else: + tensor = raw_tensor.to(dtype=torch_dtype) if key == "model.embed_tokens.weight": embed_tokens_torch_unscaled = tensor if scale_emb != 1.0: @@ -758,8 +784,21 @@ def _remap_videonsa(state_dict, config=None): def _remap_qwen3_5(state_dict, config): """Apply Qwen3.5-specific load-time weight fixes.""" state_dict = drop_keys(state_dict, ["mtp."]) + + # Filter out visual encoder keys (not used in language-only mode) + state_dict = {k: v for k, v in state_dict.items() if not k.startswith("model.visual.")} + llm_config = config["text_config"] key_dim = llm_config["linear_key_head_dim"] * llm_config["linear_num_key_heads"] + block_size = 128 # FP8 block size for scale splitting + + # 路线 B 的 GGUF 产物：llama.cpp 转换脚本写 GGUF 时已经做过 `norm.weight + 1` + # （conversion/qwen.py:393-394，除 linear_attn.norm 之外全部加），打包器按「不得再 + # 加一次」原样搬运（scripts/gguf_mapping.py 顶部第 13 行）。这里再加一次就变成 + # 2+w；融合 QKV 也已在打包期拆成 in_proj_q/k/v，不能按老键名再拆一遍。 + gguf = ( + (config.get("quantization_config") or {}).get("quant_method", "") == "gguf" + ) norm_weight_suffixes = ( "input_layernorm.weight", @@ -771,9 +810,11 @@ def _remap_qwen3_5(state_dict, config): to_drop = [] to_add = {} for key, tensor in state_dict.items(): - if key == "model.norm.weight" or key.endswith(norm_weight_suffixes): + if not gguf and ( + key == "model.norm.weight" or key.endswith(norm_weight_suffixes) + ): state_dict[key] = tensor + torch.ones_like(tensor) - elif key.endswith("linear_attn.in_proj_qkv.weight"): + elif key.endswith("linear_attn.in_proj_qkv.weight") and not gguf: prefix = key[: -len("in_proj_qkv.weight")] to_add[prefix + "in_proj_q.weight"] = state_dict[key][ :key_dim, : @@ -785,6 +826,22 @@ def _remap_qwen3_5(state_dict, config): key_dim * 2 :, : ].contiguous() to_drop.append(key) + elif key.endswith("linear_attn.in_proj_qkv.weight_scale_inv"): + # Split fused QKV scale into separate q/k/v scales + # Scale shape: [num_out_blocks, num_in_blocks] + # out dim is split: q(key_dim) | k(key_dim) | v(rest) + prefix = key[: -len("in_proj_qkv.weight_scale_inv")] + key_blocks = key_dim // block_size + to_add[prefix + "in_proj_q.weight_scale_inv"] = state_dict[key][ + :key_blocks, : + ].contiguous() + to_add[prefix + "in_proj_k.weight_scale_inv"] = state_dict[key][ + key_blocks : key_blocks * 2, : + ].contiguous() + to_add[prefix + "in_proj_v.weight_scale_inv"] = state_dict[key][ + key_blocks * 2 :, : + ].contiguous() + to_drop.append(key) state_dict = drop_keys(state_dict, to_drop) state_dict.update(to_add) @@ -850,6 +907,7 @@ def _remap_ernie4_5_moe_vl(state_dict, config=None): if ( key.endswith((".mlp.gate.weight", ".mlp.gate.weight_1")) and tensor.is_floating_point() + and tensor.dtype not in _FP8_DTYPES ): remapped[key] = tensor.to(dtype=target_dtype).contiguous() else: @@ -896,12 +954,13 @@ def fuse_expert_group(expert_ids): b1_tensors.append(torch.cat([gate_bias, up_bias], dim=0)) b2_tensors.append(down_bias) + fused_dtype = w1_tensors[0].dtype if w1_tensors[0].dtype in _FP8_DTYPES else target_dtype fused = { "w1": torch.stack(w1_tensors, dim=0) - .to(dtype=target_dtype) + .to(dtype=fused_dtype) .contiguous(), "w2": torch.stack(w2_tensors, dim=0) - .to(dtype=target_dtype) + .to(dtype=fused_dtype) .contiguous(), } if has_all_bias: diff --git a/scripts/gguf_mapping.py b/scripts/gguf_mapping.py new file mode 100644 index 000000000..84c7fdd49 --- /dev/null +++ b/scripts/gguf_mapping.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +""" +InfiniLM 路线 B —— GGUF -> InfiniLM 权重映射表（打包器与审计脚本的单一事实源）。 + +所有条目都由阶段 0 审计实测得出，不是推测： + * InfiniLM 侧参数键/shape/取向：scripts/gguf_routeb_probe_params.py 在 CPU 上 + 构造 mini qwen3_5 引擎导出的 state_dict（121 键），取向为 [out, in]，与 GGUF + blob 的行主序一致 -> 打包不需要转置。 + * GGUF 侧键与 shape：scripts/gguf_routeb_audit.py D 节对 866 个张量实测。 + * transform 依据 llama.cpp conversion/qwen.py（行号为该文件实测）： + 388 A_log -> -exp(A_log) （故打包需反解 log(-x)） + 391 dt_bias -> 改名 dt_proj.bias，值不变 （故 ssm_dt.bias 原样用） + 394 *.norm.weight -> w + 1（linear_attn.norm 除外） + （故打包不得再加 1） + 571-605 _LinearAttentionVReorderBase.modify_tensors：需逆重排的集合是 + in_proj_qkv(仅 V 行段) / in_proj_z / in_proj_a / in_proj_b(head_dim=1) / + A_log / dt_bias(head_dim=1) / conv1d(仅 V 通道段)； + 609 out_proj 重排的是 **列(in 维)** —— 本方案改为运行时对激活做 head gather， + 权重保持逐字节不变，故此处不标 transform。 + `linear_attn.norm`(=ssm_norm) 不在重排列表内，确认无需重排。 + 615 注释 "Qwen3.5 always applies interleaved MRoPE" -> mrope_interleaved 必为 True + 619 写入 GGUF 的 mrope_section 是 4 元素 [11,11,10,0]，而 InfiniLM + qwen3_5_attention.cpp:65 硬性要求 3 元素 -> 打包时去掉尾 0。 +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + + +# --------------------------------------------------------------------------- +# transform 语义 +# --------------------------------------------------------------------------- +T_NONE = "" # 原样搬运（blob 逐字节 / dense 仅换 dtype） +T_VROWS = "vrows" # 沿 out 维按 V 头分块整块搬回 grouped 序（blob 可行级置换） +T_VELEM = "velem" # 1-D、每头 1 个元素：T_VROWS 的 head_dim=1 退化形式（同一实现） +T_ALOG = "alog" # A_log = log(-ssm_a)，再置换 +T_DENSE = "dense" # 反量化为 BF16（框架不支持该参数走量化路径） + +# V 头置换在 dim0 上的作用域： +# all = 整个 dim0 都是 value 头（in_proj_v / in_proj_z / in_proj_a / in_proj_b / A_log / dt_bias） +# v_tail = 只有末尾 value_dim 个元素是 value 段（conv1d 的 [q|k|v] 通道拼接） +VPERM_ALL, VPERM_TAIL = "all", "v_tail" + +# blob 参数在产物 / 框架里的名字后缀。阶段 2 的 get_param_layout 必须用同名， +# 否则 load_state_dict(strict=False) 会把 400 个权重静默丢掉。 +BLOB_SUFFIX = "weight_bytes" + +# 两者共用一份置换实现：每头几个元素由条目 shape 推出来（见 gguf_transforms.vperm_head_dim）， +# 48 个元素 / 48 个头 = 1 ⇒ 自然就是逐元素置换，不需要第二套代码。 +VPERM_TRANSFORMS = (T_VROWS, T_VELEM) + + +def needs_vperm(e: "Entry") -> bool: + return bool(set(e.transforms) & set(VPERM_TRANSFORMS)) + + +# --------------------------------------------------------------------------- +# GGML 类型名（数值见 ggml.h；本文件不依赖 gguf-py，避免脚本互相 import 拉环境） +# 实测本 GGUF 出现的类型集合由 scripts/gguf_routeb_shape_contract.py 断言。 +# --------------------------------------------------------------------------- +F32, Q8_0, Q4_K, Q5_K, Q6_K = "F32", "Q8_0", "Q4_K", "Q5_K", "Q6_K" +IQ4_NL, IQ4_XS = "IQ4_NL", "IQ4_XS" + +# 阶段 3 v1 必须实现的 block 类型（实测本文件主模型只出现这 4 种）。 +# Q4_K 不跟 IQ4 一起延期：它与 Q5_K 同族（144B/256，只差第 5 bit 平面）， +# Q5_K 本来就要写，多支持 Q4_K 接近零成本，而它占 2 个张量 45 MiB。 +NATIVE_BLOB_TYPES = (Q8_0, Q4_K, Q5_K, Q6_K) +# v1 稠密化的 i-quants（执行方案 §2.4 决策：量小、需查码表，上原生 kernel 推到阶段 6）。 +# 实测共 5 个张量 0.23 GiB，稠密化后占 0.82 GiB，代价 +0.60 GiB（预算仍 ≤ 24 GiB）。 +V1_IQUANT_DENSE = (IQ4_NL, IQ4_XS) +DENSE_SRC_TYPES = (F32, Q8_0, Q6_K) + V1_IQUANT_DENSE + + +def apply_v1_exceptions(plan, gguf_types, enabled=True): + """把 v1 不打算写 kernel 的 i-quants 条目就地转为稠密化。 + + gguf_types: {张量名: GGML 类型名}，由调用方从真文件采集（本模块不依赖 gguf-py）。 + 阶段 6 上了 IQ4 码本后传 enabled=False 即可全部回到逐字节路径。 + """ + n = 0 + if enabled: + for e in plan: + if e.blob and gguf_types.get(e.gguf) in V1_IQUANT_DENSE: + e.blob = False + e.transforms = e.transforms + (T_DENSE,) + e.note = (e.note + "；" if e.note else "") + \ + "v1 稠密化例外（源 %s），阶段 6 上原生 kernel 后取消" % gguf_types[e.gguf] + n += 1 + return n + + +@dataclass +class Entry: + """一条 GGUF 张量 -> 一个 InfiniLM 参数。""" + + infinilm: str # InfiniLM 参数名（含 model.language_model. 前缀） + gguf: str # GGUF 张量名 + shape: tuple # InfiniLM 期望 shape（未 TP 切分的全量），取向 [out, in] + blob: bool # True = 保留 GGUF 原始 block 字节（U8 [out, row_bytes]） + transforms: tuple = () + types: tuple = () # 允许的 GGUF 源类型名；() = 不限（由 contract 脚本报告实际值） + slices: tuple = () # 沿 out 维占用的 [start, end)；共用同一 gguf 的条目做覆盖校验 + vperm: str = VPERM_ALL # T_VROWS 的作用域（仅当 transforms 含 T_VROWS 时有意义） + # 该条目的权重需要置换的是**列（in 维）**而不是行：块量化沿 in 维分块 + #（Q4_K/Q5_K/Q6_K block_size=256），打包期置换列 = 跨块重排 = 必须重量化，做不到。 + # 于是只能在运行时置换喂给它的输入激活，规则由 activation_vperm_rules() 导出进 config。 + # 故意不放进 transforms：那个元组描述的是「打包期对字节做的事」，混进去会污染字节路径。 + act_vperm: bool = False + note: str = "" + + +@dataclass +class Dims: + hidden: int + n_q_heads: int + n_kv_heads: int + head_dim: int + ffn: int + lin_k_heads: int + lin_v_heads: int + lin_k_dim: int + lin_v_dim: int + conv_kernel: int + vocab: int + n_layers: int + interval: int + mrope_section: tuple = (11, 11, 10) + rope_theta: float = 1e7 + partial_rotary_factor: float = 0.25 + rms_norm_eps: float = 1e-6 + max_position_embeddings: int = 262144 + architectures: str = "Qwen3_5ForConditionalGeneration" + + # --- 派生量 --- + @property + def q_rows(self) -> int: # q_proj 行数 = heads * head_dim * 2（q 与 gate 每头交错） + return self.n_q_heads * self.head_dim * 2 + + @property + def kv_rows(self) -> int: + return self.n_kv_heads * self.head_dim + + @property + def o_in(self) -> int: + return self.n_q_heads * self.head_dim + + @property + def key_dim(self) -> int: + return self.lin_k_heads * self.lin_k_dim + + @property + def value_dim(self) -> int: + return self.lin_v_heads * self.lin_v_dim + + @property + def qkv_rows(self) -> int: # q | k | v 融合（与 GGUF attn_qkv 一致） + return self.key_dim * 2 + self.value_dim + + @property + def conv_channels(self) -> int: + return self.qkv_rows + + @property + def v_per_k(self) -> int: + return self.lin_v_heads // self.lin_k_heads + + def layer_types(self) -> list: + """与 C++ prepare_qwen3_5_model_config 的推导完全一致：(i+1) % interval == 0。""" + return ["full_attention" if (i + 1) % self.interval == 0 else "linear_attention" + for i in range(self.n_layers)] + + +REAL = Dims(hidden=5120, n_q_heads=24, n_kv_heads=4, head_dim=256, ffn=17408, + lin_k_heads=16, lin_v_heads=48, lin_k_dim=128, lin_v_dim=128, + conv_kernel=4, vocab=248320, n_layers=64, interval=4) + +MINI = Dims(hidden=512, n_q_heads=2, n_kv_heads=1, head_dim=256, ffn=1024, + lin_k_heads=2, lin_v_heads=6, lin_k_dim=128, lin_v_dim=128, + conv_kernel=4, vocab=1024, n_layers=8, interval=4) + + +PREFIX = "model.language_model." + + +def layer_entries(d: Dims, i: int, role: str) -> list: + """第 i 层的映射条目。role ∈ {'linear_attention', 'full_attention'}。 + + 注：源类型不在表中写死（同一后缀在不同层就用过 Q4_K/Q5_K/Q6_K/Q8_0/IQ4_*）， + 由 contract 脚本从真文件采集后比对 NATIVE_BLOB_TYPES / DENSE_SRC_TYPES。 + """ + L = f"{PREFIX}layers.{i}." + G = f"blk.{i}." + kd, vd = d.key_dim, d.value_dim + out = [ + Entry(L + "input_layernorm.weight", G + "attn_norm.weight", (d.hidden,), + False, (T_DENSE,), note="GGUF 已 baked +1，打包不得再加"), + Entry(L + "post_attention_layernorm.weight", G + "post_attention_norm.weight", + (d.hidden,), False, (T_DENSE,), note="同上"), + Entry(L + "mlp.gate_proj.weight", G + "ffn_gate.weight", (d.ffn, d.hidden), True), + Entry(L + "mlp.up_proj.weight", G + "ffn_up.weight", (d.ffn, d.hidden), True), + Entry(L + "mlp.down_proj.weight", G + "ffn_down.weight", (d.hidden, d.ffn), True), + ] + if role == "full_attention": + out += [ + Entry(L + "self_attn.q_proj.weight", G + "attn_q.weight", (d.q_rows, d.hidden), + True, (), note="行数含 q|gate 每头交错，与 Qwen35FusedQKVLinear 一致"), + Entry(L + "self_attn.k_proj.weight", G + "attn_k.weight", (d.kv_rows, d.hidden), True), + Entry(L + "self_attn.v_proj.weight", G + "attn_v.weight", (d.kv_rows, d.hidden), True), + Entry(L + "self_attn.o_proj.weight", G + "attn_output.weight", (d.hidden, d.o_in), True), + Entry(L + "self_attn.q_norm.weight", G + "attn_q_norm.weight", (d.head_dim,), + False, (T_DENSE,), note="GGUF 已 baked +1"), + Entry(L + "self_attn.k_norm.weight", G + "attn_k_norm.weight", (d.head_dim,), + False, (T_DENSE,), note="GGUF 已 baked +1"), + ] + else: + out += [ + Entry(L + "linear_attn.in_proj_q.weight", G + "attn_qkv.weight", (kd, d.hidden), + True, (), slices=((0, kd),), note="attn_qkv 行 [0:kd]"), + Entry(L + "linear_attn.in_proj_k.weight", G + "attn_qkv.weight", (kd, d.hidden), + True, (), slices=((kd, 2 * kd),), note="attn_qkv 行 [kd:2kd]"), + Entry(L + "linear_attn.in_proj_v.weight", G + "attn_qkv.weight", (vd, d.hidden), + True, (T_VROWS,), slices=((2 * kd, 2 * kd + vd),), + note="attn_qkv 行 [2kd:]，V 头 tiled->grouped"), + Entry(L + "linear_attn.in_proj_z.weight", G + "attn_gate.weight", (vd, d.hidden), + True, (T_VROWS,), note="qwen.py:583 行重排（head_v_dim）"), + Entry(L + "linear_attn.in_proj_a.weight", G + "ssm_alpha.weight", + (d.lin_v_heads, d.hidden), False, (T_DENSE, T_VROWS), + note="实测源为 Q8_0；框架该参数不走量化路径 -> 稠密化；" + "qwen.py:587 行重排 head_dim=1"), + Entry(L + "linear_attn.in_proj_b.weight", G + "ssm_beta.weight", + (d.lin_v_heads, d.hidden), False, (T_DENSE, T_VROWS), note="同上"), + Entry(L + "linear_attn.A_log", G + "ssm_a", (d.lin_v_heads,), + False, (T_ALOG, T_VROWS), note="GGUF 存的是 -exp(A_log)，需 log(-x) 反解"), + Entry(L + "linear_attn.dt_bias", G + "ssm_dt.bias", (d.lin_v_heads,), + False, (T_VELEM,), note="qwen.py:589 逐头置换，值不变"), + Entry(L + "linear_attn.conv1d.weight", G + "ssm_conv1d.weight", + (d.conv_channels, 1, d.conv_kernel), False, (T_DENSE, T_VROWS), + vperm=VPERM_TAIL, + note="GGUF 已 squeeze 成 [C,K] -> 补回中间维；仅末尾 V 通道段重排"), + Entry(L + "linear_attn.norm.weight", G + "ssm_norm.weight", (d.lin_v_dim,), + False, (T_DENSE,), note="不在 qwen.py 重排列表内；两侧都不加 1"), + Entry(L + "linear_attn.out_proj.weight", G + "ssm_out.weight", + (d.hidden, vd), True, (), act_vperm=True, + note="qwen.py:609 重排的是列(in 维)，blob 不能跨块置换 -> " + "运行时对输入激活做 grouped->tiled（见 config 的 activation_vperm）"), + ] + return out + + +def build_plan(d: Dims) -> list: + """全模型映射条目（含顶层）。""" + entries = [ + Entry(PREFIX + "embed_tokens.weight", "token_embd.weight", (d.vocab, d.hidden), + False, (T_DENSE,), note="实测 GGUF 为 Q6_K -> 反量化"), + ] + for i, role in enumerate(d.layer_types()): + entries += layer_entries(d, i, role) + entries += [ + Entry(PREFIX + "norm.weight", "output_norm.weight", (d.hidden,), + False, (T_DENSE,), note="GGUF 已 baked +1"), + Entry("lm_head.weight", "output.weight", (d.vocab, d.hidden), + False, (T_DENSE,), note="实测 GGUF 为 Q8_0 -> 反量化"), + ] + return entries + + +def activation_vperm_suffix(e: "Entry") -> str: + """条目对应的 checkpoint stem 后缀（剥掉层号、含结尾 '.'），供 C++ 做尾匹配。 + + C++ 递来的 stem 形如 `layers.7.linear_attn.out_proj.`（挂在前缀下的相对形态， + 见 gguf.cpp 的 key_prefix_ 裁剪），所以这里必须同时去掉 PREFIX 和 `layers..`。 + """ + name = re.sub(r"^" + re.escape(PREFIX) + r"layers\.\d+\.", "", e.infinilm) + if name.endswith(".weight"): + name = name[:-len(".weight")] + return name + "." + + +def activation_vperm_rules(d: "Dims", plan: list) -> list: + """从映射表派生「运行时要对输入激活做的 V 头置换」清单（写进 quantization_config）。 + + 为什么必须有这件事：conversion/qwen.py:607-609 在导出 GGUF 时把 out_proj 的**列**从 + grouped 换成了 tiled；而 GDN kernel 期望/产出的 v 头序是 grouped（InfiniCore + chunk_gated_delta_rule/cuda/kernel.cuh:112 `key_head_idx = value_head_idx / + value_heads_per_key_head`）。打包期我们把 in_proj_v 等**行**向条目逆置换回 grouped， + 但 out_proj 的列置换不掉（跨块），所以只能把激活置换过去：grouped -> tiled。 + 规则在这里派生、C++ 只照单执行，两边不各抄一份（§6.0 纠正 2 的同一原则）。 + """ + n_k, r, hd = d.lin_k_heads, d.v_per_k, d.lin_v_dim + rules, seen = [], set() + for e in plan: + if not e.act_vperm: + continue + in_dim = int(e.shape[1]) + if in_dim != n_k * r * hd: + raise ValueError("%s: 条目 in 维 %d != num_k_heads*num_v_per_k*head_dim = %d，" + "无法按头分块置换" % (e.infinilm, in_dim, n_k * r * hd)) + suffix = activation_vperm_suffix(e) + if suffix in seen: + continue + seen.add(suffix) + rules.append({"suffix": suffix, "num_k_heads": n_k, + "num_v_per_k": r, "head_dim": hd}) + return rules + + +def expected_keys(d: Dims) -> list: + return [e.infinilm for e in build_plan(d)] + + +# 打包期需丢弃的 GGUF 张量。实测 blk.64 共 15 个张量 = +# 11 个普通 full-attention 层张量（attn_norm/attn_q/attn_k/attn_v/attn_q_norm/ +# attn_k_norm/attn_output/post_attention_norm/ffn_gate/ffn_up/ffn_down） +# + 4 个 nextn.*（eh_proj/enorm/hnorm/shared_head_norm），共 0.327 GiB。 +# 推论：主模型的 full-attn 层是 16 个（blk.3,7,...,63），而带 attn_q 的 block +# 共 17 个 —— 多出的那个就是 MTP block，不要误当成第 17 个注意力层。 +DROP_PREFIXES = ("blk.64.",) +MTP_BLOCK = 64 + + +def compress(shape: tuple) -> tuple: + """去掉长度为 1 的维。GGUF 写入时对 conv1d 做过 squeeze（qwen.py:393）， + 比对形状时需同样处理，否则 (C,1,K) vs (C,K) 会误报。""" + return tuple(int(x) for x in shape if int(x) != 1) + + +# --------------------------------------------------------------------------- +# 派生工具：产物参数名、type 表键、行字节、config.json +# —— 打包器 / 阶段 2 C++ / 契约脚本都必须走这里，不得各抄一份 +# --------------------------------------------------------------------------- +def ckpt_name(e: "Entry") -> str: + """写进 safetensors（以及框架 state_dict）的参数名。""" + if e.blob and e.infinilm.endswith(".weight"): + return e.infinilm[:-len(".weight")] + "." + BLOB_SUFFIX + return e.infinilm + + +def type_table_key(name: str) -> str: + """config.json:quantization_config.ggml_types 的键 = checkpoint 张量名原文。 + + 曾经用过“去 model.language_model. 前缀 + .weight_bytes 归一回 .weight”的压缩写法， + 但那要求阶段 2 的 C++ 把同一套规则逐字符重实现一遍，拼错不会报错只会静默走 + 稠密路径（能加载、显存暴涨、结果错）。现在 key 就是 safetensors 里的张量名， + C++ 只递 stem（如 `layers.0.mlp.gate_proj.`）再探 `stem+"weight_bytes"` / + `stem+"weight"`，命中 0 个或 2 个都抛错；挂载前缀由 quantization_config.key_prefix + 告知，不在 C++ 里硬编码。详见执行方案 §6.0 纠正 2。 + """ + return name + + +def row_bytes(n_in: int, block_size: int, type_size: int) -> int: + """blob 一行的字节数。本模块不依赖 gguf-py，故 (block_size, type_size) 由调用方给。""" + if n_in % block_size: + raise ValueError("in=%d 不能被块大小 %d 整除" % (n_in, block_size)) + return n_in // block_size * type_size + + +def make_text_config(d: "Dims") -> dict: + """config.json 的 text_config 段。 + + 键名集合以 scripts/gguf_routeb_probe_params.py::CFG 为准 —— 那份 config 已被 + InferEngine 实测接受（121 键全对齐），不要再引入未验证的键（如 architectures / + layer_types：layer_types 由 qwen3_5_for_causal_lm.cpp:72-87 从 interval 推导）。 + """ + return { + "model_type": "qwen3_5_text", + "hidden_size": d.hidden, + "num_hidden_layers": d.n_layers, + "num_attention_heads": d.n_q_heads, + "num_key_value_heads": d.n_kv_heads, + "head_dim": d.head_dim, + "intermediate_size": d.ffn, + "rms_norm_eps": d.rms_norm_eps, + "max_position_embeddings": d.max_position_embeddings, + "vocab_size": d.vocab, + "full_attention_interval": d.interval, + "linear_num_key_heads": d.lin_k_heads, + "linear_num_value_heads": d.lin_v_heads, + "linear_key_head_dim": d.lin_k_dim, + "linear_value_head_dim": d.lin_v_dim, + "linear_conv_kernel_dim": d.conv_kernel, + "attention_bias": False, + "rope_parameters": { + "rope_type": "mrope", + "rope_theta": d.rope_theta, + "partial_rotary_factor": d.partial_rotary_factor, + # 必须 3 元素：qwen3_5_attention.cpp:65 硬校验；且 + # position_id_axes = len(mrope_section)（qwen3_5_for_causal_lm.cpp:52-64） + "mrope_section": list(d.mrope_section), + # 无默认值，缺键即抛；conversion/qwen.py:615 注释已确认恒为交错 + "mrope_interleaved": True, + }, + } + + +def make_root_config(d: "Dims", ggml_types: dict, act_vperm: list = None) -> dict: + """config.json 根段。 + + ★ quantization_config 必须在**顶层**：ModelConfig ctor 只读 + `config_json["quantization_config"]`（model_config.cpp:5/16），而 + prepare_qwen3_5_model_config 的 text_config -> root 合并发生在 ctor **之后**； + 写在 text_config 里会得到 null => NoneQuantization 的静默降级。 + """ + return { + "model_type": "qwen3_5", + "torch_dtype": "bfloat16", + # BF16 logits collapse close top candidates into exact ties. Keep the + # dense BF16 head weights, but accumulate/write its output in FP32. + "lm_head_output_dtype": "float32", + "tie_word_embeddings": False, + "text_config": make_text_config(d), + "quantization_config": { + "quant_method": "gguf", + # C++ 侧的表 key = 本表 key 去掉这段前缀（层级以下的模块不知道自己挂在 + # model. 下）；由打包器写入，不在 C++ 里硬编码 + "key_prefix": PREFIX, + "ggml_types": ggml_types, + # 运行时激活 V 头置换规则（见 activation_vperm_rules）。空列表 = 该产物没有 + # 列向置换的条目；C++ 缺这个键会直接拒启，避免旧 config 静默跑出错位权重。 + "activation_vperm": act_vperm or [], + }, + } diff --git a/scripts/gguf_routeb_audit.py b/scripts/gguf_routeb_audit.py new file mode 100644 index 000000000..2a6b6c826 --- /dev/null +++ b/scripts/gguf_routeb_audit.py @@ -0,0 +1,555 @@ +#!/usr/bin/env python3 +""" +InfiniLM 路线 B —— 阶段 0 风险清零审计（执行方案 §4） + +检查项： + A. 容器/字节布局：GGUF 原始字节按 [out, row_bytes] 重解释 + 自研 block 解码 + 是否与 gguf-py 权威实现**逐比特相等**（Q8_0 / Q4_K / Q5_K / Q6_K） + B. 对齐事实：块起始与行 stride 的真实对齐度（写 kernel 前的硬约束） + C. V 头重排：grouped<->tiled 正向/逆向置换是否自等（执行方案 §2.7） + D. 命名/形状契约：GGUF 实际张量集合是否与打包器的映射表完全一致 + E. 元数据：rope / ssm / 层类型等 config.json 依据 + +用法： + python3 scripts/gguf_routeb_audit.py \ + [--gguf /home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf] +退出码 0 表示全部 PASS。 +""" + +from __future__ import annotations + +import argparse +import os +import sys +import collections + +import numpy as np + +_LLAMA_CPP = os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp") +sys.path.insert(0, os.path.join(_LLAMA_CPP, "gguf-py")) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from gguf import GGUFReader # noqa: E402 +from gguf.constants import ( # noqa: E402 + GGML_QUANT_SIZES, + GGMLQuantizationType as QType, +) +import gguf.quants as gq # noqa: E402 + +QK_K = 256 + +PASSED: list[str] = [] +FAILED: list[str] = [] + + +def check(name: str, ok: bool, detail: str = "") -> bool: + (PASSED if ok else FAILED).append(name) + print(f" [{'PASS' if ok else 'FAIL'}] {name}" + (f" {detail}" if detail else "")) + return ok + + +# --------------------------------------------------------------------------- +# 自研 block 解码：完全按 ggml 内存布局手写（将来 1:1 移植进 ggml_blocks.h） +# 输入统一为 uint8 blob，形状 [n_rows, row_bytes]；输出 float32 [n_rows, n_cols] +# --------------------------------------------------------------------------- + +def _rows_to_blocks(blob: np.ndarray, type_size: int) -> np.ndarray: + """[n_rows, row_bytes] -> [n_blocks, type_size]，块沿 in 连续、按 out 行排列。""" + assert blob.dtype == np.uint8 + n_rows, row_bytes = blob.shape + assert row_bytes % type_size == 0, f"row_bytes={row_bytes} 不是 type_size={type_size} 的整数倍" + return blob.reshape(-1, type_size) + + +def _f16(col: np.ndarray) -> np.ndarray: + return col.view(np.float16).astype(np.float32) + + +def decode_q8_0(blob: np.ndarray, n_cols: int) -> np.ndarray: + """块 = d(f16,2B) + qs(int8,32B)，共 34B / 32 元素。""" + bs, ts = 32, 34 + b = _rows_to_blocks(blob, ts) + d = _f16(b[:, :2]) # [nb,1] + x = b[:, 2:ts].view(np.int8).astype(np.float32) # [nb,32] + return (d * x).reshape(blob.shape[0], n_cols) + + +def _k_scale_min(scales: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Q4_K/Q5_K 的 12 字节 -> 8 组 (sc, min)，6+2 bit 交错打包。""" + n = scales.shape[0] + s = scales.reshape((n, 3, 4)) + d, m, m_d = np.split(s, 3, axis=-2) + sc = np.concatenate([d & 0x3F, (m_d & 0x0F) | ((d >> 2) & 0x30)], axis=-1) + mn = np.concatenate([m & 0x3F, (m_d >> 4) | ((m >> 2) & 0x30)], axis=-1) + return sc.reshape((n, 8)), mn.reshape((n, 8)) + + +# --- 索引表：这就是后续 CUDA 实现的 ggml_blocks.h 布局规范 ------------------- +# Q4_K / Q5_K：8 个子块 x 32 元素（不是 16x16！），子块 g 内偏移 o： +# qs 字节 = qs_base + (g // 2) * 32 + o，nibble 位移 = (g % 2) * 4 +_e = np.arange(QK_K) +_g = _e // 32 +_o = _e % 32 +K_QS_BYTE = (_g // 2) * 32 + _o +K_QS_SHIFT = (_g % 2) * 4 +K_SCALE_IDX = _g # 每 32 元素一组 scale/min + +# Q5_K 的第 5 bit：qh 字节 = o，位 = g +K5_QH_BYTE = _o +K5_QH_BIT = _g + +# Q6_K：256 元素，6 bit = 低 4(nibble) + 高 2 +# 低 4 bit：字节 = (h // 2) * 64 + r，位移 = (h % 2) * 4（h = e // 64, r = e % 64） +# 高 2 bit：字节 = (g // 4) * 32 + o，位移 = (g % 4) * 2 +# scale 索引 = e // 16（16 个子块 x 16 元素） +_h = _e // 64 +_r = _e % 64 +Q6_LO_BYTE = (_h // 2) * 64 + _r +Q6_LO_SHIFT = (_h % 2) * 4 +Q6_HI_BYTE = (_g // 4) * 32 + _o +Q6_HI_SHIFT = (_g % 4) * 2 +Q6_SCALE_IDX = _e // 16 + + +def decode_q4_k(blob: np.ndarray, n_cols: int) -> np.ndarray: + """块 = d(2) dmin(2) scales(12) qs(128) = 144B / 256 元素。""" + ts = 144 + b = _rows_to_blocks(blob, ts) + d = _f16(b[:, 0:2]) + dmin = _f16(b[:, 2:4]) + sc, mn = _k_scale_min(b[:, 4:16]) + qs = b[:, 16:ts] + q = ((qs[:, K_QS_BYTE] >> K_QS_SHIFT.astype(np.uint8)) + & np.uint8(0x0F)).reshape(b.shape[0], 8, 32).astype(np.float32) + d_eff = (d * sc.astype(np.float32)).reshape(b.shape[0], 8, 1) + m_eff = (dmin * mn.astype(np.float32)).reshape(b.shape[0], 8, 1) + return (d_eff * q - m_eff).reshape(blob.shape[0], n_cols) + + +def decode_q5_k(blob: np.ndarray, n_cols: int) -> np.ndarray: + """块 = d(2) dmin(2) scales(12) qh(32) qs(128) = 176B / 256 元素。""" + ts = 176 + b = _rows_to_blocks(blob, ts) + d = _f16(b[:, 0:2]) + dmin = _f16(b[:, 2:4]) + sc, mn = _k_scale_min(b[:, 4:16]) + qh = b[:, 16:48] + qs = b[:, 48:ts] + n = b.shape[0] + lo = (qs[:, K_QS_BYTE] >> K_QS_SHIFT.astype(np.uint8)) & np.uint8(0x0F) + hi = (qh[:, K5_QH_BYTE] >> K5_QH_BIT.astype(np.uint8)) & np.uint8(0x01) + q = (lo | (hi << np.uint8(4))).reshape(n, 8, 32).astype(np.float32) + d_eff = (d * sc.astype(np.float32)).reshape(n, 8, 1) + m_eff = (dmin * mn.astype(np.float32)).reshape(n, 8, 1) + return (d_eff * q - m_eff).reshape(blob.shape[0], n_cols) + + +def decode_q6_k(blob: np.ndarray, n_cols: int) -> np.ndarray: + """块 = ql(128) qh(64) scales(16,int8) d(2) = 210B / 256 元素。""" + ts = 210 + b = _rows_to_blocks(blob, ts) + n = b.shape[0] + ql = b[:, :128] + qh = b[:, 128:192] + sc = b[:, 192:208].view(np.int8).astype(np.float32) + d = _f16(b[:, 208:210]) + lo = (ql[:, Q6_LO_BYTE] >> Q6_LO_SHIFT.astype(np.uint8)) & np.uint8(0x0F) + hi = (qh[:, Q6_HI_BYTE] >> Q6_HI_SHIFT.astype(np.uint8)) & np.uint8(0x03) + q = ((lo | (hi << np.uint8(4))).astype(np.int16) - 32 + ).reshape(n, 16, 16).astype(np.float32) + step = (d * sc).reshape(n, 16, 1) + return (step * q).reshape(blob.shape[0], n_cols) + + +DECODERS = { + QType.Q8_0: decode_q8_0, + QType.Q4_K: decode_q4_k, + QType.Q5_K: decode_q5_k, + QType.Q6_K: decode_q6_k, +} + + +# --------------------------------------------------------------------------- +# A. 容器 / 字节布局 / block 位运算 +# --------------------------------------------------------------------------- + +def pick_samples(tensors: dict[str, object], per_type: int = 3) -> list: + """每种量化类型最多挑 per_type 个（按 (in,out) 形状去重），只解部分行以省时。""" + by_type = collections.defaultdict(list) + for name, t in tensors.items(): + qt = QType(int(t.tensor_type)) + if qt in DECODERS and name.startswith("blk.") and ".nextn." not in name: + by_type[qt].append(t) + out = [] + for qt, lst in sorted(by_type.items(), key=lambda kv: int(kv[0])): + seen = set() + picked = 0 + for t in sorted(lst, key=lambda x: x.name): + key = (int(t.shape[0]), int(t.shape[1])) + if key in seen: + continue + seen.add(key) + out.append(t) + picked += 1 + if picked >= per_type: + break + return out + + +def section_a(reader) -> None: + print("\n== A. 容器与 block 位运算（逐比特）==") + tensors = {t.name: t for t in reader.tensors} + samples = pick_samples(tensors) + assert samples, "未取到任何样本" + all_ok = True + for t in samples: + qt = QType(int(t.tensor_type)) + bs, ts = GGML_QUANT_SIZES[int(qt)] + n_in, n_out = int(t.shape[0]), int(t.shape[1]) # GGML: ne[0]=in, ne[1]=out + row_bytes = n_in // bs * ts + blob = np.ascontiguousarray(t.data) # 解析器已给 [out, row_bytes] + ok_shape = blob.shape == (n_out, row_bytes) + dec = DECODERS[qt] + n_rows = min(64, n_out) # 只解前 n_rows 行，省时 + ours = dec(blob[:n_rows], n_in) + ref_full = gq.dequantize(blob[:n_rows], qt) # 权威实现，输入为字节形状 + ref = np.asarray(ref_full, dtype=np.float32) + exact = ours.shape == ref.shape and np.array_equal(ours, ref) + # 单行独立性：逐行解码必须与整体解码一致（证明行是连续独立单元） + one = dec(blob[7:8], n_in) + indep = np.array_equal(one, ref[7:8]) + all_ok &= check( + f"{t.name} {qt.name} in={n_in} out={n_out} row_bytes={row_bytes}", + ok_shape and exact and indep, + f"bit-exact={exact} row-indep={indep}", + ) + check("A 汇总", all_ok) + # 非量化张量的轴序（打包器是否需要转置的依据） + conv = tensors["blk.0.ssm_conv1d.weight"] + check("F32 张量的 data 也是 C 序 [shape[1], shape[0]]（= HF 取向，打包器不转置）", + conv.data.shape == (int(conv.shape[1]), int(conv.shape[0])), + f"ne={list(map(int, conv.shape))} data={conv.data.shape} -> HF [10240,1,4]") + norm = tensors["blk.0.attn_norm.weight"] + check("1-D norm 保持 dtype=float32且长度 = hidden", + norm.data.dtype == np.float32 and norm.data.shape == (5120,)) + + +# --------------------------------------------------------------------------- +# B. 对齐事实 +# --------------------------------------------------------------------------- + +def section_b() -> None: + print("\n== B. 对齐事实（kernel 的硬约束）==") + facts = [] + for qt in (QType.Q8_0, QType.Q4_K, QType.Q5_K, QType.Q6_K, QType.IQ4_NL, QType.IQ4_XS): + bs, ts = GGML_QUANT_SIZES[int(qt)] + align_block = 2 if ts % 2 == 0 else 1 + for n_in in (5120, 6144, 10240, 17408, 248320): + if n_in % bs: + continue + rb = n_in // bs * ts + a = 16 + while a > 1 and rb % a: + a //= 2 + facts.append((qt.name, bs, ts, n_in, rb, a, align_block)) + print(f" {'type':8s} {'bs':>4s} {'ts':>4s} {'in':>7s} {'row_bytes':>10s} " + f"{'行对齐':>7s} {'块起始对齐':>10s}") + worst_row, worst_block = 16, 2 + for name, bs, ts, n_in, rb, a, ab in facts: + print(f" {name:8s} {bs:4d} {ts:4d} {n_in:7d} {rb:10d} {str(a)+'B':>7s} {str(ab)+'B':>10s}") + worst_row = min(worst_row, a) + worst_block = min(worst_block, ab) + check("块起始地址仅保证 2B 对齐（Q6_K=210B / Q8_0=34B 非 4 倍数）", + worst_block == 2, f"min_block_align={worst_block}B") + check("Q6_K 在 in=5120/17408 时行 stride 仅 8B 对齐", + any(f[0] == "Q6_K" and f[5] == 8 for f in facts)) + print(" -> 结论：kernel 不得对单块起始地址做 >2B 向量化加载假设；容器不做 pad。") + + +# --------------------------------------------------------------------------- +# C. V 头重排（grouped <-> tiled） +# --------------------------------------------------------------------------- + +def reorder_v(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: + """与 llama.cpp conversion/qwen.py::_reorder_v_heads 同语义（沿 dim0 的整头置换）。""" + rest = t.shape[1:] + return (t.reshape((n_k, n_v_per_k, hd) + rest) + .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) + .reshape((n_k * n_v_per_k * hd,) + rest)) + + +def reorder_v_inverse(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: + """逆变换 = 两个轴参数对调后再调用一次。""" + rest = t.shape[1:] + return (t.reshape((n_v_per_k, n_k, hd) + rest) + .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) + .reshape((n_k * n_v_per_k * hd,) + rest)) + + +def section_c() -> None: + print("\n== C. V 头重排（执行方案 §2.7）==") + n_k, n_v_per_k, hd = 16, 3, 128 # Qwen3.8: 16 key heads, 48 value heads + n_v = n_k * n_v_per_k + rng = np.random.default_rng(0) + + grouped = rng.standard_normal((n_v * hd, 7)).astype(np.float32) + tiled = reorder_v(grouped, n_k, n_v_per_k, hd) + back = reorder_v_inverse(tiled, n_k, n_v_per_k, hd) + check("grouped -> tiled -> grouped 自等", np.array_equal(grouped, back)) + check("reorder_v 是整头搬运（每个 head 的 hd 行连续不被打散）", + all(np.array_equal(tiled[i * hd:(i + 1) * hd], + grouped[((i % n_k) * n_v_per_k + i // n_k) * hd + :((i % n_k) * n_v_per_k + i // n_k) * hd + hd]) + for i in range(n_v))) + + # 槽位 j（value head 编号）-> 真实 k 头 的两种语义 + k_grouped = [j // n_v_per_k for j in range(n_v)] # InfiniCore kernel 的假设 + k_tiled = [j % n_k for j in range(n_v)] # GGUF(tiled) 的真实归属 + check("tiled 序直接喂给 `value_head_idx / value_heads_per_key_head` 会错配 k 头", + k_grouped != k_tiled, + f"错配槽位数={sum(a != b for a, b in zip(k_grouped, k_tiled))}/{n_v}") + + # 逆重排后回到 grouped 语义 + _ = np.repeat(np.arange(n_v), hd) # labels 仅用于形状参考 + check("逆变换后槽位归属恢复 grouped 语义", + np.array_equal( + reorder_v_inverse( + np.array([k * n_v_per_k + v for v in range(n_v_per_k) for k in range(n_k)]), + n_k, n_v_per_k, 1), + np.arange(n_v)), + "逆变换后 slot i 的 head 编号 = i，kernel 的 k = i // n_v_per_k 成立") + check("in_proj_a/b・A_log・dt_bias 的 head_dim=1 退化形式（逐元素置换）同样自等", + np.array_equal( + reorder_v_inverse(reorder_v(np.arange(n_v), n_k, n_v_per_k, 1), + n_k, n_v_per_k, 1), + np.arange(n_v))) + check("多维情形（如 conv1d 的 [channels, 1, kernel]）仅置换头维、尾部轴不动", + np.array_equal( + reorder_v_inverse(reorder_v(grouped[:, :1], n_k, n_v_per_k, hd), + n_k, n_v_per_k, hd), + grouped[:, :1])) + # 行置换对量化 blob 是「整块搬运」：以 Q6_K 为例验证字节级可置换性 + bs, ts = GGML_QUANT_SIZES[int(QType.Q6_K)] + row_bytes = 5120 // bs * ts + blob = rng.integers(0, 256, size=(n_v, row_bytes), dtype=np.uint8) + perm = np.arange(n_v)[::-1].copy() + check("量化 blob 的行置换 == 字节整行置换（无需重新量化）", + np.array_equal(blob[perm], np.ascontiguousarray(blob)[perm])) + print(" -> 结论：整行置换可字节级完成；ssm_out 的列(in 维)置换不可，改用运行时激活 gather。") + + +# --------------------------------------------------------------------------- +# D. 命名 / 形状契约 +# --------------------------------------------------------------------------- + +def gguf_meta(reader, suffix: str): + """元数据键带架构前缀（qwen35.*），允许传短名；contents() 对单元素返回标量，统一成列表。""" + for key in (f"qwen35.{suffix}", f"general.{suffix}", suffix): + if key in reader.fields: + v = reader.fields[key].contents() + return v if isinstance(v, (list, tuple, np.ndarray)) else [v] + raise KeyError(f"GGUF 元数据缺少：{suffix}（qwen35./general. 前缀均未命中）") + + +def section_d(reader) -> None: + print("\n== D. GGUF 张量集合 vs 打包器映射表 ==") + tensors = {t.name: t for t in reader.tensors} + n_layer_gguf = int(gguf_meta(reader, "block_count")[0]) + interval = int(gguf_meta(reader, "full_attention_interval")[0]) + n_main = 64 + full = [i for i in range(n_main) if (i + 1) % interval == 0] + gdn = [i for i in range(n_main) if i not in full] + check("主模型层数 64（block_count 含 1 个 MTP 层）", + n_layer_gguf == n_main + 1, f"block_count={n_layer_gguf}") + check("full attention 层 = 3,7,...,63 共 16 层", + len(full) == 16 and full[0] == 3 and full[-1] == 63) + check("GDN 层 48 层", len(gdn) == 48) + + need_full = ["attn_norm.weight", "post_attention_norm.weight", "attn_q.weight", + "attn_k.weight", "attn_v.weight", "attn_output.weight", + "attn_q_norm.weight", "attn_k_norm.weight", + "ffn_gate.weight", "ffn_up.weight", "ffn_down.weight"] + need_gdn = ["attn_norm.weight", "post_attention_norm.weight", "attn_qkv.weight", + "attn_gate.weight", "ssm_a", "ssm_alpha.weight", "ssm_beta.weight", + "ssm_conv1d.weight", "ssm_dt.bias", "ssm_norm.weight", "ssm_out.weight", + "ffn_gate.weight", "ffn_up.weight", "ffn_down.weight"] + missing = [] + for i in full: + missing += [f"blk.{i}.{r}" for r in need_full if f"blk.{i}.{r}" not in tensors] + for i in gdn: + missing += [f"blk.{i}.{r}" for r in need_gdn if f"blk.{i}.{r}" not in tensors] + check("64 层全部所需张量存在", not missing, f"missing={missing[:6]}") + + shapes = { + "attn_q": (5120, 12288), "attn_k": (5120, 1024), "attn_v": (5120, 1024), + "attn_output": (6144, 5120), "attn_qkv": (5120, 10240), "attn_gate": (5120, 6144), + "ssm_out": (6144, 5120), "ffn_gate": (5120, 17408), "ffn_up": (5120, 17408), + "ffn_down": (17408, 5120), "ssm_conv1d": (4, 10240), + } + bad = [] + for name, want in shapes.items(): + probe = {"attn_q": f"blk.{full[0]}.", "attn_k": f"blk.{full[0]}.", + "attn_v": f"blk.{full[0]}.", "attn_output": f"blk.{full[0]}.", + "attn_qkv": f"blk.{gdn[0]}.", "attn_gate": f"blk.{gdn[0]}.", + "ssm_out": f"blk.{gdn[0]}.", "ssm_conv1d": f"blk.{gdn[0]}.", + "ffn_gate": f"blk.{0}.", "ffn_up": f"blk.{0}.", "ffn_down": f"blk.{0}."}[name] + t = tensors.get(probe + name + ".weight") + if t is None or (int(t.shape[0]), int(t.shape[1])) != want: + bad.append((name, None if t is None else list(map(int, t.shape)))) + check("代表张量 (in,out) 与映射表一致", not bad, f"bad={bad}") + + # attn_q 的 12288 = 24 * (256 q + 256 gate) 交错 + n_q, hd_q, n_kv, hd_k = 24, 256, 4, 256 + check("attn_q 行数 = n_q*head*2（q 与 gate 每头交错）", + shapes["attn_q"][1] == n_q * hd_q * 2) + check("Qwen35FusedQKVLinear 期望 out = 12288 + 1024 + 1024 = 14336", + 12288 + 1024 + 1024 == 14336) + check("GDN in_proj_qkv 行数 = q2048 + k2048 + v6144 = 10240", + 2048 + 2048 + 6144 == shapes["attn_qkv"][1]) + check("conv 通道 = 2*head_k*n_k + head_v*n_v = 10240", + 2 * 128 * 16 + 128 * 48 == 10240) + mtp = [n for n in tensors if n.startswith("blk.64.")] + nextn = [n for n in tensors if ".nextn." in n] + check("MTP 丢弃规则 = 整块 blk.64.*（不止 .nextn.*，包含完整一层）", + len(mtp) == 15 and len(nextn) == 4, + f"blk.64.*={len(mtp)} 个（其中 .nextn.* 仅 {len(nextn)} 个）") + max_blk = max(int(n.split(".")[1]) for n in tensors if n.startswith("blk.")) + check("块号集合 = 0..64（64 主层 + 1 MTP 层，无其它残留）", + max_blk == 64 and len({int(n.split(".")[1]) for n in tensors if n.startswith("blk.")}) == 65, + f"max_blk={max_blk}") + + +# --------------------------------------------------------------------------- +# F. 打包器字节核算（修正后的 MTP 规则） +# --------------------------------------------------------------------------- + +# 阶段 3 kernel 需直接吃块的格式集合不再在本文件定义：见 gguf_mapping.NATIVE_BLOB_TYPES +# （由 gguf_routeb_shape_contract.py 对真文件校验），避免两处清单漂移。 + + +def section_f(reader) -> None: + print("\n== F. 打包器字节核算 ==") + GiB = 2 ** 30 + tensors = {t.name: t for t in reader.tensors} + # 核算必须由映射表驱动：之前本脚本自写一套分桶，把 7 个 IQ4 张量当成“反量化”、 + # 把实为 Q8_0 的 ssm_alpha/ssm_beta（框架不能量化它们）当成 blob，两处失真共 + # 高估 0.70 GiB。单一事实源 = gguf_mapping.build_plan(REAL)。 + import gguf_mapping as M + plan = M.build_plan(M.REAL) + _tn = {int(v.value): str(v.name) for v in QType} + M.apply_v1_exceptions(plan, {n: _tn[int(t.tensor_type)] for n, t in tensors.items()}) + blob_src = {e.gguf for e in plan if e.blob and e.gguf in tensors} + dense_e = [e for e in plan if not e.blob] + bucket = collections.Counter() + cnt = collections.Counter() + for n in blob_src: + bucket[f"U8 blob {QType(int(tensors[n].tensor_type)).name}"] += int(tensors[n].n_bytes) + cnt[f"U8 blob {QType(int(tensors[n].tensor_type)).name}"] += 1 + d_emb = sum(int(np.prod(e.shape)) * 2 for e in dense_e + if e.gguf in ("token_embd.weight", "output.weight")) + d_other = sum(int(np.prod(e.shape)) * 2 for e in dense_e + if e.gguf not in ("token_embd.weight", "output.weight")) + bucket["BF16 稠密(emb/lm_head)"] = d_emb + cnt["BF16 稠密(emb/lm_head)"] = 2 + bucket["BF16 稠密(其余稠密化条目)"] = d_other + cnt["BF16 稠密(其余稠密化条目)"] = len(dense_e) - 2 + bucket["丢弃(MTP)"] = sum(int(t.n_bytes) for n, t in tensors.items() + if n.startswith("blk.64.")) + cnt["丢弃(MTP)"] = sum(1 for n in tensors if n.startswith("blk.64.")) + + total = sum(v / GiB for k, v in bucket.items() if k != "丢弃(MTP)") + for k in sorted(bucket): + print(f" {k:26s} {bucket[k] / GiB:8.3f} GiB ({cnt[k]:4d} 条目)") + print(f" {'-'*52}") + print(f" v1 加载后权重合计 {total:8.3f} GiB") + check("v1 权重合计 ≤ 24.0 GiB（单卡 32607 MiB 可容纳权重+KV+激活）", + total <= 24.0, f"total={total:.3f} GiB") + check("MTP 丢弃量 < 0.4 GiB（不影响预算）", + bucket["丢弃(MTP)"] / GiB < 0.4, f"{bucket['丢弃(MTP)'] / GiB:.3f} GiB") + check("v1 blob 桶恰好只含阶段 3 实现的 4 种类型", + {k.replace("U8 blob ", "") for k in bucket if k.startswith("U8 blob ")} + == set(M.NATIVE_BLOB_TYPES), + f"{sorted(k for k in bucket if k.startswith('U8'))}") + check("blob 条目数与映射表一致", + sum(cnt[k] for k in bucket if k.startswith("U8")) + == len({e.gguf for e in plan if e.blob}), + f"{sum(cnt[k] for k in bucket if k.startswith('U8'))}") + + emb, out = tensors["token_embd.weight"], tensors["output.weight"] + check("token_embd / output 也是量化的（Q6_K / Q8_0），v1 必须反量化它们", + int(emb.tensor_type) == int(QType.Q6_K) and int(out.tensor_type) == int(QType.Q8_0), + f"emb={emb.tensor_type} out={out.tensor_type}") + check("emb/output 均为 [hidden, vocab] 且 vocab 与元数据一致", + list(map(int, emb.shape)) == list(map(int, out.shape)) == [5120, 248320] + and len(gguf_meta(reader, "tokenizer.ggml.tokens")) == 248320, + f"shape={list(map(int, emb.shape))}") + print(" -> 阶段 6 可选项：emb 走 Q6_K 行 gather-dequant、lm_head 走 linear_gguf(Q8_0)，" + f"可再省 ≈ {(d_emb - (emb.n_bytes + out.n_bytes)) / GiB:.2f} GiB") + + +# --------------------------------------------------------------------------- +# E. 元数据 -> config.json +# --------------------------------------------------------------------------- + +def section_e(reader) -> None: + print("\n== E. 元数据与 config.json 依据 ==") + + def kv(suffix, idx=0): + return gguf_meta(reader, suffix)[idx] + + rope_secs = [int(x) for x in gguf_meta(reader, "rope.dimension_sections")] + dim_cnt = int(kv("rope.dimension_count")) + base = float(kv("rope.freq_base")) + eps = float(kv("attention.layer_norm_rms_epsilon")) + head_dim = int(kv("attention.key_length")) + check("head_dim = key_length = value_length = 256", + head_dim == int(kv("attention.value_length")) == 256) + check("partial rotary: dimension_count=64, head_dim=256 -> factor 0.25", + dim_cnt == 64 and dim_cnt * 4 == head_dim, f"dimension_count={dim_cnt}") + check("mrope sections [11,11,10,0] 之和 = 32 = dimension_count/2", + sum(rope_secs) == dim_cnt // 2, f"sections={rope_secs}") + check("rope_theta = 1e7", base == 1e7, f"base={base}") + check("mtp 层数声明为 1（与 block_count=65 = 64+1 一致）", + int(kv("nextn_predict_layers")) == 1) + n_k = int(kv("ssm.group_count")) + inner = int(kv("ssm.inner_size")) + st = int(kv("ssm.state_size")) + dt = int(kv("ssm.time_step_rank")) + check("ssm: inner 6144 / group 16 / state 128 / time_step_rank 48 / conv 4", + (inner, n_k, st, dt, int(kv("ssm.conv_kernel"))) == (6144, 16, 128, 48, 4)) + check("value heads = inner/state = 48 = time_step_rank（两路推导一致）", + inner // st == dt == 48, f"inner/state={inner // st} time_step_rank={dt}") + check("num_k_heads * state = 2048 = q/k 段长度", + n_k * st == 2048) + vocab = len(gguf_meta(reader, "tokenizer.ggml.tokens")) + print(f" arch={gguf_meta(reader, 'architecture')[0]!r} " + f"name={gguf_meta(reader, 'name')[0]!r} rms_eps={eps:g} " + f"ctx={int(kv('context_length'))} vocab={vocab} " + f"heads={int(kv('attention.head_count'))}/{int(kv('attention.head_count_kv'))} " + f"hidden={int(kv('embedding_length'))} ffn={int(kv('feed_forward_length'))}") + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--gguf", default="/home/liuxd/models/Qwen3.8-27B-GGUF/" + "Qwen3.8-27B-UD-Q6_K.gguf") + args = ap.parse_args() + print(f"审计对象：{args.gguf}\n大小：{os.path.getsize(args.gguf):,} bytes") + reader = GGUFReader(args.gguf) + section_a(reader) + section_b() + section_c() + section_d(reader) + section_e(reader) + section_f(reader) + print(f"\n===== 结果：PASS {len(PASSED)} / FAIL {len(FAILED)} =====") + if FAILED: + for f in FAILED: + print(" FAIL:", f) + return 1 + print("阶段 0 全部通过，可进入阶段 1（打包器）。") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_routeb_blocks_probe.cpp b/scripts/gguf_routeb_blocks_probe.cpp new file mode 100644 index 000000000..d0f57aadc --- /dev/null +++ b/scripts/gguf_routeb_blocks_probe.cpp @@ -0,0 +1,82 @@ +// Host-side driver for ggml_blocks.h, used by scripts/gguf_routeb_blocks_ref.py. +// +// g++ -O2 -std=c++17 -I /src/infiniop/ops/linear_gguf \ +// gguf_routeb_blocks_probe.cpp -o blocks_probe_host +// +// blocks_probe_host +// +// This file is a test harness, not part of any library target: it exists so the +// decoders can be checked against numpy / gguf-py block by block before the +// linear_gguf kernels exist. +#include +#include +#include + +#include "ggml_blocks.h" + +int main(int argc, char **argv) { + if (argc != 6) { + std::fprintf(stderr, + "usage: %s \n", + argv[0]); + return 2; + } + const int32_t type = std::atoi(argv[1]); + const int64_t n_blocks = std::atoll(argv[2]); + const int32_t bytes = ggml_blocks::block_bytes(type); + const int32_t elems = ggml_blocks::block_elems(type); + if (bytes < 0 || elems < 0) { + std::fprintf(stderr, "probe: ggml type %d has no decoder here\n", type); + return 3; + } + if (n_blocks <= 0) { + std::fprintf(stderr, "probe: n_blocks must be positive\n"); + return 2; + } + + FILE *in = std::fopen(argv[3], "rb"); + if (!in) { + std::fprintf(stderr, "probe: cannot open %s\n", argv[3]); + return 4; + } + const size_t want = (size_t)n_blocks * bytes; + std::vector buf(want); + if (std::fread(buf.data(), 1, want, in) != want) { + std::fprintf(stderr, "probe: short read on %s (wanted %zu)\n", argv[3], want); + std::fclose(in); + return 4; + } + std::fclose(in); + + std::vector f32((size_t)n_blocks * elems); + std::vector bf16((size_t)n_blocks * elems); + if (!ggml_blocks::decode_blocks(type, buf.data(), n_blocks, f32.data())) { + std::fprintf(stderr, "probe: decode_blocks failed\n"); + return 3; + } + if (!ggml_blocks::decode_blocks_bf16(type, buf.data(), n_blocks, + bf16.data())) { + std::fprintf(stderr, "probe: decode_blocks_bf16 failed\n"); + return 3; + } + + FILE *o1 = std::fopen(argv[4], "wb"); + FILE *o2 = std::fopen(argv[5], "wb"); + if (!o1 || !o2) { + std::fprintf(stderr, "probe: cannot open output files\n"); + return 4; + } + const size_t n_f32 = f32.size() * sizeof(float); + const size_t n_bf16 = bf16.size() * sizeof(uint16_t); + const bool ok = std::fwrite(f32.data(), 1, n_f32, o1) == n_f32 && + std::fwrite(bf16.data(), 1, n_bf16, o2) == n_bf16; + std::fclose(o1); + std::fclose(o2); + if (!ok) { + std::fprintf(stderr, "probe: short write\n"); + return 4; + } + std::printf("probe host type=%d n_blocks=%lld elems=%d ok\n", type, + (long long)n_blocks, elems); + return 0; +} diff --git a/scripts/gguf_routeb_blocks_probe.cu b/scripts/gguf_routeb_blocks_probe.cu new file mode 100644 index 000000000..9ada52ba7 --- /dev/null +++ b/scripts/gguf_routeb_blocks_probe.cu @@ -0,0 +1,125 @@ +// Device-side driver for ggml_blocks.h, used by scripts/gguf_routeb_blocks_ref.py. +// +// nvcc -O2 -std=c++17 -I /src/infiniop/ops/linear_gguf \ +// gguf_routeb_blocks_probe.cu -o blocks_probe_cuda +// +// blocks_probe_cuda +// +// Same job as gguf_routeb_blocks_probe.cpp, but every block is decoded by one +// thread through the very same ggml_blocks.h entry points, which is what proves +// the header is device-safe (no host-only call, no unaligned struct punning) and +// that the host and device results are bit-identical. +#include +#include +#include + +#include "ggml_blocks.h" + +__global__ void decode_f32_kernel(int32_t type, const uint8_t *blk, int64_t n_blocks, + int32_t bytes, int32_t elems, float *out) { + const int64_t i = (int64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n_blocks) return; + ggml_blocks::decode_blocks(type, blk + (int64_t)i * bytes, 1, out + i * elems); +} + +__global__ void decode_bf16_kernel(int32_t type, const uint8_t *blk, int64_t n_blocks, + int32_t bytes, int32_t elems, uint16_t *out) { + const int64_t i = (int64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n_blocks) return; + ggml_blocks::decode_blocks_bf16(type, blk + (int64_t)i * bytes, 1, + out + i * elems); +} + +#define CUDA_CHECK(call) \ + do { \ + cudaError_t err__ = (call); \ + if (err__ != cudaSuccess) { \ + std::fprintf(stderr, "probe cuda: %s failed: %s\n", #call, \ + cudaGetErrorString(err__)); \ + return 5; \ + } \ + } while (0) + +int main(int argc, char **argv) { + if (argc != 6) { + std::fprintf(stderr, + "usage: %s \n", + argv[0]); + return 2; + } + const int32_t type = std::atoi(argv[1]); + const int64_t n_blocks = std::atoll(argv[2]); + const int32_t bytes = ggml_blocks::block_bytes(type); + const int32_t elems = ggml_blocks::block_elems(type); + if (bytes < 0 || elems < 0) { + std::fprintf(stderr, "probe cuda: ggml type %d has no decoder here\n", type); + return 3; + } + if (n_blocks <= 0) { + std::fprintf(stderr, "probe cuda: n_blocks must be positive\n"); + return 2; + } + + FILE *in = std::fopen(argv[3], "rb"); + if (!in) { + std::fprintf(stderr, "probe cuda: cannot open %s\n", argv[3]); + return 4; + } + const size_t want = (size_t)n_blocks * bytes; + std::vector buf(want); + const size_t got = std::fread(buf.data(), 1, want, in); + std::fclose(in); + if (got != want) { + std::fprintf(stderr, "probe cuda: short read on %s (wanted %zu, got %zu)\n", argv[3], + want, got); + return 4; + } + + uint8_t *d_blk = nullptr; + float *d_f32 = nullptr; + uint16_t *d_bf16 = nullptr; + CUDA_CHECK(cudaMalloc(&d_blk, want)); + CUDA_CHECK(cudaMalloc(&d_f32, (size_t)n_blocks * elems * sizeof(float))); + CUDA_CHECK(cudaMalloc(&d_bf16, (size_t)n_blocks * elems * sizeof(uint16_t))); + CUDA_CHECK(cudaMemcpy(d_blk, buf.data(), want, cudaMemcpyHostToDevice)); + + const int threads = 256; + const int64_t blocks_grid = (n_blocks + threads - 1) / threads; + decode_f32_kernel<<<(unsigned)blocks_grid, threads>>>(type, d_blk, n_blocks, bytes, elems, + d_f32); + CUDA_CHECK(cudaGetLastError()); + decode_bf16_kernel<<<(unsigned)blocks_grid, threads>>>(type, d_blk, n_blocks, bytes, elems, + d_bf16); + CUDA_CHECK(cudaGetLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + std::vector h_f32((size_t)n_blocks * elems); + std::vector h_bf16((size_t)n_blocks * elems); + CUDA_CHECK(cudaMemcpy(h_f32.data(), d_f32, h_f32.size() * sizeof(float), + cudaMemcpyDeviceToHost)); + CUDA_CHECK(cudaMemcpy(h_bf16.data(), d_bf16, h_bf16.size() * sizeof(uint16_t), + cudaMemcpyDeviceToHost)); + cudaFree(d_blk); + cudaFree(d_f32); + cudaFree(d_bf16); + + FILE *o1 = std::fopen(argv[4], "wb"); + FILE *o2 = std::fopen(argv[5], "wb"); + if (!o1 || !o2) { + std::fprintf(stderr, "probe cuda: cannot open output files\n"); + return 4; + } + const size_t n_f32 = h_f32.size() * sizeof(float); + const size_t n_bf16 = h_bf16.size() * sizeof(uint16_t); + const bool ok = std::fwrite(h_f32.data(), 1, n_f32, o1) == n_f32 && + std::fwrite(h_bf16.data(), 1, n_bf16, o2) == n_bf16; + std::fclose(o1); + std::fclose(o2); + if (!ok) { + std::fprintf(stderr, "probe cuda: short write\n"); + return 4; + } + std::printf("probe cuda type=%d n_blocks=%lld elems=%d ok\n", type, (long long)n_blocks, + elems); + return 0; +} diff --git a/scripts/gguf_routeb_blocks_ref.py b/scripts/gguf_routeb_blocks_ref.py new file mode 100644 index 000000000..db56508b6 --- /dev/null +++ b/scripts/gguf_routeb_blocks_ref.py @@ -0,0 +1,616 @@ +#!/usr/bin/env python3 +""" +InfiniLM 路线 B —— 阶段 3.1 验收：ggml_blocks.h 的 block 解码位精正确认 + +四方交叉，任何两方不一致都会炸出来： + + A. numpy reference（本文件）：照 llama.cpp `ggml/src/ggml-quants.c` 的 + `dequantize_row_q8_0/q4_K/q5_K/q6_K` 标量语义逐行翻过来，含 + `get_scale_min_k4` 的 6-bit 解包与**浮点结合顺序**（先 d*scale 再碰 quant）。 + B. gguf-py 的 numpy 实现（`gguf.quants.Q8_0/Q4_K/Q5_K/Q6_K.dequantize_blocks`）： + 它是阶段 4「单 block 级 max|Δ| == 0」的基准。它解包 scale 用的是 + reshape/split 另一条路径，与 A 相互独立 —— 两边逐位相同才说明 6-bit + 打包的解读没读歪。 + C. 被测对象 `InfiniCore/src/infiniop/ops/linear_gguf/ggml_blocks.h`，经 + `scripts/gguf_routeb_blocks_probe.cpp` 编出的 host driver 跑真数据。 + D. 同一个头经 `scripts/gguf_routeb_blocks_probe.cu` 编出的 CUDA driver： + 证明这个头确实设备无关（GPU 上编得过、跑得动、与 host 逐位相同）， + 顺带验证 bf16 舍入 `float_to_bf16()` 与 torch 的 `.to(bfloat16)` 一致。 + +样本来自真实打包产物里 `*.weight_bytes` 的 block 字节，再加一批手造边界 block +（次正规 half、int8 scale = -128、scale/min 全 63、全 FF），因为 K-quant 的 +scale 解包最容易在极值上翻车。随机 block 的 half 域被限制为有限值，好让判据 +能要求 100% 逐位相同，而不是退化成"近似"。 + +用法： + /usr/bin/python3 scripts/gguf_routeb_blocks_ref.py \ + [--model-path /home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8] \ + [--blocks 20000] [--no-cuda] [--keep] +退出码 0 = 全部 PASS。 +""" + +from __future__ import annotations + +import argparse +import collections +import json +import os +import re +import struct +import subprocess +import sys + +import numpy as np + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_LLAMA_CPP = os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp") +_INFINICORE = os.environ.get("INFINICORE_DIR", "/home/liuxd/InfiniCore") +sys.path.insert(0, os.path.join(_LLAMA_CPP, "gguf-py")) + +import gguf.quants as gq # noqa: E402 +from gguf.constants import GGML_QUANT_SIZES, GGMLQuantizationType as Q # noqa: E402 + +HEADER_DIR = os.path.join(_INFINICORE, "src", "infiniop", "ops", "linear_gguf") +PROBE_CPP = os.path.join(_HERE, "gguf_routeb_blocks_probe.cpp") +PROBE_CU = os.path.join(_HERE, "gguf_routeb_blocks_probe.cu") + +TYPES = (8, 12, 13, 14) # 与 pack_report.json 的 blob_type_ids 一致 +QK_K, QK8_0 = 256, 32 +TYPE_SIZE = {t: GGML_QUANT_SIZES[Q(t)][1] for t in TYPES} +BLOCK_SIZE = {t: GGML_QUANT_SIZES[Q(t)][0] for t in TYPES} + +_PASS = 0 +_FAIL = 0 +_SKIP = 0 + + +def check(name, ok, detail=""): + global _PASS, _FAIL + if ok: + _PASS += 1 + print(" PASS %s" % name) + else: + _FAIL += 1 + print(" FAIL %s%s" % (name, ("\n %s" % detail) if detail else "")) + return ok + + +def skip(name, why): + global _SKIP + _SKIP += 1 + print(" SKIP %s（%s）" % (name, why)) + + +# ------------------------------------------------------- A. numpy 参考实现 +def _u16_le(col0, col1): + return col0.astype(np.uint32) | (col1.astype(np.uint32) << np.uint32(8)) + + +def half_to_float(bits): + """IEEE binary16 -> float32，等价于 ggml FP16_TO_FP32 / __half2float。""" + bits = np.asarray(bits, np.uint32) + sign = (bits >> np.uint32(15)) << np.uint32(31) + exp = (bits >> np.uint32(10)) & np.uint32(0x1F) + mant = bits & np.uint32(0x3FF) + out = np.zeros(bits.shape, np.uint32) + zneg = (exp == 0) & (mant == 0) # ±0：符号位必须留住，否则 -0.0 被写成正零 + out[zneg] = sign[zneg] + norm = (exp != 0) & (exp != 31) + out[norm] = sign[norm] | ((exp[norm] + np.uint32(112)) << np.uint32(23)) | ( + mant[norm] << np.uint32(13)) + special = exp == 31 + out[special] = sign[special] | np.uint32(0x7F800000) | (mant[special] << np.uint32(13)) + sub = (exp == 0) & (mant != 0) + if sub.any(): + m = mant[sub].astype(np.int64) + e = np.full(m.shape, -14, np.int64) + for _ in range(11): + need = (m & 0x400) == 0 + if not need.any(): + break + m[need] <<= 1 + e[need] -= 1 + out[sub] = (sign[sub].astype(np.int64) | ((e + 127) << 23) + | ((m & 0x3FF) << 13)).astype(np.uint32) + return out.view(np.float32) + + +def float_to_bf16_bits(f): + """binary32 -> bf16 位模式，round-to-nearest-even，与头里那份同语义。""" + b = np.asarray(f, np.float32).view(np.uint32).astype(np.int64) + exp = (b >> 23) & 0xFF + nan = (exp == 0xFF) & ((b & 0x7FFFFF) != 0) + bias = 0x7FFF + ((b >> 16) & 1) + out = ((b + bias) >> 16).astype(np.uint32) + out[nan] = ((b[nan] >> 16) | 0x0040).astype(np.uint32) + return out.astype(np.uint16) + + +def get_scale_min_k4(scales): + """q4_K / q5_K：12 字节 -> 8 组 (scale, min)，照抄 ggml-quants.c 的分支。""" + nb = scales.shape[0] + d = np.empty((nb, 8), np.uint8) + m = np.empty((nb, 8), np.uint8) + for j in range(8): + if j < 4: + d[:, j] = scales[:, j] & 63 + m[:, j] = scales[:, j + 4] & 63 + else: + d[:, j] = (scales[:, j + 4] & 0xF) | ((scales[:, j - 4] >> 6) << 4) + m[:, j] = (scales[:, j + 4] >> 4) | ((scales[:, j] >> 6) << 4) + return d, m + + +def ref_q8_0(blk): + nb = blk.shape[0] + d = half_to_float(_u16_le(blk[:, 0], blk[:, 1])).reshape(nb, 1) + q = blk[:, 2:34].view(np.int8).astype(np.float32) + return q * d # C: qs[j] * d + + +def ref_q4_K(blk): + nb = blk.shape[0] + d = half_to_float(_u16_le(blk[:, 0], blk[:, 1])) + dmin = half_to_float(_u16_le(blk[:, 2], blk[:, 3])) + sc, m = get_scale_min_k4(blk[:, 4:16]) + d_eff = (d[:, None] * sc.astype(np.float32)).reshape(nb, 8, 1) + m_eff = (dmin[:, None] * m.astype(np.float32)).reshape(nb, 8, 1) + qs = blk[:, 16:144].reshape(nb, 4, 32) + q = np.stack([qs & 0xF, qs >> 4], axis=2).reshape(nb, 8, 32).astype(np.float32) + return (d_eff * q - m_eff).reshape(nb, QK_K) + + +def ref_q5_K(blk): + nb = blk.shape[0] + d = half_to_float(_u16_le(blk[:, 0], blk[:, 1])) + dmin = half_to_float(_u16_le(blk[:, 2], blk[:, 3])) + sc, m = get_scale_min_k4(blk[:, 4:16]) + d_eff = (d[:, None] * sc.astype(np.float32)).reshape(nb, 8, 1) + m_eff = (dmin[:, None] * m.astype(np.float32)).reshape(nb, 8, 1) + qs = blk[:, 48:176].reshape(nb, 4, 32) + qh = blk[:, 16:48][:, None, :] + lo_shift = (2 * np.arange(4)).reshape(4, 1) # u1 = 1 << 2g + hi_shift = lo_shift + 1 # u2 = 2 << 2g + lo = (qs & 0xF) | (((qh >> lo_shift) & 1) << 4).astype(np.uint8) + hi = (qs >> 4) | (((qh >> hi_shift) & 1) << 4).astype(np.uint8) + q = np.stack([lo, hi], axis=2).reshape(nb, 8, 32).astype(np.float32) + return (d_eff * q - m_eff).reshape(nb, QK_K) + + +def ref_q6_K(blk): + nb = blk.shape[0] + d = half_to_float(_u16_le(blk[:, 208], blk[:, 209])) + sc = blk[:, 192:208].view(np.int8).astype(np.float32) + d_eff = d[:, None] * sc # (nb,16) 先 d*sc，同 C 结合顺序 + out = np.empty((nb, QK_K), np.float32) + l = np.arange(32) + isidx = l // 16 + for c in (0, 1): + ql = blk[:, 64 * c:64 * c + 64] + qh = blk[:, 128 + 32 * c:128 + 32 * c + 32] + base = 128 * c + q1 = ((ql[:, 0:32] & 0xF) | (((qh >> 0) & 3) << 4)).astype(np.int32) - 32 + q2 = ((ql[:, 32:64] & 0xF) | (((qh >> 2) & 3) << 4)).astype(np.int32) - 32 + q3 = ((ql[:, 0:32] >> 4) | (((qh >> 4) & 3) << 4)).astype(np.int32) - 32 + q4 = ((ql[:, 32:64] >> 4) | ((qh >> 6) << 4)).astype(np.int32) - 32 + for part, (q, off) in enumerate(((q1, 0), (q2, 32), (q3, 64), (q4, 96))): + # C 里每处理一个 128 元素段就 `sc += 8`，所以段 1 的 scale 下标整体偏移 8 + s = d_eff[:, 8 * c + isidx + 2 * part] + out[:, base + off:base + off + 32] = s * q.astype(np.float32) + return out + + +REF = {8: ref_q8_0, 12: ref_q4_K, 13: ref_q5_K, 14: ref_q6_K} + + +def gguf_py_dequant(t, blk): + return getattr(gq, Q(t).name).dequantize_blocks(np.ascontiguousarray(blk)) + + +def check_half_decode(): + """参考实现自己的回归护袋：全部 65536 个 half 位模式与 numpy 硬件转换逐位相同。 + + 次正规 / 0 / inf 都在这 65536 个里，负数那一半特别重要（曾经把符号位 + 当成 bit16 丢掉了，只会让带负 d 的 Q6_K block 整批错）。 + NaN 只要求“也是 NaN”，不比 payload。 + """ + h = np.arange(65536, dtype=np.uint16) + truth = h.view(np.float16).astype(np.float32) + mine = half_to_float(h.astype(np.uint32)) + finite = np.isfinite(truth) + ok = np.array_equal(mine[finite].view(np.uint32), truth[finite].view(np.uint32)) + n_nan = int(np.isnan(truth).sum()) + ok_nan = bool(np.array_equal(np.isnan(mine), np.isnan(truth)) + and np.array_equal(np.isinf(mine) & (mine > 0), np.isinf(truth) & (truth > 0))) + neq = np.flatnonzero(mine.view(np.uint32) != truth.view(np.uint32)) + check("numpy 参考的 half_to_float：有限值逐位相同（%d 个）+ NaN 仍为 NaN（%d 个）" + % (int(finite.sum()), n_nan), ok and ok_nan, + "不同 %d 个，首个 0x%04X：%s vs %s" % (neq.size, int(h[neq[0]]) if neq.size else 0, + float(mine[neq[0]]) if neq.size else 0, + float(truth[neq[0]]) if neq.size else 0)) + + +# ------------------------------------------------- 差异度量（要求逐位相同） +def bitwise_diff(a, b): + """返回 (非有限值个数, 逐位不同的元素数, max|Δ|, 首个差异描述)。""" + fa, fb = np.asarray(a, np.float32), np.asarray(b, np.float32) + bad = ~np.isfinite(fa) | ~np.isfinite(fb) + n_bad = int(bad.sum()) + ok_mask = ~bad + ua = fa[ok_mask].view(np.uint32) + ub = fb[ok_mask].view(np.uint32) + neq = ua != ub + n_diff = int(neq.sum()) + maxabs = float(np.abs(fa[ok_mask] - fb[ok_mask]).max()) if ok_mask.any() else 0.0 + first = "" + if n_diff: + i = int(np.flatnonzero(neq)[0]) + first = ("第 %d 个非有限值以外的元素 a=%s(0x%08X) b=%s(0x%08X)" + % (i, float(ua[i]), ua[i], float(ub[i]), ub[i])) + elif n_bad: + i = int(np.flatnonzero(bad)[0]) + first = "非有限值 a=%s b=%s @flat %d" % (fa.reshape(-1)[i], fb.reshape(-1)[i], i) + return n_bad, n_diff, maxabs, first + + +# ---------------------------------------------------------- 产物字节取样 +class Artifact: + """打包产物的 blob 张量字节入口。 + + 两个代表产物的类型表键形态不同：mini8 表键 = 张量名（带前缀 + .weight_bytes）； + 全量表键 = `layers.0...in_proj_q.weight`（不带前缀、不带 .weight_bytes，而 + key_prefix 又是 None ⇒ 与 index 张量名零交集）。所以既不能拿表键直接当张量名， + 也不能只剔一个前缀：先只剔尾缀归一，再要求“全等或唯一后缀匹配”， + 匹配不唯一 / 找不到都是打包回归，直接报错而不是猜。 + """ + + def __init__(self, path): + self.path = path + cfg = json.load(open(os.path.join(path, "config.json"))) + qc = cfg["quantization_config"] + table = qc["ggml_types"] + self.prefix = qc.get("key_prefix") or "" + idx = json.load(open(os.path.join(path, "model.safetensors.index.json")))["weight_map"] + self.shards = {} + for name in sorted(set(idx.values())): + p = os.path.join(path, name) + with open(p, "rb") as f: + n = struct.unpack(" 表键；要求全等或唯一后缀命中。""" + if tn in self.table_norm: + return tn, "exact" + cands = [k for k in self.table_norm if tn.endswith("." + k)] + if len(cands) == 1: + return cands[0], "suffix" + if len(cands) > 1: + raise RuntimeError("张量 %s 在表里后缀命中 %d 个键，歧义：%s" + % (tn, len(cands), sorted(cands)[:5])) + raise RuntimeError("张量 %s 在类型表里找不到对应条目" % tn) + + self.blobs = {} + self.match_form = collections.Counter() + self.matched_table_keys = set() + for tname in sorted(idx): + if not tname.endswith(".weight_bytes"): + continue + key, form = lookup(norm(tname)) + self.match_form[form] += 1 + self.matched_table_keys.add(key) + t = self.table_norm[key] + if t not in TYPES: + raise RuntimeError("%s 的 ggml type %d 不在路线 B 支持的 %s 里" + % (tname, t, list(TYPES))) + shard = os.path.join(self.path, idx[tname]) + base, hdr = self.shards[shard] + e = hdr[tname] + if e["dtype"] != "U8" or len(e["shape"]) != 2: + raise RuntimeError("%s 应为 U8 [rows, row_bytes]，实为 %s %s" + % (tname, e["dtype"], e["shape"])) + self.blobs[tname] = (t, shard, base + e["data_offsets"][0], + int(e["shape"][1]), int(e["shape"][0])) + # 表里说自己是 blob、但产物里没有对应 weight_bytes 张量的条目（应为 0） + self.orphan_table_keys = sorted(set(self.table_norm) - self.matched_table_keys) + + def type_names(self, t): + return sorted(n for n, v in self.blobs.items() if v[0] == t) + + def sample(self, t, want, rng): + """从该类型的真实张量里按整行取 block，返回 (n, type_size) uint8。""" + ts = TYPE_SIZE[t] + names = self.type_names(t) + handles = {} + out, touched = [], set() + per_name = max(1, int(np.ceil(want / max(1, len(names))))) + try: + for name in names: + _t, shard, base, row_bytes, nrows = self.blobs[name] + bpr = row_bytes // ts + if bpr * ts != row_bytes: + raise RuntimeError("%s 的 row_bytes=%d 不是 block_size %d 的整数倍" + % (name, row_bytes, ts)) + rows_needed = int(np.ceil(per_name / bpr)) + rows = np.sort(rng.choice(nrows, size=min(rows_needed, nrows), replace=False)) + if shard not in handles: + handles[shard] = open(shard, "rb") + fh = handles[shard] + buf = np.empty((rows.size, row_bytes), np.uint8) + for i, r in enumerate(rows): + fh.seek(base + int(r) * row_bytes) + buf[i] = np.frombuffer(fh.read(row_bytes), np.uint8) + flat = buf.reshape(-1, ts) + out.append(flat) + touched.add(name) + if sum(o.shape[0] for o in out) >= want: + break + finally: + for fh in handles.values(): + fh.close() + if not out: + return np.zeros((0, ts), np.uint8), touched + blocks = np.concatenate(out, axis=0)[:want] + return blocks, touched + + +def edge_blocks(t, rng, n_random=2048): + """手造边界 block：全 0、全 FF、次正规 d、scale 极值，再加有限值随机块。""" + ts = TYPE_SIZE[t] + rows = [np.zeros(ts, np.uint8), np.full(ts, 0xFF, np.uint8), + np.full(ts, 0x00, np.uint8), np.full(ts, 0x01, np.uint8)] + b = np.full(ts, 0xFF, np.uint8) + b[:] = 0 + if t == 8: # d = 最小次正规 half，qs 极值 + b[0:2] = [0x01, 0x00] + b[2:] = 0x80 # int8 -128 + rows.append(b.copy()) + b[2:] = 0x7F # int8 +127 + rows.append(b.copy()) + elif t in (12, 13): # d / dmin 次正规，6-bit scale/min 全 63 + b[0:2] = [0x01, 0x00] + b[2:4] = [0xFF, 0x00] # dmin = 1023 * 2^-24 + b[4:16] = 0xFF + rows.append(b.copy()) + b[0:2] = [0xFE, 0x7B] # d = 65534（最大有限 half） + b[2:4] = [0x00, 0x00] + rows.append(b.copy()) + else: # Q6_K：int8 scale = -128 / +127 + b[192:208] = 0x80 + b[208:210] = [0x01, 0x00] + rows.append(b.copy()) + b[192:208] = 0x7F + b[208:210] = [0xFE, 0x7B] + rows.append(b.copy()) + # 有限值随机块：随机字节 + 把 half 域换成非 inf/nan 的随机值 + for _ in range(n_random): + r = rng.integers(0, 256, ts, dtype=np.uint8) + for off in _half_offsets(t): + h = int(rng.integers(0, 0x7BFF + 1)) # exp != 0x1F + r[off], r[off + 1] = h & 0xFF, (h >> 8) & 0xFF + rows.append(r) + return np.stack(rows) + + +def _half_offsets(t): + if t == 8: + return (0,) + if t in (12, 13): + return (0, 2) + return (208,) + + +def half_sweep_blocks(t, rng): + """让每个 half 字段各自遍历全 65536 个位模式，其余字节随机。 + + 真实数据不一定会把次正规、负零、inf 这些 d 值送到解码路径上，全域扫描才能 + 钉住头里那份 half_to_float()（包括上面 numpy 参考刚犯过的符号位错误）。 + 返回 (blocks, 每个字段的扫描块起始行) 。 + """ + ts, offs = TYPE_SIZE[t], _half_offsets(t) + per = 65536 + blocks = rng.integers(0, 256, (per * len(offs), ts), dtype=np.uint8) + pats = np.arange(per, dtype=np.uint16) + for i, off in enumerate(offs): + sl = slice(i * per, (i + 1) * per) + # 其他 half 字段固定为 1.0，避免 NaN/inf 乘上本字段后把结果全糊成 NaN + for o2 in offs: + if o2 != off: + blocks[sl, o2] = 0x00 + blocks[sl, o2 + 1] = 0x3C + blocks[sl, off] = (pats & 0xFF).astype(np.uint8) + blocks[sl, off + 1] = (pats >> np.uint16(8)).astype(np.uint8) + return blocks, [(off, i * per) for i, off in enumerate(offs)] + + +# ------------------------------------------------------------ probe 编译/调用 +def build_probe(src, out, compiler, extra=()): + cmd = [compiler, "-O2", "-std=c++17", "-I", HEADER_DIR, src, "-o", out] + list(extra) + p = subprocess.run(cmd, capture_output=True, text=True) + if p.returncode != 0: + raise RuntimeError("编译失败：%s\n%s" % (" ".join(cmd), (p.stderr or p.stdout)[-4000:])) + return out + + +def run_probe(binary, t, blocks, workdir, tag): + ts, bs = TYPE_SIZE[t], BLOCK_SIZE[t] + inbin = os.path.join(workdir, "%s_t%d.in" % (tag, t)) + f32bin = os.path.join(workdir, "%s_t%d.f32" % (tag, t)) + bf16bin = os.path.join(workdir, "%s_t%d.bf16" % (tag, t)) + np.ascontiguousarray(blocks).tofile(inbin) + p = subprocess.run([binary, str(t), str(blocks.shape[0]), inbin, f32bin, bf16bin], + capture_output=True, text=True) + if p.returncode != 0: + raise RuntimeError("%s 失败（type=%d, rc=%d）：%s" + % (os.path.basename(binary), t, p.returncode, + (p.stderr or p.stdout).strip()[-2000:])) + f32 = np.fromfile(f32bin, np.float32).reshape(-1, bs) + bf16 = np.fromfile(bf16bin, np.uint16).reshape(-1, bs) + m = re.search(r"elems=(\d+)", p.stdout) + return f32, bf16, (int(m.group(1)) if m else -1) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model-path", default="/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8") + ap.add_argument("--blocks", type=int, default=20000, help="每种类型取多少真实 block") + ap.add_argument("--workdir", default="/home/liuxd/tmp_routeb/blocks31") + ap.add_argument("--cxx", default=os.environ.get("CXX", "g++")) + ap.add_argument("--nvcc", default=os.environ.get("CUDACXX", "nvcc")) + ap.add_argument("--no-cuda", action="store_true") + ap.add_argument("--seed", type=int, default=20260829) + args = ap.parse_args() + + rng = np.random.default_rng(args.seed) + os.makedirs(args.workdir, exist_ok=True) + print("产物：%s\n头文件：%s\n临时目录：%s\n每类型真实 block 目标：%d" + % (args.model_path, os.path.join(HEADER_DIR, "ggml_blocks.h"), args.workdir, + args.blocks)) + + print("\n[0] 参考实现自检") + check_half_decode() + + art = Artifact(args.model_path) + n_blob_total = len(art.blobs) + print("产物 blob 张量 %d 个（key_prefix=%r），按类型：%s" + % (n_blob_total, art.prefix, {t: len(art.type_names(t)) for t in TYPES})) + check("类型表 blob 条目与产物 weight_bytes 张量双向对平（表 %d / 张量 %d，孤儿 %d，" + "匹配形态 %s）" + % (art.n_table_blob, n_blob_total, len(art.orphan_table_keys), dict(art.match_form)), + art.n_table_blob == n_blob_total and not art.orphan_table_keys, + "孤儿键：%s" % art.orphan_table_keys[:5]) + + print("\n[1] 编译 probe driver") + host_bin = build_probe(PROBE_CPP, os.path.join(args.workdir, "blocks_probe_host"), args.cxx) + print(" host driver ok：%s" % host_bin) + dev_bin = None + if args.no_cuda: + skip("cuda driver 编译", "--no-cuda") + else: + try: + dev_bin = build_probe(PROBE_CU, os.path.join(args.workdir, "blocks_probe_cuda"), + args.nvcc, extra=["-x", "cu"]) + print(" cuda driver ok：%s" % dev_bin) + except Exception as e: + print(" ! %s" % e) + dev_bin = None + + print("\n[2] 逐类型对拍（真实 block + 边界 block）") + for t in TYPES: + name = Q(t).name + want = args.blocks + blocks, touched = art.sample(t, want, rng) + if not check("%s 采到 %d 个真实 block（目标 %d，覆盖 %d 个张量）" + % (name, blocks.shape[0], want, len(touched)), + blocks.shape[0] >= min(want, 100)): + continue + + ref = REF[t](np.ascontiguousarray(blocks)) + py = gguf_py_dequant(t, blocks) + n_bad, n_diff, maxabs, first = bitwise_diff(ref, py) + check("%s numpy 参考 vs gguf-py（%d block 逐位相同）" + % (name, blocks.shape[0]), + n_diff == 0 and n_bad == 0, + "差异 %d/%d 元素，非有限 %d，max|Δ|=%.3g，首个：%s" + % (n_diff, ref.size, n_bad, maxabs, first)) + + try: + h_f32, h_bf16, elems = run_probe(host_bin, t, blocks, args.workdir, "host") + except Exception as e: + check("%s host probe 运行" % name, False, str(e)) + continue + check("%s 头的 block_elems 与 GGML_QUANT_SIZES 一致（%d == %d）" + % (name, elems, BLOCK_SIZE[t]), elems == BLOCK_SIZE[t]) + n_bad, n_diff, maxabs, first = bitwise_diff(h_f32, ref) + check("%s 头(host) fp32 vs numpy 参考（%d 元素逐位相同）" + % (name, h_f32.size), n_diff == 0 and n_bad == 0, + "差异 %d，首个：%s" % (n_diff, first)) + + want_bf16 = float_to_bf16_bits(ref) + same_own = np.array_equal(h_bf16, want_bf16) + check("%s 头(host) bf16 vs numpy RNE 舍入" % name, same_own, + "首个差异 %s" % (np.flatnonzero(h_bf16 != want_bf16)[:5],)) + try: + import torch + tv = torch.from_numpy(np.ascontiguousarray(ref)).to(torch.bfloat16) \ + .view(torch.uint16).numpy() + check("%s 头(host) bf16 vs torch .to(bfloat16)" % name, np.array_equal(h_bf16, tv)) + except Exception as e: + skip("%s bf16 vs torch" % name, str(e).splitlines()[0][:80]) + + if dev_bin is not None: + try: + d_f32, d_bf16, _ = run_probe(dev_bin, t, blocks, args.workdir, "cuda") + except Exception as e: + check("%s cuda probe 运行" % name, False, str(e)) + else: + check("%s 头(cuda) fp32 vs 头(host) 逐位相同" % name, + np.array_equal(d_f32.view(np.uint32), h_f32.view(np.uint32))) + check("%s 头(cuda) bf16 vs 头(host) 逐位相同" % name, + np.array_equal(d_bf16, h_bf16)) + + eb = edge_blocks(t, rng) + eref = REF[t](np.ascontiguousarray(eb)) + epy = gguf_py_dequant(t, eb) + _, n_diff_e, maxabs_e, first_e = bitwise_diff(eref, epy) + n_bad_e = int((~np.isfinite(eref) | ~np.isfinite(epy)).sum()) + h_e, _, _ = run_probe(host_bin, t, eb, args.workdir, "host_edge") + _, n_diff_h, maxabs_h, first_h = bitwise_diff(h_e, eref) + check("%s 边界块（%d 个）numpy vs gguf-py 逐位相同" % (name, eb.shape[0]), + n_diff_e == 0, "差异 %d，非有限 %d，max|Δ|=%.3g，首个：%s" + % (n_diff_e, n_bad_e, maxabs_e, first_e)) + check("%s 边界块（%d 个）头(host) vs numpy 逐位相同" % (name, eb.shape[0]), + n_diff_h == 0, "差异 %d，max|Δ|=%.3g，首个：%s" % (n_diff_h, maxabs_h, first_h)) + + print("\n[3] half 字段全域扫描（每个字段 65536 个位模式）") + for t in TYPES: + name = Q(t).name + sb, _marks = half_sweep_blocks(t, rng) + with np.errstate(all="ignore"): # 扫描里故意喂 inf/nan half，告警与判据无关 + sref = REF[t](np.ascontiguousarray(sb)) + sh, _, _ = run_probe(host_bin, t, sb, args.workdir, "host_sweep") + n_bad, n_diff, maxabs, first = bitwise_diff(sh, sref) + check("%s 头(host) vs numpy 参考：half 全域扫描 %d block 逐位相同" + % (name, sb.shape[0]), n_diff == 0, + "差异 %d/%d 元素，非有限 %d（inf/nan 乘出的正常现象），max|Δ|=%.3g，首个：%s" + % (n_diff, sh.size, n_bad, maxabs, first)) + + print("\n[4] 不支持的类型必须被头拒绝") + inbin = os.path.join(args.workdir, "reject.in") + np.zeros(TYPE_SIZE[8], np.uint8).tofile(inbin) + p = subprocess.run([host_bin, "10", "1", inbin, + os.path.join(args.workdir, "reject.f32"), + os.path.join(args.workdir, "reject.bf16")], + capture_output=True, text=True) + check("头对 ggml type 10（TQ1_0，非本头范围）返回拒绝", + p.returncode == 3 and "no decoder" in (p.stderr + p.stdout), + "rc=%d stderr=%s" % (p.returncode, (p.stderr or p.stdout).strip()[-200:])) + + print("\n== 结果：%d PASS / %d FAIL / %d SKIP ==" % (_PASS, _FAIL, _SKIP)) + print("临时目录：%s" % args.workdir) + return 0 if _FAIL == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_routeb_compare.py b/scripts/gguf_routeb_compare.py new file mode 100755 index 000000000..4d8f9d96d --- /dev/null +++ b/scripts/gguf_routeb_compare.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Compare deterministic llama.cpp and InfiniLM token results.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--llama", required=True) + ap.add_argument("--infinilm", required=True) + ap.add_argument("--out", required=True) + ap.add_argument( + "--case-ids", + help="Optional comma-separated case IDs for focused comparisons", + ) + args = ap.parse_args() + + with open(args.llama, encoding="utf-8") as f: + llama = json.load(f) + with open(args.infinilm, encoding="utf-8") as f: + infini = json.load(f) + lmap = {x["id"]: x for x in llama["cases"]} + imap = {x["id"]: x for x in infini["cases"]} + if args.case_ids: + case_ids = [x.strip() for x in args.case_ids.split(",") if x.strip()] + missing = [x for x in case_ids if x not in lmap or x not in imap] + if missing: + raise ValueError("requested case IDs missing from one side: %s" % missing) + lmap = {x: lmap[x] for x in case_ids} + imap = {x: imap[x] for x in case_ids} + elif set(lmap) != set(imap): + raise ValueError("case sets differ: llama-only=%s infini-only=%s" % ( + sorted(set(lmap) - set(imap)), sorted(set(imap) - set(lmap)))) + + cases = [] + exact = 0 + matched = total = 0 + for case_id in lmap: + left = lmap[case_id] + right = imap[case_id] + if left["input_ids"] != right["input_ids"]: + raise ValueError("input ids differ for %s" % case_id) + lt = left["runs"][0]["tokens"] + rt = right["runs"][0]["tokens"] + first_difference = next((i for i, (a, b) in enumerate(zip(lt, rt)) if a != b), None) + if first_difference is None and len(lt) != len(rt): + first_difference = min(len(lt), len(rt)) + is_exact = lt == rt + exact += int(is_exact) + same = sum(a == b for a, b in zip(lt, rt)) + matched += same + total += max(len(lt), len(rt)) + cases.append({ + "id": case_id, + "exact_sequence_match": is_exact, + "matched_tokens": same, + "total_tokens": max(len(lt), len(rt)), + "first_difference": first_difference, + "llama_tokens": lt, + "infinilm_tokens": rt, + "llama_first_top_logprobs": left["runs"][0].get( + "first_token_top_logprobs", []), + }) + print("%-10s exact=%s first_diff=%s llama=%s infini=%s" % ( + case_id, is_exact, first_difference, lt, rt)) + + result = { + "cases": cases, + "n_cases": len(cases), + "exact_cases": exact, + "prompt_exact_rate": exact / len(cases) if cases else 0.0, + "matched_tokens": matched, + "total_tokens": total, + "token_match_rate": matched / total if total else 0.0, + "all_exact": exact == len(cases), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + print("RESULT exact=%d/%d token_match=%d/%d all_exact=%s" % ( + exact, len(cases), matched, total, result["all_exact"])) + return 0 if result["all_exact"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_routeb_env.sh b/scripts/gguf_routeb_env.sh new file mode 100644 index 000000000..bf4f6577b --- /dev/null +++ b/scripts/gguf_routeb_env.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# InfiniLM Route B (native GGUF quantization) development environment. +# Source this file after setting CUDA_HOME and optional CUTLASS_ROOT/CUDNN_ROOT. +ROUTEB_SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ROUTEB_INFINILM_DIR="$(cd -- "${ROUTEB_SCRIPT_DIR}/.." && pwd)" +: "${INFINICORE_DIR:=$(cd -- "${ROUTEB_INFINILM_DIR}/../InfiniCore" && pwd)}" +: "${INFINI_ROOT:=${HOME}/.infini}" + +if [[ -n "${CUDA_HOME:-}" ]]; then + export CUDACXX="${CUDACXX:-${CUDA_HOME}/bin/nvcc}" + export PATH="${CUDA_HOME}/bin:${PATH}" + ROUTEB_CUDA_LIB="${CUDA_HOME}/lib64:" +else + ROUTEB_CUDA_LIB="" +fi + +export INFINICORE_DIR INFINI_ROOT +export PYTHONPATH="${INFINICORE_DIR}/python:${ROUTEB_INFINILM_DIR}/python:${PYTHONPATH:-}" +export LD_LIBRARY_PATH="${INFINICORE_DIR}/python/infinicore/lib:${ROUTEB_INFINILM_DIR}/python/infinilm/lib:${INFINI_ROOT}/lib:${ROUTEB_CUDA_LIB}${LD_LIBRARY_PATH:-}" +export HF_HUB_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 diff --git a/scripts/gguf_routeb_first_diff.py b/scripts/gguf_routeb_first_diff.py new file mode 100755 index 000000000..b13084e4a --- /dev/null +++ b/scripts/gguf_routeb_first_diff.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Inspect llama.cpp and InfiniLM logits at the first token divergence.""" + +from __future__ import annotations + +import argparse +import ctypes +import json +import math +import os +import sys +import time +import urllib.error +import urllib.request + + +def post_json(url: str, body: dict, timeout: int = 180) -> dict: + request = urllib.request.Request( + url, + data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.load(response) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", "replace") + raise RuntimeError("HTTP %d: %s" % (exc.code, detail[:2000])) from exc + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--inputs", required=True) + ap.add_argument("--compare", required=True) + ap.add_argument("--case-id", required=True) + ap.add_argument("--model-path", required=True) + ap.add_argument("--server", default="http://127.0.0.1:18080") + ap.add_argument("--top-k", type=int, default=100) + ap.add_argument("--num-blocks", type=int, default=64) + ap.add_argument("--block-size", type=int, default=256) + ap.add_argument("--out", required=True) + args = ap.parse_args() + + import numpy as np + import infinicore + from infinilm.cache import PagedKVCacheConfig + from infinilm.distributed import DistConfig + from infinilm.infer_engine import InferEngine + from infinilm.lib import _infinilm + from infinilm.modeling_utils import load_model_state_dict_by_file + + with open(args.inputs, encoding="utf-8") as f: + inputs = {x["id"]: x for x in json.load(f)["cases"]} + with open(args.compare, encoding="utf-8") as f: + compared = {x["id"]: x for x in json.load(f)["cases"]} + item = compared[args.case_id] + first_diff = item["first_difference"] + if first_diff is None: + raise ValueError("case %s has no divergence" % args.case_id) + common_generated = item["llama_tokens"][:first_diff] + assert common_generated == item["infinilm_tokens"][:first_diff] + prefix = [int(x) for x in inputs[args.case_id]["input_ids"] + common_generated] + + llama_body = { + "prompt": prefix, + "n_predict": 1, + "temperature": 0.0, + "top_k": 1, + "top_p": 1.0, + "min_p": 0.0, + "typical_p": 1.0, + "repeat_penalty": 1.0, + "repeat_last_n": 0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "seed": 1, + "ignore_eos": True, + "cache_prompt": False, + "return_tokens": True, + "n_probs": args.top_k, + "stream": False, + "samplers": ["top_k", "temperature"], + } + llama_response = post_json( + args.server.rstrip("/") + "/completion", llama_body) + llama_probs = llama_response["completion_probabilities"][0]["top_logprobs"] + + load_started = time.time() + engine = InferEngine( + model_path=args.model_path, + device=infinicore.device("cuda:0"), + distributed_config=DistConfig(1), + cache_config=PagedKVCacheConfig( + args.num_blocks, args.block_size, max_batch_size=1), + attention_backend="paged-attn", + ) + load_model_state_dict_by_file(engine, args.model_path, dtype=engine.dtype) + load_s = time.time() - load_started + + length = len(prefix) + positions = list(range(length)) + if engine.position_id_axes > 1: + positions = [positions for _ in range(engine.position_id_axes)] + tensors = { + "input_ids": infinicore.from_list([prefix], dtype=infinicore.int64).view([1, length]), + "position_ids": infinicore.from_list(positions, dtype=infinicore.int64), + "past_kv_lengths": infinicore.from_list([0], dtype=infinicore.int32), + "total_kv_lengths": infinicore.from_list([length], dtype=infinicore.int32), + "input_offsets": infinicore.from_list([0, length], dtype=infinicore.int32), + "cu_seqlens": infinicore.from_list([0, length], dtype=infinicore.int32), + "block_tables": infinicore.from_list([[0]], dtype=infinicore.int32), + "slot_mapping": infinicore.from_list(list(range(length)), dtype=infinicore.int64), + "mamba_init_state_indices": infinicore.from_list([0], dtype=infinicore.int32), + "mamba_final_state_indices": infinicore.from_list([1], dtype=infinicore.int32), + } + cpp_input = engine._build_input( + tensors["input_ids"], + position_ids=tensors["position_ids"], + past_kv_lengths=tensors["past_kv_lengths"], + total_kv_lengths=tensors["total_kv_lengths"], + input_offsets=tensors["input_offsets"], + cu_seqlens=tensors["cu_seqlens"], + block_tables=tensors["block_tables"], + slot_mapping=tensors["slot_mapping"], + mamba_init_state_indices=tensors["mamba_init_state_indices"], + mamba_final_state_indices=tensors["mamba_final_state_indices"], + sample_all_positions=False, + temperature=0.0, + top_k=1, + top_p=1.0, + ) + output = _infinilm.InferEngine.forward(engine, cpp_input) + raw_logits = infinicore.Tensor(output.logits) + logits_shape = list(raw_logits.shape) + cpu_logits = raw_logits.to(infinicore.device("cpu", 0)) + if cpu_logits.dtype != infinicore.bfloat16: + raise TypeError("expected BF16 logits, got %s" % cpu_logits.dtype) + bits_type = ctypes.c_uint16 * cpu_logits.numel() + bits = np.ctypeslib.as_array(bits_type.from_address(cpu_logits.data_ptr())).copy() + all_logits = (bits.astype(np.uint32) << 16).view(np.float32).reshape(logits_shape) + logits = all_logits.reshape(-1, logits_shape[-1])[-1] + order = np.argpartition(logits, -args.top_k)[-args.top_k:] + order = order[np.argsort(logits[order])[::-1]] + max_logit = float(logits[order[0]]) + infini_top = [{"id": int(i), "logit": float(logits[i]), + "delta_from_top": float(logits[i] - max_logit)} for i in order] + + llama_map = {int(x["id"]): float(x["logprob"]) for x in llama_probs} + infini_map = {int(x["id"]): float(x["delta_from_top"]) for x in infini_top} + candidate_ids = sorted(set(llama_map) | set(infini_map)) + candidate_table = [{ + "id": token_id, + "llama_logprob": llama_map.get(token_id), + "infini_delta_from_top": infini_map.get(token_id), + } for token_id in candidate_ids] + + result = { + "case_id": args.case_id, + "first_difference": first_diff, + "base_input_ids": inputs[args.case_id]["input_ids"], + "common_generated_prefix": common_generated, + "diagnostic_prefix": prefix, + "llama_selected": int(llama_response["tokens"][0]), + "infinilm_selected": int(order[0]), + "llama_top_logprobs": llama_probs, + "infinilm_top_logits": infini_top, + "candidate_table": candidate_table, + "infinilm_logits_shape": logits_shape, + "infinilm_logits_finite": bool(np.isfinite(logits).all()), + "infinilm_load_s": round(load_s, 4), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + print("CASE=%s diff=%d prefix_len=%d llama=%d infini=%d" % ( + args.case_id, first_diff, len(prefix), result["llama_selected"], + result["infinilm_selected"])) + print("LLAMA_TOP5 %s" % [(x["id"], round(x["logprob"], 6)) + for x in llama_probs[:5]]) + print("INFINI_TOP5 %s" % [(x["id"], round(x["delta_from_top"], 6)) + for x in infini_top[:5]]) + print("FINITE=%s SHAPE=%s LOAD=%.3fs" % ( + result["infinilm_logits_finite"], result["infinilm_logits_shape"], load_s)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_routeb_first_diff_batch.py b/scripts/gguf_routeb_first_diff_batch.py new file mode 100644 index 000000000..c9f6a7d66 --- /dev/null +++ b/scripts/gguf_routeb_first_diff_batch.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Inspect first-divergence logits for every non-exact Route-B case.""" + +from __future__ import annotations + +import argparse +import ctypes +import json +import os +import time +import urllib.request + + +def post_json(url: str, body: dict, timeout: int = 180) -> dict: + req = urllib.request.Request( + url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, method="POST") + with urllib.request.urlopen(req, timeout=timeout) as response: + return json.load(response) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--inputs", required=True) + ap.add_argument("--compare", required=True) + ap.add_argument("--model-path", required=True) + ap.add_argument("--server", default="http://127.0.0.1:18080") + ap.add_argument("--top-k", type=int, default=100) + ap.add_argument("--num-blocks", type=int, default=64) + ap.add_argument("--block-size", type=int, default=256) + ap.add_argument("--out", required=True) + args = ap.parse_args() + + import numpy as np + import infinicore + from infinilm.cache import PagedKVCacheConfig + from infinilm.distributed import DistConfig + from infinilm.infer_engine import InferEngine + from infinilm.lib import _infinilm + from infinilm.modeling_utils import load_model_state_dict_by_file + + with open(args.inputs, encoding="utf-8") as f: + inputs = {x["id"]: x for x in json.load(f)["cases"]} + with open(args.compare, encoding="utf-8") as f: + compared = json.load(f)["cases"] + divergent = [x for x in compared if x["first_difference"] is not None] + + started = time.time() + engine = InferEngine( + model_path=args.model_path, device=infinicore.device("cuda:0"), + distributed_config=DistConfig(1), + cache_config=PagedKVCacheConfig( + args.num_blocks, args.block_size, max_batch_size=1), + attention_backend="paged-attn") + load_model_state_dict_by_file(engine, args.model_path, dtype=engine.dtype) + load_s = time.time() - started + + results = [] + for item in divergent: + case_id = item["id"] + first_diff = item["first_difference"] + common = item["llama_tokens"][:first_diff] + assert common == item["infinilm_tokens"][:first_diff] + prefix = [int(x) for x in inputs[case_id]["input_ids"] + common] + body = { + "prompt": prefix, "n_predict": 1, "temperature": 0.0, + "top_k": 1, "top_p": 1.0, "min_p": 0.0, + "typical_p": 1.0, "repeat_penalty": 1.0, + "repeat_last_n": 0, "presence_penalty": 0.0, + "frequency_penalty": 0.0, "seed": 1, "ignore_eos": True, + "cache_prompt": False, "return_tokens": True, + "n_probs": args.top_k, "stream": False, + "samplers": ["top_k", "temperature"], + } + llama = post_json(args.server.rstrip("/") + "/completion", body) + llama_probs = llama["completion_probabilities"][0]["top_logprobs"] + + length = len(prefix) + positions = list(range(length)) + if engine.position_id_axes > 1: + positions = [positions for _ in range(engine.position_id_axes)] + tensors = { + "input_ids": infinicore.from_list([prefix], dtype=infinicore.int64).view([1, length]), + "position_ids": infinicore.from_list(positions, dtype=infinicore.int64), + "past_kv_lengths": infinicore.from_list([0], dtype=infinicore.int32), + "total_kv_lengths": infinicore.from_list([length], dtype=infinicore.int32), + "input_offsets": infinicore.from_list([0, length], dtype=infinicore.int32), + "cu_seqlens": infinicore.from_list([0, length], dtype=infinicore.int32), + "block_tables": infinicore.from_list([[0]], dtype=infinicore.int32), + "slot_mapping": infinicore.from_list(list(range(length)), dtype=infinicore.int64), + "mamba_init_state_indices": infinicore.from_list([0], dtype=infinicore.int32), + "mamba_final_state_indices": infinicore.from_list([1], dtype=infinicore.int32), + } + cpp_input = engine._build_input( + tensors["input_ids"], position_ids=tensors["position_ids"], + past_kv_lengths=tensors["past_kv_lengths"], + total_kv_lengths=tensors["total_kv_lengths"], + input_offsets=tensors["input_offsets"], cu_seqlens=tensors["cu_seqlens"], + block_tables=tensors["block_tables"], slot_mapping=tensors["slot_mapping"], + mamba_init_state_indices=tensors["mamba_init_state_indices"], + mamba_final_state_indices=tensors["mamba_final_state_indices"], + sample_all_positions=False, temperature=0.0, top_k=1, top_p=1.0) + output = _infinilm.InferEngine.forward(engine, cpp_input) + raw = infinicore.Tensor(output.logits) + shape = list(raw.shape) + cpu = raw.to(infinicore.device("cpu", 0)) + if cpu.dtype != infinicore.bfloat16: + raise TypeError("expected BF16 logits, got %s" % cpu.dtype) + bits_type = ctypes.c_uint16 * cpu.numel() + bits = np.ctypeslib.as_array(bits_type.from_address(cpu.data_ptr())).copy() + logits = (bits.astype(np.uint32) << 16).view(np.float32).reshape(shape) + logits = logits.reshape(-1, shape[-1])[-1] + order = np.argpartition(logits, -args.top_k)[-args.top_k:] + order = order[np.argsort(logits[order], kind="stable")[::-1]] + top_logit = float(logits[order[0]]) + infini_top = [{"id": int(i), "logit": float(logits[i]), + "delta_from_top": float(logits[i] - top_logit)} for i in order] + llama_map = {int(x["id"]): float(x["logprob"]) for x in llama_probs} + infini_map = {x["id"]: x["delta_from_top"] for x in infini_top} + llama_selected = int(llama["tokens"][0]) + infini_selected = int(order[0]) + candidate_ids = sorted(set(llama_map) | set(infini_map)) + candidate_table = [{ + "id": token_id, "llama_logprob": llama_map.get(token_id), + "infini_delta_from_top": infini_map.get(token_id), + "infini_logit": float(logits[token_id]), + } for token_id in candidate_ids] + selected_logits = { + "llama_token_infini_logit": float(logits[llama_selected]), + "infini_token_infini_logit": float(logits[infini_selected]), + "infini_margin_selected_minus_llama": + float(logits[infini_selected] - logits[llama_selected]), + "llama_margin_selected_minus_infini": + float(llama_map[llama_selected] - llama_map.get(infini_selected, float("nan"))), + } + result = { + "case_id": case_id, "first_difference": first_diff, + "prefix_length": len(prefix), "llama_selected": llama_selected, + "infinilm_selected": infini_selected, + "llama_top_logprobs": llama_probs, "infinilm_top_logits": infini_top, + "selected_pair": selected_logits, "candidate_table": candidate_table, + "infinilm_logits_shape": shape, + "infinilm_logits_finite": bool(np.isfinite(logits).all()), + } + results.append(result) + print("%-10s diff=%2d llama=%6d infini=%6d llama_margin=%+.6f infini_margin=%+.6f" % ( + case_id, first_diff, llama_selected, infini_selected, + selected_logits["llama_margin_selected_minus_infini"], + selected_logits["infini_margin_selected_minus_llama"])) + + report = {"load_s": round(load_s, 4), "case_count": len(results), "cases": results} + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w", encoding="utf-8") as f: + json.dump(report, f, ensure_ascii=False, indent=2) + print("RESULT cases=%d finite=%s load=%.3fs" % ( + len(results), all(x["infinilm_logits_finite"] for x in results), load_s)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gguf_routeb_gemv_check.py b/scripts/gguf_routeb_gemv_check.py new file mode 100644 index 000000000..dcb3457c4 --- /dev/null +++ b/scripts/gguf_routeb_gemv_check.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +""" +InfiniLM 路线 B —— 阶段 3.2 + 3.3 验收：linear_gguf 两条 NVIDIA 路径的数值正确性 + +被测对象（两条路径都由算子本体所在的头文件提供，probe 直接 include）： + * `InfiniCore/src/infiniop/ops/linear_gguf/nvidia/linear_gguf_gemv.cuh` + —— M <= kMaxDecodeM 的 decode 路径（一 warp 一行、寄存器内解码 + fp32 累加）； + * `InfiniCore/src/infiniop/ops/linear_gguf/nvidia/linear_gguf_dequant.cuh` + —— M > kMaxDecodeM 的 prefill 路径（64 行权重解码到 BF16 scratch + cublasGemmEx）。 +`scripts/gguf_routeb_gemv_probe.cu` 里的路由谓词与算子 `calculate` 用的是同一个 +`kMaxDecodeM`，所以每条用例走的真是发布路径上那条 kernel；probe 还会在 stdout 报 +`path=gemv|prefill`，脚本据此**断言路由本身**（见下面的“路径”判据）。 + +判据不新造： + * 主判据 = 方案 §1.2 第 2 条「GEMM 输出与稠密 BF16 权重 @ x 的 cos_sim > 0.999」。 + 这里的“稠密权重”用的是 3.1 已证与 gguf-py / 头逐位相同的 numpy 参考（`REF`）， + 所以这条判据同时就把「解码正确」与「GEMV / prefill 正确」两件事串在了一起。 + prefill 路径把权重先舍到 BF16 再乘，与这条基准口径一致。 + * 权重字节全部取自真实打包产物的 `*.weight_bytes` **整行**（不是随机 block）， + 因为 kernel 依赖“一行 = 整数个 block”这个契约，随机 block 拼不出来。每种类型 + 取首/中/尾三个张量（跨层），避开“两份产物挑到同一层同一张量”的假独立性。 + * 行数默认 200（不是 64 的整数倍），这样 prefill 的 tile 循环会走到 + “最后一片不满”的分支。 + * 附带两条拒绝（必须报错、不许静默出结果）：未知 type、K 不是 block 元素数整数倍。 + （原来那条「M=9 超过 decode 上限必须被拒」在 3.3 之后不再成立，M=9 现在既是 + prefill 的下边界、又是一条正例，见 --ms 默认值。） + +累加顺序与 numpy 不同（gemv：块内顺序求和 -> 沿 block 累加 -> warp shuffle 归约； +prefill：cublas 分块），所以这里**不要求逐位相同**，而是把逐位相同率当作观测量报出来， +cos_sim 当判据。 + +用法： + /usr/bin/python3 scripts/gguf_routeb_gemv_check.py \ + [--model-path /home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8] \ + [--rows 200] [--ms 1,8,9,16,32,64,256,1024] [--keep] +退出码 0 = 全部 PASS。 +""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys + +import numpy as np + +_HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, _HERE) + +import gguf_routeb_blocks_ref as bref # noqa: E402 +from gguf_routeb_blocks_ref import (Artifact, BLOCK_SIZE, REF, # noqa: E402 + TYPE_SIZE, TYPES, check, skip) + +GEMV_DIR = os.path.join(bref.HEADER_DIR, "nvidia") +PROBE_SRC = os.path.join(_HERE, "gguf_routeb_gemv_probe.cu") +MAX_M = 8 # kMaxDecodeM：M <= 8 走 gemv，M > 8 走 prefill +PREFILL_MS = "9,16,32,64,256,1024" # 9 = prefill 下边界（§1.2 第 3 条含 16/32/64/256/1024） +PATH_RE = re.compile(r"path=(\w+)") +T_NAME = {8: "Q8_0", 12: "Q4_K", 13: "Q5_K", 14: "Q6_K"} + + +def reset_counters(): + bref._PASS = bref._FAIL = bref._SKIP = 0 + + +# --------------------------------------------------------------- bf16 位模式 +def bf16_to_f32(bits): + return (np.asarray(bits, np.uint16).astype(np.uint32) + << np.uint32(16)).view(np.float32) + + +def f32_to_bf16(x): + return bref.float_to_bf16_bits(x) + + +def cos_sim(a, b): + a = np.asarray(a, np.float64).reshape(-1) + b = np.asarray(b, np.float64).reshape(-1) + na, nb = np.linalg.norm(a), np.linalg.norm(b) + if na == 0.0 or nb == 0.0: + return float(np.array_equal(a, b)) + return float(a @ b / (na * nb)) + + +# ------------------------------------------------------------- 真实权重整行 +def pick_rows(art, t, want_rows, rng, which=0): + """从类型 t 的某个真实张量里取连续若干行字节（which 指定取哪个）。""" + names = art.type_names(t) + if not names: + return None + ts = TYPE_SIZE[t] + name = names[min(which, len(names) - 1)] + _t, shard, base, row_bytes, nrows = art.blobs[name] + blocks_per_row = row_bytes // ts + if blocks_per_row * ts != row_bytes or blocks_per_row < 1: + raise RuntimeError("%s 的 row_bytes=%d 不是 block_size %d 的整数倍" + % (name, row_bytes, ts)) + rows = min(want_rows, nrows) + r0 = int(rng.integers(0, nrows - rows + 1)) + with open(shard, "rb") as fh: + fh.seek(base + r0 * row_bytes) + buf = np.frombuffer(fh.read(rows * row_bytes), np.uint8) + W = buf.reshape(rows, row_bytes).copy() + return name, W, blocks_per_row * BLOCK_SIZE[t], r0 + + +def dense_weights(t, W, rows, K): + """numpy 参考反量化：W[rows, row_bytes] -> float32 [rows, K]。""" + ts, bs = TYPE_SIZE[t], BLOCK_SIZE[t] + blocks = W.reshape(-1, ts) + dec = REF[t](blocks) # (n_blocks, bs) float32，3.1 已证逐位正确 + return dec.reshape(rows, K) + + +# ------------------------------------------------------------------ 驱动调用 +# probe 一个可执行文件覆盖两条路径（名字沿用 3.2），具体走哪条由它内部的 +# m > kMaxDecodeM 谓词决定，并由 stdout 的 path= 字段报回来。 +def run_gemv(binary, t, A_bf16, W, K, workdir, tag): + m, _ = A_bf16.shape + n, row_bytes = W.shape + abin = os.path.join(workdir, "%s_m%d_t%d.a" % (tag, m, t)) + wbin = os.path.join(workdir, "%s_m%d_t%d.w" % (tag, m, t)) + cbin = os.path.join(workdir, "%s_m%d_t%d.c" % (tag, m, t)) + A_bf16.astype(np.uint16).tofile(abin) + np.ascontiguousarray(W).tofile(wbin) + cmd = [binary, str(t), str(m), str(n), str(K), str(row_bytes), abin, wbin, cbin] + p = subprocess.run(cmd, capture_output=True, text=True) + return p, cbin + + +def check_type(binary, art, t, rows, Ms, rng, workdir, which_list): + """对同一类型的多个张量（刻意跨层）各跑一轮。 + + 只取排序后第一个张量会在两份产物上挑到同一个张量（字节完全相同），那 + 时候选产物就只是“两种键形态”而不是两份独立权重证据，所以这里固定取 + 首/中/尾三个（不足则去重）。注意 mini8 是完整模型的前若干层，layer 0/1 + 的张量在两份产物里字节相同，取样点落在这些层时仍然撞——这是数据的性质， + 不是取样能修的（Q4_K 尤其：全模型只有 4 个张量且都在 layer 1）。 + """ + names = art.type_names(t) + picks = sorted({min(w, len(names) - 1) for w in which_list}) + done = set() + for wi in picks: + picked = pick_rows(art, t, rows, rng, wi) + if picked is None: + skip("%d 张量 #%d" % (t, wi), "产物不含该类型") + continue + name, W, K, r0 = picked + if name in done: + continue + done.add(name) + check_type_one(binary, art, t, name, W, K, r0, Ms, rng, workdir) + + +def check_type_one(binary, art, t, name, W, K, r0, Ms, rng, workdir): + n = W.shape[0] + Wf32 = dense_weights(t, W, n, K) + # 稠密 BF16 权重 @ x 这条基准：先把反量化结果舍到 bf16 再算，同 §1.2 第 2 条口径 + Wdense = bf16_to_f32(f32_to_bf16(Wf32)) + print(" %s：%s（起始行 %d），%d 行 x K=%d，row_bytes=%d" + % (T_NAME.get(t, t), name, r0, n, K, W.shape[1])) + for m in Ms: + A = (rng.standard_normal((m, K)) * 0.5).astype(np.float32) + Abits = f32_to_bf16(A) + Af = bf16_to_f32(Abits) # kernel 看到的就是这份值 + p, cbin = run_gemv(binary, t, Abits, W, K, workdir, "gemv") + if not check("%s M=%d：kernel 退出码 0" % (T_NAME.get(t, t), m), p.returncode == 0, + "rc=%d %s" % (p.returncode, (p.stderr or p.stdout).strip()[-600:])): + continue + # 路由判据：probe 报的 path 必须等于算子在该 M 上会选的路径。数值过了但 + # 路走错了同样不可接受（那意味着门测的不是发布路径）。 + want_path = "gemv" if m <= MAX_M else "prefill" + pm = PATH_RE.search(p.stdout or "") + got_path = pm.group(1) if pm else "?" + check("%s M=%d：走 %s 路径（与算子 calculate 的谓词一致）" % (T_NAME.get(t, t), m, want_path), + got_path == want_path, "probe 报 path=%s" % got_path) + got = bf16_to_f32(np.fromfile(cbin, np.uint16).reshape(m, n)) + assert got.shape == (m, n) + ref = (Af @ Wdense.T).astype(np.float32) # §1.2 第 2 条口径的基准 + ref_exact = (Af @ Wf32.T).astype(np.float32) # 不先把权重舍到 bf16 + c = cos_sim(got, ref) + check("%s M=%d：cos_sim(kernel, 稠密 BF16 权重 @ x) > 0.999" % (T_NAME.get(t, t), m), + c > 0.999, "cos_sim=%.8f" % c) + # 观测量（不作判据）：bf16 位相同率、最大绝对/相对偏差、vs 未舍入基准的 cos_sim + same = float(np.mean(f32_to_bf16(got) == f32_to_bf16(ref))) + dg = got.astype(np.float64) - ref.astype(np.float64) + absd = float(np.max(np.abs(dg))) + # 相对偏差只在“有意义的元素”上算（|ref| >= 最大幅值的 1%），否则会被近零 + # 元素除出几十倍的假大数，那种数字没有判读价值。 + sig = np.abs(ref.astype(np.float64)) >= 0.01 * float(np.max(np.abs(ref))) + rel = float(np.max(np.abs(dg[sig]) / np.abs(ref.astype(np.float64)[sig]))) if sig.any() else 0.0 + print(" 观测：cos_sim(kernel, 稠密 BF16 权重)=%.10f" + " cos_sim(kernel, 未舍入基准)=%.10f bf16 逐位相同率=%.4f" + " max|Δ|=%.3e max 相对偏差(|ref|≥最大幅值1%% 的子集)=%.3e %s" + % (c, cos_sim(got, ref_exact), same, absd, rel, + p.stdout.strip().split("ok")[-1].strip())) + + +def check_rejections(binary, art, workdir): + """必须报错的输入：不许静默出结果。 + + 3.3 之前这里还有一条「M=9 超过 decode 上限被拒」，现在 prefill 接管了 M>8， + 该用例已反转成 --ms 里的正例（prefill 下边界）。 + """ + name_ok = None + for t in TYPES: + picked = pick_rows(art, t, 4, np.random.default_rng(7)) + if picked: + name_ok, W, K = t, picked[1], picked[2] + break + A = np.zeros((1, K), np.float32) + Abits = f32_to_bf16(A) + + p, _ = run_gemv(binary, 10, Abits, W, K, workdir, "rej") + check("未知 ggml type 10 被拒（rc=3，不启动 kernel）", p.returncode == 3, + "rc=%d %s" % (p.returncode, p.stderr.strip()[-300:])) + + bad_k = K + (BLOCK_SIZE[name_ok] - 1) # 不再是整数个 block + A_bad = f32_to_bf16(np.zeros((1, bad_k), np.float32)) + p, _ = run_gemv(binary, name_ok, A_bad, W, bad_k, workdir, "rej") + check("K 不是 block 元素数整数倍被拒（rc=3）", p.returncode == 3, + "rc=%d %s" % (p.returncode, p.stderr.strip()[-300:])) + + # 同一条约束在 prefill 路径上也必须成立（两条路径各自有谓词，不能只查 gemv） + A_bad_p = f32_to_bf16(np.zeros((MAX_M + 1, bad_k), np.float32)) + p, _ = run_gemv(binary, name_ok, A_bad_p, W, bad_k, workdir, "rej") + check("prefill 路径同样拒掉不整除的 K（rc=3）", p.returncode == 3, + "rc=%d %s" % (p.returncode, p.stderr.strip()[-300:])) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model-path", default="/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8") + ap.add_argument("--rows", type=int, default=200, help="每个张量取多少行权重（不是 64 的整数倍才能盖住 tile 余数）") + ap.add_argument("--ms", default="1,8," + PREFILL_MS, + help="逗号分隔；<=8 走 gemv，>8 走 prefill（两条路径同一份门）") + ap.add_argument("--workdir", default="/home/liuxd/tmp_routeb/gemv32") + ap.add_argument("--nvcc", default=os.environ.get("CUDACXX", "nvcc")) + ap.add_argument("--skip-build", action="store_true") + ap.add_argument("--no-reject", action="store_true") + ap.add_argument("--seed", type=int, default=20260829) + args = ap.parse_args() + + reset_counters() + rng = np.random.default_rng(args.seed) + os.makedirs(args.workdir, exist_ok=True) + Ms = [int(x) for x in args.ms.split(",") if x.strip()] + print("产物：%s\n被测：\n %s\n %s\n %s\n临时目录：%s\n每种类型权重行数：%d，M 取 %s" + % (args.model_path, + os.path.join(GEMV_DIR, "linear_gguf_gemv.cuh"), + os.path.join(GEMV_DIR, "linear_gguf_dequant.cuh"), + PROBE_SRC, args.workdir, args.rows, Ms)) + + binary = os.path.join(args.workdir, "gemv_probe") + print("\n[1] 编译两条路径的驱动（prefill 需要 -lcublas）") + if args.skip_build: + skip("编译", "--skip-build") + else: + try: + bref.build_probe(PROBE_SRC, binary, args.nvcc, + extra=["-I", GEMV_DIR, "-lcublas"]) + check("nvcc 编译 %s 通过（含两个 kernel 头 + cublas）" + % os.path.basename(PROBE_SRC), True) + except Exception as exc: # noqa: BLE001 + check("nvcc 编译 %s 通过" % os.path.basename(PROBE_SRC), False, str(exc)[-2000:]) + return 1 + + art = Artifact(args.model_path) + print("\n[2] 真实权重对 numpy 稠密基准（gemv + prefill，判据：cos_sim > 0.999）") + print("产物 blob 张量 %d 个（key_prefix=%r），按类型：%s" + % (len(art.blobs), art.prefix, {t: len(art.type_names(t)) for t in TYPES})) + for t in TYPES: + n_avail = len(art.type_names(t)) + # 张量本来就少（Q4_K 全模型只有 4 个，且都在 layer 1）时全取，否则首/中/尾 + which = list(range(n_avail)) if n_avail <= 6 else [0, n_avail // 2, n_avail - 1] + which = which or [0] + check_type(binary, art, t, args.rows, Ms, rng, args.workdir, which) + + if args.no_reject: + skip("非法输入用例", "--no-reject") + else: + print("\n[3] 非法输入必须被拒（不许静默出结果）") + check_rejections(binary, art, args.workdir) + + print("\n== 结果：%d PASS / %d FAIL / %d SKIP ==" % (bref._PASS, bref._FAIL, bref._SKIP)) + print("临时目录：%s" % args.workdir) + return 0 if bref._FAIL == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_routeb_gemv_probe.cu b/scripts/gguf_routeb_gemv_probe.cu new file mode 100644 index 000000000..9ed747829 --- /dev/null +++ b/scripts/gguf_routeb_gemv_probe.cu @@ -0,0 +1,158 @@ +// Standalone driver for linear_gguf's two NVIDIA paths, used by +// scripts/gguf_routeb_gemv_check.py: +// +// M <= kMaxDecodeM -> launch_gemv_decode (stage 3.2, decode path) +// M > kMaxDecodeM -> launch_prefill (stage 3.3, prefill path) +// +// The routing predicate is the one the op itself applies in +// linear_gguf_nvidia.cu::calculate, so a case run here goes through the same +// function the shipped kernel goes through. +// +// nvcc -O2 -std=c++17 -I /src/infiniop/ops/linear_gguf/nvidia \ +// gguf_routeb_gemv_probe.cu -o gemv_probe -lcublas +// +// gemv_probe +// +// A test harness, not a library target: it includes the two kernel headers and +// links cublas directly, so the paths can be checked numerically without a +// registered op or an InfiniCore handle. +#include +#include +#include + +#include +#include + +#include "linear_gguf_dequant.cuh" + +#define CUDA_CHECK(call) \ + do { \ + cudaError_t err__ = (call); \ + if (err__ != cudaSuccess) { \ + std::fprintf(stderr, "gemv probe: %s failed: %s\n", #call, \ + cudaGetErrorString(err__)); \ + return 5; \ + } \ + } while (0) + +static std::vector read_all(const char *path, size_t want) { + FILE *f = std::fopen(path, "rb"); + if (!f) { + std::fprintf(stderr, "gemv probe: cannot open %s\n", path); + exit(4); + } + std::vector buf(want); + const size_t got = std::fread(buf.data(), 1, want, f); + std::fclose(f); + if (got != want) { + std::fprintf(stderr, "gemv probe: short read on %s (wanted %zu, got %zu)\n", path, want, + got); + exit(4); + } + return buf; +} + +int main(int argc, char **argv) { + if (argc != 9) { + std::fprintf(stderr, + "usage: %s " + "\n", + argv[0]); + return 2; + } + const int32_t type = std::atoi(argv[1]); + const int m_count = std::atoi(argv[2]); + const int n_count = std::atoi(argv[3]); + const int k = std::atoi(argv[4]); + const int64_t row_bytes = std::atoll(argv[5]); + if (m_count <= 0 || n_count <= 0 || k <= 0 || row_bytes <= 0) { + std::fprintf(stderr, "gemv probe: bad geometry\n"); + return 2; + } + const bool prefill = m_count > op::linear_gguf::nvidia::kMaxDecodeM; + + std::vector h_a = read_all(argv[6], static_cast(m_count) * k * 2); + std::vector h_w = read_all(argv[7], static_cast(n_count) * row_bytes); + + __nv_bfloat16 *d_a = nullptr; + uint8_t *d_w = nullptr; + __nv_bfloat16 *d_c = nullptr; + void *d_scratch = nullptr; + cublasHandle_t blas = nullptr; + CUDA_CHECK(cudaMalloc(&d_a, h_a.size())); + CUDA_CHECK(cudaMalloc(&d_w, h_w.size())); + CUDA_CHECK(cudaMalloc(&d_c, static_cast(m_count) * n_count * 2)); + CUDA_CHECK(cudaMemcpy(d_a, h_a.data(), h_a.size(), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_w, h_w.data(), h_w.size(), cudaMemcpyHostToDevice)); + + // The prefill scratch is the op's workspace tensor; sized through the same + // helper the descriptor uses in create(), so the gate also pins that formula. + const size_t scratch_bytes = op::linear_gguf::nvidia::prefill_scratch_bytes(k); + if (prefill) { + CUDA_CHECK(cudaMalloc(&d_scratch, scratch_bytes)); + if (cublasCreate(&blas) != CUBLAS_STATUS_SUCCESS) { + std::fprintf(stderr, "gemv probe: cublasCreate failed\n"); + return 5; + } + } + + auto run_once = [&]() -> bool { + if (prefill) { + return op::linear_gguf::nvidia::launch_prefill( + blas, type, d_a, d_w, d_c, m_count, n_count, k, row_bytes, + d_scratch, scratch_bytes, nullptr); + } + return op::linear_gguf::nvidia::launch_gemv_decode( + type, d_a, d_w, d_c, m_count, n_count, k, row_bytes, nullptr); + }; + + if (!run_once()) { + std::fprintf(stderr, "gemv probe: %s rejected type %d (no decoder or bad K/row_bytes)\n", + prefill ? "prefill" : "gemv", type); + cublasDestroy(blas); + return 3; + } + CUDA_CHECK(cudaDeviceSynchronize()); + + // One timed run. Interpret with care: this probe is a numeric harness, the + // geometry comes from the caller (scripts/gguf_routeb_gemv_check.py) and small + // N makes the number latency-bound rather than bandwidth-bound. The bandwidth + // work is stage 6. + cudaEvent_t ev0, ev1; + CUDA_CHECK(cudaEventCreate(&ev0)); + CUDA_CHECK(cudaEventCreate(&ev1)); + CUDA_CHECK(cudaEventRecord(ev0)); + for (int i = 0; i < 10; ++i) { + run_once(); + } + CUDA_CHECK(cudaEventRecord(ev1)); + CUDA_CHECK(cudaEventSynchronize(ev1)); + float ms = 0.0f; + CUDA_CHECK(cudaEventElapsedTime(&ms, ev0, ev1)); + cudaEventDestroy(ev0); + cudaEventDestroy(ev1); + + std::vector h_c(static_cast(m_count) * n_count * 2); + CUDA_CHECK(cudaMemcpy(h_c.data(), d_c, h_c.size(), cudaMemcpyDeviceToHost)); + cudaFree(d_a); + cudaFree(d_w); + cudaFree(d_c); + cudaFree(d_scratch); + cublasDestroy(blas); + + FILE *out = std::fopen(argv[8], "wb"); + if (!out) { + std::fprintf(stderr, "gemv probe: cannot open %s\n", argv[8]); + return 4; + } + const bool wrote = std::fwrite(h_c.data(), 1, h_c.size(), out) == h_c.size(); + std::fclose(out); + if (!wrote) { + std::fprintf(stderr, "gemv probe: short write\n"); + return 4; + } + std::printf("gemv probe type=%d M=%d N=%d K=%d path=%s ok %.3f ms/iter %.2f GiB/s of weight\n", + type, m_count, n_count, k, prefill ? "prefill" : "gemv", ms / 10.0, + h_w.size() / (ms / 10.0 * 1e-3) / (1024.0 * 1024.0 * 1024.0)); + return 0; +} diff --git a/scripts/gguf_routeb_head_precision.py b/scripts/gguf_routeb_head_precision.py new file mode 100644 index 000000000..6c36d67f5 --- /dev/null +++ b/scripts/gguf_routeb_head_precision.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Recompute divergent lm_head rows in FP32 from GGUF weights and traced hidden states.""" + +import argparse +import json +import os +import sys + +sys.path.insert(0, os.path.join( + os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py")) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--gguf", required=True) + ap.add_argument("--model-path", required=True) + ap.add_argument("--compare", required=True) + ap.add_argument("--infinilm-trace", required=True) + ap.add_argument("--out", required=True) + args = ap.parse_args() + + import numpy as np + from gguf import GGUFReader + from gguf.constants import GGMLQuantizationType + from gguf.quants import dequantize + + with open(args.compare, encoding="utf-8") as f: + compared = {x["id"]: x for x in json.load(f)["cases"]} + with open(args.infinilm_trace, encoding="utf-8") as f: + traced = json.load(f)["cases"] + with open(os.path.join(args.model_path, "model.safetensors.index.json"), + encoding="utf-8") as f: + weight_map = json.load(f)["weight_map"] + from safetensors import safe_open + native_shard = safe_open( + os.path.join(args.model_path, weight_map["lm_head.weight"]), + framework="pt", device="cpu") + native_head = native_shard.get_slice("lm_head.weight") + reader = GGUFReader(args.gguf, "r") + output = next(t for t in reader.tensors if t.name == "output.weight") + type_name = GGMLQuantizationType(int(output.tensor_type)).name + results = [] + for case in traced: + item = compared[case["case_id"]] + diff = int(item["first_difference"]) + llama_token = int(item["llama_tokens"][diff]) + infini_token = int(item["infinilm_tokens"][diff]) + step = case["steps"][diff] + bits = np.asarray(step["hidden_bf16_bits"], dtype=np.uint16) + hidden = (bits.astype(np.uint32) << 16).view(np.float32) + rows = [] + for token_id in (llama_token, infini_token): + raw_row = output.data[token_id:token_id + 1] + row = np.asarray( + dequantize(raw_row, GGMLQuantizationType(int(output.tensor_type))), + dtype=np.float32).reshape(-1) + rows.append(row) + logits = [float(np.dot(hidden, row)) for row in rows] + native_rows = [ + native_head[token_id:token_id + 1].float().numpy().reshape(-1) + for token_id in (llama_token, infini_token) + ] + native_logits = [float(np.dot(hidden, row)) for row in native_rows] + result = { + "case_id": case["case_id"], "first_difference": diff, + "llama_token": llama_token, "infinilm_token": infini_token, + "llama_token_fp32_logit": logits[0], + "infinilm_token_fp32_logit": logits[1], + "fp32_margin_llama_minus_infinilm": logits[0] - logits[1], + "fp32_winner": llama_token if logits[0] > logits[1] else infini_token, + "bf16_weight_fp32_margin_llama_minus_infinilm": + native_logits[0] - native_logits[1], + "bf16_weight_fp32_winner": + llama_token if native_logits[0] > native_logits[1] else infini_token, + "hidden_shape": step["hidden_shape"], + } + results.append(result) + print("%-10s llama=%6d infini=%6d gguf_f32=%+.8f bf16w_f32=%+.8f winner=%d" % ( + result["case_id"], llama_token, infini_token, + result["fp32_margin_llama_minus_infinilm"], + result["bf16_weight_fp32_margin_llama_minus_infinilm"], + result["bf16_weight_fp32_winner"]), + flush=True) + report = {"gguf_lm_head_type": type_name, "cases": results} + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w", encoding="utf-8") as f: + json.dump(report, f, ensure_ascii=False, indent=2) + print("RESULT gguf_f32_llama_wins=%d/%d bf16_weight_f32_llama_wins=%d/%d" % ( + sum(x["fp32_winner"] == x["llama_token"] for x in results), len(results), + sum(x["bf16_weight_fp32_winner"] == x["llama_token"] for x in results), len(results)), + flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/gguf_routeb_infinilm_ref.py b/scripts/gguf_routeb_infinilm_ref.py new file mode 100755 index 000000000..1b25baa98 --- /dev/null +++ b/scripts/gguf_routeb_infinilm_ref.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Run deterministic raw-token completions through InfiniLM paged generate().""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--inputs", required=True) + ap.add_argument("--model-path", required=True) + ap.add_argument("--new-tokens", type=int, default=8) + ap.add_argument("--repeats", type=int, default=2) + ap.add_argument("--num-blocks", type=int, default=64) + ap.add_argument("--block-size", type=int, default=256) + ap.add_argument( + "--case-ids", + help="Optional comma-separated case IDs for focused regression runs.", + ) + ap.add_argument("--out", required=True) + args = ap.parse_args() + + import numpy as np + import infinicore + from infinilm.cache import PagedKVCacheConfig + from infinilm.distributed import DistConfig + from infinilm.infer_engine import GenerationConfig, InferEngine + from infinilm.modeling_utils import load_model_state_dict_by_file + + with open(args.inputs, encoding="utf-8") as f: + source = json.load(f) + if args.case_ids: + requested = {item.strip() for item in args.case_ids.split(",") if item.strip()} + source["cases"] = [item for item in source["cases"] if item["id"] in requested] + found = {item["id"] for item in source["cases"]} + missing = sorted(requested - found) + if missing: + raise ValueError(f"unknown --case-ids: {missing}") + + started = time.time() + engine = InferEngine( + model_path=args.model_path, + device=infinicore.device("cuda:0"), + distributed_config=DistConfig(1), + cache_config=PagedKVCacheConfig( + args.num_blocks, args.block_size, max_batch_size=1), + attention_backend="paged-attn", + ) + load_model_state_dict_by_file(engine, args.model_path, dtype=engine.dtype) + load_s = time.time() - started + print("MODEL_LOADED %.3fs cases=%d" % (load_s, len(source["cases"])), flush=True) + + outputs = [] + all_ok = True + for case in source["cases"]: + runs = [] + for repeat in range(args.repeats): + prompt = infinicore.from_list( + [[int(x) for x in case["input_ids"]]], dtype=infinicore.int64) + config = GenerationConfig( + max_new_tokens=args.new_tokens, + temperature=0.0, + top_k=1, + top_p=1.0, + eos_token_id=None, + stop_on_eos=False, + ignore_eos=True, + ) + run_started = time.time() + generated = engine.generate(prompt, config) + tokens = [int(np.asarray(x.to_numpy()).reshape(-1)[0]) for x in generated] + runs.append({ + "repeat": repeat, + "tokens": tokens, + "elapsed_s": round(time.time() - run_started, 4), + }) + deterministic = all(x["tokens"] == runs[0]["tokens"] for x in runs[1:]) + exact_length = all(len(x["tokens"]) == args.new_tokens for x in runs) + ok = deterministic and exact_length + all_ok &= ok + outputs.append({ + "id": case["id"], + "prompt": case["prompt"], + "input_ids": case["input_ids"], + "deterministic": deterministic, + "exact_length": exact_length, + "runs": runs, + }) + print("%-10s deterministic=%s length=%s tokens=%s" % ( + case["id"], deterministic, exact_length, runs[0]["tokens"]), flush=True) + + result = { + "engine": "InfiniLM", + "model_path": os.path.abspath(args.model_path), + "new_tokens": args.new_tokens, + "repeats": args.repeats, + "load_s": round(load_s, 4), + "cases": outputs, + "all_pass": all_ok, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + print("RESULT cases=%d all_pass=%s" % (len(outputs), all_ok), flush=True) + return 0 if all_ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_routeb_infinilm_trace.py b/scripts/gguf_routeb_infinilm_trace.py new file mode 100644 index 000000000..aab053a7c --- /dev/null +++ b/scripts/gguf_routeb_infinilm_trace.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Trace InfiniLM's exact paged decode path and capture BF16 logits.""" + +import argparse +import ctypes +import json +import os +import time + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--inputs", required=True) + ap.add_argument("--compare", required=True) + ap.add_argument( + "--expected-results", + help="Optional InfiniLM result JSON supplying the sequence that the current runtime must reproduce; first-difference positions still come from --compare.", + ) + ap.add_argument("--model-path", required=True) + ap.add_argument("--new-tokens", type=int, default=32) + ap.add_argument("--top-k", type=int, default=100) + ap.add_argument("--num-blocks", type=int, default=64) + ap.add_argument("--block-size", type=int, default=256) + ap.add_argument( + "--stop-at-first-diff", action="store_true", + help="Stop each case immediately after its known first-difference step.") + ap.add_argument( + "--prenorm-dump-root", + help="Optional root for per-case pre-final-RMSNorm binary dumps.") + ap.add_argument( + "--case-id", action="append", + help="Optionally trace only the named case; repeat for multiple cases.") + ap.add_argument( + "--operator-dump-layer", type=int, + help="Override the per-case layer selected for generic operator dumps.") + ap.add_argument("--gdn-dump-layer", type=int, + help="Enable GDN intermediate dumps for this layer.") + ap.add_argument("--gdn-dump-seq-len", type=int, default=1, + help="Sequence length for GDN intermediate dumps (default: 1).") + ap.add_argument("--out", required=True) + ap.add_argument("--allow-token-mismatch", action="store_true", + help="Diagnostic only: keep output even if selected tokens differ from expected.") + args = ap.parse_args() + + import numpy as np + import infinicore + from infinilm.cache import PagedKVCacheConfig + from infinilm.distributed import DistConfig + from infinilm.infer_engine import InferEngine + from infinilm.lib import _infinilm + from infinilm.modeling_utils import load_model_state_dict_by_file + + with open(args.inputs, encoding="utf-8") as f: + inputs = {x["id"]: x for x in json.load(f)["cases"]} + with open(args.compare, encoding="utf-8") as f: + divergent = [x for x in json.load(f)["cases"] + if x["first_difference"] is not None] + if args.case_id: + selected = set(args.case_id) + divergent = [x for x in divergent if x["id"] in selected] + missing = selected - {x["id"] for x in divergent} + if missing: + raise ValueError("unknown or non-divergent case ids: %s" % sorted(missing)) + expected_by_id = None + if args.expected_results: + with open(args.expected_results, encoding="utf-8") as f: + current = json.load(f) + expected_by_id = { + x["id"]: [int(t) for t in x["runs"][0]["tokens"]] + for x in current["cases"] + } + if len(divergent) >= max(2, args.num_blocks // 4): + raise ValueError("not enough independent Mamba cache rows") + + started = time.time() + cache_config = PagedKVCacheConfig( + args.num_blocks, args.block_size, max_batch_size=1) + engine = InferEngine( + model_path=args.model_path, device=infinicore.device("cuda:0"), + distributed_config=DistConfig(1), cache_config=cache_config, + attention_backend="paged-attn") + load_model_state_dict_by_file(engine, args.model_path, dtype=engine.dtype) + load_s = time.time() - started + results = [] + operator_dump_layers = { + "zh_04": 63, + "zh_06": 0, + "code_04": 20, + "math_04": 55, + } + + for case_index, item in enumerate(divergent): + case_id = item["id"] + if args.prenorm_dump_root: + case_dump_dir = os.path.join(args.prenorm_dump_root, case_id) + os.makedirs(case_dump_dir, exist_ok=True) + os.environ["INFINILM_FINAL_PRENORM_DUMP_DIR"] = case_dump_dir + os.environ["INFINILM_FINAL_PRENORM_DUMP_NUMEL"] = "5120" + # The final fused add-RMSNorm computes its scale from the unrounded + # FP32 sum of layer-63 residual and FFN output, then normalizes the + # BF16 materialized residual. Preserve both inputs for exact replay. + os.environ["INFINILM_LAYER_DUMP_DIR"] = case_dump_dir + os.environ["INFINILM_LAYER_DUMP_NUMEL"] = "5120" + os.environ["INFINILM_OPERATOR_DUMP_LAYER"] = str( + args.operator_dump_layer + if args.operator_dump_layer is not None + else operator_dump_layers.get(case_id, 63) + ) + if case_id in operator_dump_layers: + os.environ["INFINILM_ATTENTION_DUMP_DIR"] = case_dump_dir + os.environ["INFINILM_ATTENTION_DUMP_LAYER"] = str( + operator_dump_layers[case_id] + ) + if args.gdn_dump_layer is not None: + os.environ["INFINILM_GDN_DUMP_LAYER"] = str(args.gdn_dump_layer) + os.environ["INFINILM_GDN_DUMP_SEQ_LEN"] = str(args.gdn_dump_seq_len) + prompt = [int(x) for x in inputs[case_id]["input_ids"]] + expected = ( + expected_by_id[case_id] + if expected_by_id is not None + else [int(x) for x in item["infinilm_tokens"]] + ) + first_diff = int(item["first_difference"]) + kv_block = case_index + mamba_row = case_index + 1 + past = 0 + current = prompt + steps = [] + generated = [] + case_new_tokens = first_diff + 1 if args.stop_at_first_diff else args.new_tokens + for step in range(case_new_tokens): + if args.prenorm_dump_root and step == first_diff: + os.environ["INFINILM_LAYER_DUMP_FIRST_N"] = "64" + else: + os.environ.pop("INFINILM_LAYER_DUMP_FIRST_N", None) + seq_len = len(current) + total = past + seq_len + positions = list(range(past, total)) + if engine.position_id_axes > 1: + positions = [positions for _ in range(engine.position_id_axes)] + slot_base = kv_block * args.block_size + slot_mapping = [slot_base + i for i in range(past, total)] + tensors = { + "input_ids": infinicore.from_list([current], dtype=infinicore.int64).view([1, seq_len]), + "position_ids": infinicore.from_list(positions, dtype=infinicore.int64), + "past_kv_lengths": infinicore.from_list([past], dtype=infinicore.int32), + "total_kv_lengths": infinicore.from_list([total], dtype=infinicore.int32), + "input_offsets": infinicore.from_list([0, seq_len], dtype=infinicore.int32), + "cu_seqlens": infinicore.from_list([0, total], dtype=infinicore.int32), + "block_tables": infinicore.from_list([[kv_block]], dtype=infinicore.int32), + "slot_mapping": infinicore.from_list(slot_mapping, dtype=infinicore.int64), + "mamba_init_state_indices": infinicore.from_list( + [0 if step == 0 else mamba_row], dtype=infinicore.int32), + "mamba_final_state_indices": infinicore.from_list( + [mamba_row], dtype=infinicore.int32), + } + cpp_input = engine._build_input( + tensors["input_ids"], position_ids=tensors["position_ids"], + past_kv_lengths=tensors["past_kv_lengths"], + total_kv_lengths=tensors["total_kv_lengths"], + input_offsets=tensors["input_offsets"], cu_seqlens=tensors["cu_seqlens"], + block_tables=tensors["block_tables"], slot_mapping=tensors["slot_mapping"], + mamba_init_state_indices=tensors["mamba_init_state_indices"], + mamba_final_state_indices=tensors["mamba_final_state_indices"], + sample_all_positions=False, temperature=0.0, top_k=1, top_p=1.0) + output = _infinilm.InferEngine.forward(engine, cpp_input) + token = int(np.asarray(infinicore.Tensor(output.output_ids).to_numpy()).reshape(-1)[0]) + raw = infinicore.Tensor(output.logits) + shape = list(raw.shape) + cpu = raw.to(infinicore.device("cpu", 0)) + if cpu.dtype == infinicore.bfloat16: + bits_type = ctypes.c_uint16 * cpu.numel() + bits = np.ctypeslib.as_array(bits_type.from_address(cpu.data_ptr())).copy() + logits = (bits.astype(np.uint32) << 16).view(np.float32).reshape(shape) + elif cpu.dtype == infinicore.float32: + logits = np.ctypeslib.as_array( + (ctypes.c_float * cpu.numel()).from_address(cpu.data_ptr()) + ).copy().reshape(shape) + else: + raise TypeError("expected BF16 or F32 logits, got %s" % cpu.dtype) + logits = logits.reshape(-1, shape[-1])[-1] + order = np.argpartition(logits, -args.top_k)[-args.top_k:] + order = order[np.argsort(logits[order], kind="stable")[::-1]] + top_logit = float(logits[order[0]]) + candidates = [{"id": int(i), "logit": float(logits[i]), + "delta_from_top": float(logits[i] - top_logit)} for i in order] + step_result = {"step": step, "selected": token, + "logits_shape": shape, "top_logits": candidates} + if step == first_diff: + hidden = infinicore.Tensor(output.hidden_states) + hidden_shape = list(hidden.shape) + hidden_cpu = hidden.to(infinicore.device("cpu", 0)) + step_result["hidden_shape"] = hidden_shape + if hidden_cpu.dtype == infinicore.bfloat16: + hidden_bits_type = ctypes.c_uint16 * hidden_cpu.numel() + hidden_bits = np.ctypeslib.as_array( + hidden_bits_type.from_address(hidden_cpu.data_ptr())).copy() + step_result["hidden_dtype"] = "bfloat16" + step_result["hidden_bf16_bits"] = [int(x) for x in hidden_bits] + elif hidden_cpu.dtype == infinicore.float32: + hidden_values = np.ctypeslib.as_array( + (ctypes.c_float * hidden_cpu.numel()).from_address( + hidden_cpu.data_ptr())).copy() + step_result["hidden_dtype"] = "float32" + step_result["hidden_f32"] = [float(x) for x in hidden_values] + else: + raise TypeError("expected BF16 or F32 hidden state, got %s" % hidden_cpu.dtype) + steps.append(step_result) + generated.append(token) + current = [token] + past = total + expected = expected[:case_new_tokens] + stable = generated == expected + if not stable and not args.allow_token_mismatch: + raise RuntimeError("%s trace changed: %s != %s" % (case_id, generated, expected)) + results.append({"case_id": case_id, "tokens": generated, "steps": steps}) + print("%-10s tokens=%d stable=%s" % (case_id, len(generated), stable), flush=True) + + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w", encoding="utf-8") as f: + json.dump({"load_s": round(load_s, 4), "cases": results}, + f, ensure_ascii=False, indent=2) + print("RESULT cases=%d load=%.3fs" % (len(results), load_s), flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/gguf_routeb_llama_probe.py b/scripts/gguf_routeb_llama_probe.py new file mode 100644 index 000000000..1407abe08 --- /dev/null +++ b/scripts/gguf_routeb_llama_probe.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Dump llama-server one-token responses at Route-B divergence prefixes.""" + +import argparse +import json +import urllib.request + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--inputs", required=True) + ap.add_argument("--compare", required=True) + ap.add_argument("--case-ids", nargs="+", required=True) + ap.add_argument("--server", default="http://127.0.0.1:18080") + ap.add_argument("--out", required=True) + args = ap.parse_args() + with open(args.inputs, encoding="utf-8") as f: + inputs = {x["id"]: x for x in json.load(f)["cases"]} + with open(args.compare, encoding="utf-8") as f: + cases = {x["id"]: x for x in json.load(f)["cases"]} + results = [] + for case_id in args.case_ids: + item = cases[case_id] + diff = item["first_difference"] + prefix = inputs[case_id]["input_ids"] + item["llama_tokens"][:diff] + body = { + "prompt": prefix, "n_predict": 1, "temperature": 0.0, + "top_k": 1, "top_p": 1.0, "min_p": 0.0, "typical_p": 1.0, + "repeat_penalty": 1.0, "repeat_last_n": 0, + "presence_penalty": 0.0, "frequency_penalty": 0.0, + "seed": 1, "ignore_eos": True, "cache_prompt": False, + "return_tokens": True, "n_probs": 100, "stream": False, + "samplers": ["top_k", "temperature"], + } + req = urllib.request.Request( + args.server.rstrip("/") + "/completion", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, method="POST") + with urllib.request.urlopen(req, timeout=180) as response: + raw = json.load(response) + results.append({"case_id": case_id, "diff": diff, "prefix": prefix, + "response": raw}) + probs = raw.get("completion_probabilities", []) + print(case_id, "tokens=", raw.get("tokens"), "prob_entry=", probs[:1]) + with open(args.out, "w", encoding="utf-8") as f: + json.dump({"cases": results}, f, ensure_ascii=False, indent=2) + + +if __name__ == "__main__": + main() diff --git a/scripts/gguf_routeb_llama_ref.py b/scripts/gguf_routeb_llama_ref.py new file mode 100755 index 000000000..5df2bf84b --- /dev/null +++ b/scripts/gguf_routeb_llama_ref.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Run deterministic raw-token completions through llama-server.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +import urllib.error +import urllib.request + + +def post_json(url: str, body: dict, timeout: int) -> dict: + request = urllib.request.Request( + url, + data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.load(response) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", "replace") + raise RuntimeError("HTTP %d: %s" % (exc.code, detail[:2000])) from exc + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--inputs", required=True) + ap.add_argument("--server", default="http://127.0.0.1:18080") + ap.add_argument("--new-tokens", type=int, default=8) + ap.add_argument("--repeats", type=int, default=2) + ap.add_argument("--n-probs", type=int, default=20) + ap.add_argument("--timeout", type=int, default=180) + ap.add_argument("--out", required=True) + args = ap.parse_args() + + with open(args.inputs, encoding="utf-8") as f: + source = json.load(f) + outputs = [] + all_ok = True + for case in source["cases"]: + runs = [] + for repeat in range(args.repeats): + body = { + "prompt": case["input_ids"], + "n_predict": args.new_tokens, + "temperature": 0.0, + "top_k": 1, + "top_p": 1.0, + "min_p": 0.0, + "typical_p": 1.0, + "repeat_penalty": 1.0, + "repeat_last_n": 0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "seed": 1, + "ignore_eos": True, + "cache_prompt": False, + "return_tokens": True, + "n_probs": args.n_probs, + "stream": False, + "samplers": ["top_k", "temperature"], + } + started = time.time() + response = post_json( + args.server.rstrip("/") + "/completion", body, args.timeout) + tokens = [int(x) for x in response.get("tokens", [])] + probabilities = response.get("completion_probabilities", []) + runs.append({ + "repeat": repeat, + "tokens": tokens, + "content": response.get("content", ""), + "first_token_top_logprobs": ( + probabilities[0].get("top_logprobs", []) if probabilities else []), + "elapsed_s": round(time.time() - started, 4), + "tokens_evaluated": response.get("tokens_evaluated"), + "tokens_predicted": response.get("tokens_predicted"), + }) + deterministic = all(x["tokens"] == runs[0]["tokens"] for x in runs[1:]) + exact_length = all(len(x["tokens"]) == args.new_tokens for x in runs) + ok = deterministic and exact_length + all_ok &= ok + outputs.append({ + "id": case["id"], + "prompt": case["prompt"], + "input_ids": case["input_ids"], + "deterministic": deterministic, + "exact_length": exact_length, + "runs": runs, + }) + print("%-10s deterministic=%s length=%s tokens=%s" % ( + case["id"], deterministic, exact_length, runs[0]["tokens"])) + + result = { + "engine": "llama.cpp", + "server": args.server, + "new_tokens": args.new_tokens, + "repeats": args.repeats, + "cases": outputs, + "all_pass": all_ok, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + print("RESULT cases=%d all_pass=%s" % (len(outputs), all_ok)) + return 0 if all_ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_routeb_llama_trace.py b/scripts/gguf_routeb_llama_trace.py new file mode 100644 index 000000000..778b1b347 --- /dev/null +++ b/scripts/gguf_routeb_llama_trace.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Capture all llama-server token probabilities for divergent Route-B cases.""" + +import argparse +import json +import os +import urllib.request + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--inputs", required=True) + ap.add_argument("--compare", required=True) + ap.add_argument("--server", default="http://127.0.0.1:18080") + ap.add_argument("--new-tokens", type=int, default=32) + ap.add_argument("--n-probs", type=int, default=100) + ap.add_argument("--out", required=True) + args = ap.parse_args() + with open(args.inputs, encoding="utf-8") as f: + inputs = {x["id"]: x for x in json.load(f)["cases"]} + with open(args.compare, encoding="utf-8") as f: + divergent = [x for x in json.load(f)["cases"] + if x["first_difference"] is not None] + results = [] + for item in divergent: + case_id = item["id"] + body = { + "prompt": inputs[case_id]["input_ids"], + "n_predict": args.new_tokens, "temperature": 0.0, + "top_k": 1, "top_p": 1.0, "min_p": 0.0, "typical_p": 1.0, + "repeat_penalty": 1.0, "repeat_last_n": 0, + "presence_penalty": 0.0, "frequency_penalty": 0.0, + "seed": 1, "ignore_eos": True, "cache_prompt": False, + "return_tokens": True, "n_probs": args.n_probs, + "stream": False, "samplers": ["top_k", "temperature"], + } + req = urllib.request.Request( + args.server.rstrip("/") + "/completion", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, method="POST") + with urllib.request.urlopen(req, timeout=300) as response: + raw = json.load(response) + tokens = [int(x) for x in raw.get("tokens", [])] + expected = [int(x) for x in item["llama_tokens"]] + if tokens != expected: + raise RuntimeError("%s rerun changed: %s != %s" % (case_id, tokens, expected)) + results.append({ + "case_id": case_id, "tokens": tokens, + "completion_probabilities": raw.get("completion_probabilities", []), + }) + print("%-10s tokens=%d probabilities=%d stable=%s" % ( + case_id, len(tokens), len(results[-1]["completion_probabilities"]), tokens == expected), + flush=True) + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w", encoding="utf-8") as f: + json.dump({"cases": results}, f, ensure_ascii=False, indent=2) + print("RESULT cases=%d" % len(results), flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/gguf_routeb_probe_params.py b/scripts/gguf_routeb_probe_params.py new file mode 100644 index 000000000..6b32994d2 --- /dev/null +++ b/scripts/gguf_routeb_probe_params.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""探针：用 mini qwen3_5 config 构造 InferEngine，导出 C++ 侧权威参数键与 shape。""" +import json +import os +import sys +import tempfile + +import infinicore +from infinilm.cache import StaticKVCacheConfig +from infinilm.distributed import DistConfig +from infinilm.infer_engine import InferEngine + +CFG = { + "model_type": "qwen3_5", + "torch_dtype": "bfloat16", + "tie_word_embeddings": False, + "text_config": { + "model_type": "qwen3_5_text", + "hidden_size": 512, + "num_hidden_layers": 8, + "num_attention_heads": 2, + "num_key_value_heads": 1, + "head_dim": 256, + "intermediate_size": 1024, + "rms_norm_eps": 1e-6, + "max_position_embeddings": 262144, + "vocab_size": 1024, + "full_attention_interval": 4, + "linear_num_key_heads": 2, + "linear_num_value_heads": 6, + "linear_key_head_dim": 128, + "linear_value_head_dim": 128, + "linear_conv_kernel_dim": 4, + "attention_bias": False, + "rope_parameters": { + "rope_type": "mrope", + "rope_theta": 10000000.0, + "partial_rotary_factor": 0.25, + "mrope_section": [11, 11, 10], + "mrope_interleaved": True, + }, + }, +} + + +def main(): + dev = sys.argv[1] if len(sys.argv) > 1 else "cpu" + d = infinicore.device(dev, 0) + tmp = tempfile.mkdtemp(prefix="mini_qwen35_") + with open(os.path.join(tmp, "config.json"), "w") as f: + json.dump(CFG, f, indent=2) + eng = InferEngine( + model_path=tmp, + device=d, + distributed_config=DistConfig(1), + cache_config=StaticKVCacheConfig(max_batch_size=1, max_cache_len=16), + ) + keys = list(eng.state_dict_keyname()) + sd = eng.state_dict()[0] + print("# device=%s 参数总数=%d" % (dev, len(keys))) + for k in sorted(keys): + t = sd.get(k) + shape = tuple(t.shape) if t is not None else "" + dt = getattr(t, "dtype", "") + print("%-58s %-22s %s" % (k, str(shape), dt)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_routeb_prompts.jsonl b/scripts/gguf_routeb_prompts.jsonl new file mode 100644 index 000000000..d9cb51da0 --- /dev/null +++ b/scripts/gguf_routeb_prompts.jsonl @@ -0,0 +1,32 @@ +{"id":"en_01","category":"english","prompt":"The capital of France is","max_new_tokens":32} +{"id":"zh_01","category":"chinese","prompt":"中国的首都是","max_new_tokens":32} +{"id":"code_01","category":"code","prompt":"def fibonacci(n):\n ","max_new_tokens":32} +{"id":"en_02","category":"english","prompt":"Water freezes at a temperature of","max_new_tokens":32} +{"id":"en_03","category":"english","prompt":"The three primary colors of light are","max_new_tokens":32} +{"id":"en_04","category":"english","prompt":"In one concise sentence, photosynthesis is","max_new_tokens":32} +{"id":"en_05","category":"english","prompt":"A good unit test should verify that","max_new_tokens":32} +{"id":"en_06","category":"english","prompt":"The next number in the sequence 2, 4, 8, 16 is","max_new_tokens":32} +{"id":"en_07","category":"english","prompt":"To make a cup of tea, first","max_new_tokens":32} +{"id":"en_08","category":"english","prompt":"The opposite of expensive is","max_new_tokens":32} +{"id":"zh_02","category":"chinese","prompt":"请用一句话解释什么是重力：","max_new_tokens":32} +{"id":"zh_03","category":"chinese","prompt":"一年有四个季节，分别是","max_new_tokens":32} +{"id":"zh_04","category":"chinese","prompt":"计算机中的CPU主要负责","max_new_tokens":32} +{"id":"zh_05","category":"chinese","prompt":"如果今天是星期一，那么三天后是","max_new_tokens":32} +{"id":"zh_06","category":"chinese","prompt":"健康作息通常包括","max_new_tokens":32} +{"id":"zh_07","category":"chinese","prompt":"长城是中国著名的","max_new_tokens":32} +{"id":"zh_08","category":"chinese","prompt":"把下面这句话续写完整：人工智能可以帮助人们","max_new_tokens":32} +{"id":"code_02","category":"code","prompt":"def is_even(x):\n return","max_new_tokens":32} +{"id":"code_03","category":"code","prompt":"SELECT name FROM users WHERE","max_new_tokens":32} +{"id":"code_04","category":"code","prompt":"for i in range(5):\n print(","max_new_tokens":32} +{"id":"math_01","category":"math","prompt":"12 * 7 =","max_new_tokens":32} +{"id":"math_02","category":"math","prompt":"If x + 5 = 12, then x =","max_new_tokens":32} +{"id":"math_03","category":"math","prompt":"List the first three prime numbers:","max_new_tokens":32} +{"id":"math_04","category":"structured","prompt":"Return a JSON object with keys name and age:","max_new_tokens":32} +{"id":"ctx_01","category":"context","prompt":"Alice placed the red book on the kitchen table. Bob moved the blue cup to the shelf. Question: Where is the red book? Answer:","max_new_tokens":32} +{"id":"ctx_02","category":"context","prompt":"The meeting starts at 09:30 and lasts 45 minutes. Question: At what time does it end? Answer:","max_new_tokens":32} +{"id":"ctx_03","category":"context","prompt":"A shop has apples, pears, and oranges. Only the pears are on sale today. Question: Which fruit is on sale? Answer:","max_new_tokens":32} +{"id":"ctx_04","category":"context","prompt":"小明把钥匙放进了书包，然后把书包放在椅子上。问题：钥匙在哪里？回答：","max_new_tokens":32} +{"id":"mix_01","category":"mixed","prompt":"Translate into English: 机器学习","max_new_tokens":32} +{"id":"mix_02","category":"mixed","prompt":"解释 API endpoint 的含义：","max_new_tokens":32} +{"id":"mix_03","category":"mixed","prompt":"Symbols test: α + β =","max_new_tokens":32} +{"id":"mix_04","category":"mixed","prompt":"Complete the pair: 北京 -> China; Tokyo ->","max_new_tokens":32} diff --git a/scripts/gguf_routeb_shape_contract.py b/scripts/gguf_routeb_shape_contract.py new file mode 100644 index 000000000..7140d60d0 --- /dev/null +++ b/scripts/gguf_routeb_shape_contract.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +""" +InfiniLM 路线 B —— 阶段 0.3 shape 契约回归（执行方案 §4.2 第 3 条） + +三方对账，任何一处对不上都在此暴露，而不是留到阶段 5 被 strict=False 静默丢权重： + + 1. 框架侧：CPU 构造 mini qwen3_5 InferEngine，导出 C++ 真实参数键 + shape， + 与 gguf_mapping.build_plan(MINI) 做双向 diff（缺键 / 多键 / shape 不符即 FAIL）。 + 2. GGUF 侧：build_plan(REAL) 的每条 gguf 名必须在真文件中存在，shape 必须与 + ne 反序一致（含 conv1d 的 squeeze），blob 条目的行字节必须能被块大小整除； + 共用同一源张量的条目（attn_qkv -> q|k|v）其 slices 必须无重叠地精确覆盖全行。 + 3. 反向无遗漏：真文件中未被丢弃、又未被任何条目消费的张量 = 0。 + 4. 阶段 3 作用域：统计 blob 实际用到的 GGML 类型集合，作为 kernel 必须覆盖的清单。 + +用法： + source scripts/gguf_routeb_env.sh + python3 scripts/gguf_routeb_shape_contract.py [--skip-min] [--engine-device cpu] +退出码 0 表示全部 PASS。 +""" + +from __future__ import annotations + +import argparse +import collections +from math import prod +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py")) + +import gguf_mapping as M # noqa: E402 +from gguf import GGUFReader # noqa: E402 +from gguf.constants import GGML_QUANT_SIZES, GGMLQuantizationType as Q # noqa: E402 + +DEFAULT_GGUF = "/home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf" +TYPE_NAME = {int(v.value): str(v.name) for v in Q} + +_PASS = 0 +_FAIL = 0 + + +def check(name, ok, detail=""): + global _PASS, _FAIL + if ok: + _PASS += 1 + print(" PASS %s" % name) + else: + _FAIL += 1 + print(" FAIL %s%s" % (name, ("\n %s" % detail) if detail else "")) + return ok + + +def dims_from_text_config(tc): + """config.json 的 text_config 段 -> Dims。打包器写出 config.json 后也用它自检。""" + return M.Dims( + hidden=tc["hidden_size"], n_q_heads=tc["num_attention_heads"], + n_kv_heads=tc["num_key_value_heads"], head_dim=tc["head_dim"], + ffn=tc["intermediate_size"], lin_k_heads=tc["linear_num_key_heads"], + lin_v_heads=tc["linear_num_value_heads"], lin_k_dim=tc["linear_key_head_dim"], + lin_v_dim=tc["linear_value_head_dim"], conv_kernel=tc["linear_conv_kernel_dim"], + vocab=tc["vocab_size"], n_layers=tc["num_hidden_layers"], + interval=tc["full_attention_interval"], + ) + + +def framework_side(engine_device): + print("\n== 1. 框架侧：mini InferEngine vs build_plan(MINI) ==") + import infinicore + from infinilm.cache import StaticKVCacheConfig + from infinilm.distributed import DistConfig + from infinilm.infer_engine import InferEngine + from gguf_routeb_probe_params import CFG + + check("探针 CFG 与 MINI 维度一致", dims_from_text_config(CFG["text_config"]) == M.MINI, + "cfg=%s\n MINI=%s" % (dims_from_text_config(CFG["text_config"]), M.MINI)) + + # 不能用 /tmp：开发机上只读，写不进去。缓存在 HOME 下，无需清理权限。 + tmp = os.path.join(os.environ.get("XDG_CACHE_HOME") + or os.path.expanduser("~/.cache"), "gguf_routeb_mini_cfg") + os.makedirs(tmp, exist_ok=True) + with open(os.path.join(tmp, "config.json"), "w") as f: + json.dump(CFG, f) + eng = InferEngine(model_path=tmp, device=infinicore.device(engine_device, 0), + distributed_config=DistConfig(1), + cache_config=StaticKVCacheConfig(max_batch_size=1, max_cache_len=16)) + sd = eng.state_dict()[0] + actual = {k: tuple(int(x) for x in sd[k].shape) for k in eng.state_dict_keyname()} + print(" -> 引擎导出 %d 个参数（device=%s）" % (len(actual), engine_device)) + + plan = M.build_plan(M.MINI) + want = {} + for e in plan: + assert e.infinilm not in want, "映射表内重复键：%s" % e.infinilm + want[e.infinilm] = M.compress(e.shape) + check("映射表无重复键（%d 条）" % len(plan), len(want) == len(plan), "%d vs %d" % (len(want), len(plan))) + + missing = sorted(set(actual) - set(want)) + extra = sorted(set(want) - set(actual)) + check("无缺键（框架要但映射表未提供 -> 会保持随机初始化）", not missing, str(missing[:12])) + check("无多键（映射表提供但框架无此参数 -> strict=False 下静默丢）", not extra, str(extra[:12])) + + bad = [(k, want[k], M.compress(actual[k])) for k in sorted(set(actual) & set(want)) + if M.compress(actual[k]) != want[k]] + check("逐键 shape 全等（压缩长度为 1 的维后）", not bad, str(bad[:8])) + + +def dense_iq_bf16(plan, tensors, gguf_types, prod): + """v1 被稠密化的那 5 个 IQ4 张量若改回 blob，可省下的显存字节数。""" + return sum(prod(e.shape) * 2 - int(tensors[e.gguf].n_bytes) for e in plan + if not e.blob and gguf_types.get(e.gguf) in M.V1_IQUANT_DENSE + and e.gguf in tensors) + + +def gguf_side(path): + print("\n== 2. GGUF 侧：build_plan(REAL) vs 真文件 ==") + reader = GGUFReader(path) + tensors = {t.name: t for t in reader.tensors} + gguf_types = {n: TYPE_NAME.get(int(t.tensor_type), str(t.tensor_type)) + for n, t in tensors.items()} + plan = M.build_plan(M.REAL) + n_exc = M.apply_v1_exceptions(plan, gguf_types) # v1 稠密化 IQ4（阶段 6 取消） + check("映射条目数 = %d" % len(plan), len(plan) == 947, str(len(plan))) + check("v1 稠密化例外命中 5 个 IQ4 张量", n_exc == 5, str(n_exc)) + + bad_name, bad_shape, bad_type, bad_rows = [], [], [], [] + ok_blob = 0 + type_hist = collections.defaultdict(collections.Counter) + for e in plan: + t = tensors.get(e.gguf) + if t is None: + bad_name.append(e.gguf) + continue + ne = tuple(int(x) for x in t.shape) # GGML ne 序 = [in, out] + hf = tuple(reversed(ne)) # HF/InfiniLM 序 = [out, in] + tn = TYPE_NAME.get(int(t.tensor_type), str(t.tensor_type)) + suffix = e.gguf.split(".", 2)[2] if e.gguf.startswith("blk.") else e.gguf + type_hist[suffix][tn] += 1 + + allowed = M.NATIVE_BLOB_TYPES if e.blob else M.DENSE_SRC_TYPES + if tn not in allowed: + bad_type.append("%s: %s 不在 %s" % (e.gguf, tn, allowed)) + # 共用源张量的条目只占一个行段，比对该段长度而非全量 + exp = M.compress(e.shape) + got = M.compress(hf) + if e.slices: + s, ep = e.slices[0] + got = (ep - s,) + got[1:] + if exp != got: + bad_shape.append("%s: 表 %s vs GGUF %s%s" % (e.gguf, exp, got, + "" if not e.slices else "（按行段 %s）" % (e.slices[0],))) + continue + if e.blob: + blk, ts = (int(x) for x in GGML_QUANT_SIZES[Q[tn]]) + n_in = hf[-1] + if n_in % blk: + bad_rows.append("%s: in=%d 不能被块大小 %d 整除" % (e.gguf, n_in, blk)) + else: + row_bytes = n_in // blk * ts + if row_bytes * hf[0] != int(t.n_bytes): + bad_rows.append("%s: %d 行 x %d B != n_bytes %d" + % (e.gguf, hf[0], row_bytes, t.n_bytes)) + else: + ok_blob += 1 + + check("每条目的 GGUF 源张量都存在", not bad_name, str(sorted(set(bad_name))[:10])) + check("源类型均在可实现集合内", not bad_type, str(bad_type[:6])) + check("shape 与 ne 反序全等（含 conv1d squeeze）", not bad_shape, str(bad_shape[:8])) + check("blob 条目行字节可整除且与 n_bytes 相符（%d 条）" % ok_blob, not bad_rows, str(bad_rows[:6])) + + print("\n== 3. 切片覆盖 + 反向无遗漏 ==") + shared = collections.defaultdict(list) + for e in plan: + shared[e.gguf].append(e) + cov = [] + for name, es in shared.items(): + if name not in tensors: + continue + n_out = int(tuple(reversed(tensors[name].shape))[0]) + segs = sorted((s, ep) for e in es for s, ep in e.slices) + if len(es) == 1 and not segs: + continue + if not segs: + cov.append("%s: %d 个条目共用但无 slices 声明" % (name, len(es))) + elif segs[0][0] != 0 or segs[-1][1] != n_out or \ + any(segs[i][1] != segs[i + 1][0] for i in range(len(segs) - 1)): + cov.append("%s: 切片 %s 未无重叠覆盖 [0,%d)" % (name, segs, n_out)) + check("共用源张量的切片精确覆盖全行", not cov, str(cov[:6])) + + used = {e.gguf for e in plan} + dropped = {n for n in tensors if n.startswith(M.DROP_PREFIXES)} + orphan = sorted(set(tensors) - used - dropped) + check("无既未消费又未丢弃的张量", not orphan, str(orphan[:10])) + print(" -> 消费 %d 个 / 丢弃 %d 个（MTP blk.%d.*）/ 文件共 %d 个" + % (len(set(tensors) & used), len(dropped), M.MTP_BLOCK, len(tensors))) + + print("\n== 4. 阶段 3 kernel 作用域 ==") + all_types = collections.Counter() + for hist in type_hist.values(): + all_types.update(hist) + print(" 按条目统计：" + ", ".join("%s x%d" % (tn, c) for tn, c in all_types.most_common())) + blob_types = {e.gguf: TYPE_NAME.get(int(tensors[e.gguf].tensor_type)) + for e in plan if e.blob and e.gguf in tensors} + seen = collections.Counter(blob_types.values()) + print(" blob 条目源类型：" + ", ".join("%s x%d" % (tn, c) for tn, c in seen.most_common())) + check("阶段 3 v1 需实现的类型集合 = %s" % sorted(seen), + set(seen) == set(M.NATIVE_BLOB_TYPES), + "缺 %s / 多 %s" % (set(M.NATIVE_BLOB_TYPES) - set(seen), set(seen) - set(M.NATIVE_BLOB_TYPES))) + check("IQ4_* 已被 v1 稠密化例外排除", not ({"IQ4_NL", "IQ4_XS"} & set(seen)), str(sorted(seen))) + giB = 2 ** 30 + total = sum(int(t.n_bytes) for t in reader.tensors) + blob_src = {e.gguf for e in plan if e.blob and e.gguf in tensors} + dense_src = {e.gguf for e in plan if not e.blob and e.gguf in tensors} - blob_src + b = sum(int(tensors[n].n_bytes) for n in blob_src) + d_src = sum(int(tensors[n].n_bytes) for n in dense_src) + drop = sum(int(t.n_bytes) for n, t in tensors.items() if n in dropped) + print(" -> 文件 %.3f GiB = 逐字节 blob %.3f（%d 个） + 稠密化源 %.3f（%d 个）" + " + MTP 丢弃 %.3f" % (total / giB, b / giB, len(blob_src), + d_src / giB, len(dense_src), drop / giB)) + # 稠密化条目的显存 = InfiniLM 元素数 x 2B（按行段拆分的条目只算自己那段） + dense_bf16 = sum(prod(e.shape) * 2 for e in plan if not e.blob) + budget = (b + dense_bf16) / giB + print(" -> v1 显存预算：blob %.3f + 稠密化 BF16 %.3f = %.3f GiB" + % (b / giB, dense_bf16 / giB, budget)) + check("v1 权重预算 <= 24.0 GiB（单卡 5090 32.6 GiB 留 KV 余量）", + budget <= 24.0, "%.3f GiB" % budget) + # 阶段 6 复利：IQ4 上原生 kernel 后再省；emb/lm_head 上 kernel 再省 2.51 GiB + st6 = budget - dense_iq_bf16(plan, tensors, gguf_types, prod) / giB + emb_out_blob = int(tensors["token_embd.weight"].n_bytes) + int(tensors["output.weight"].n_bytes) + emb_out_bf16 = sum(prod(e.shape) * 2 for e in plan + if not e.blob and e.gguf in ("token_embd.weight", "output.weight")) + st6b = st6 - (emb_out_bf16 - emb_out_blob) / giB + print(" -> 阶段 6：IQ4 原生 kernel %.3f GiB；再 emb/lm_head 原生 %.3f GiB" + % (st6, st6b)) + check("阶段 6 预算单调下降", st6b < st6 < budget, "%.3f / %.3f / %.3f" % (st6b, st6, budget)) + check("阶段 6 目标态 <= 20.5 GiB（相对路线 A 的 26.6 GiB 权重）", st6b <= 20.5, + "%.3f GiB" % st6b) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--gguf", default=DEFAULT_GGUF) + ap.add_argument("--skip-min", action="store_true", help="跳过需要 infinilm 的框架侧检查") + ap.add_argument("--engine-device", default="cpu") + a = ap.parse_args() + + if not a.skip_min: + framework_side(a.engine_device) + gguf_side(a.gguf) + + print("\n== 结果：%d PASS / %d FAIL ==" % (_PASS, _FAIL)) + return 0 if _FAIL == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_routeb_stage2_check.py b/scripts/gguf_routeb_stage2_check.py new file mode 100644 index 000000000..fb20ba130 --- /dev/null +++ b/scripts/gguf_routeb_stage2_check.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +""" +InfiniLM 路线 B —— 阶段 2 验收（执行方案 §6.3 判据 1–3） + +拿 mini8 产物（8 层 / 121 条目 / blob 61 + 稠密 60）在**新写的 +GGUFBlockQuantization** 上走一遍「构造 -> 键对账 -> 加载 -> 首次 forward」。 +每条判据都能独立失败，不是「能加载」的同义反复： + + 1. 构造：C++ 侧每个 Linear 都用自己的 checkpoint stem 查类型表。stem 拼错 / + 融合组没登记 / 表外 ggml type -> resolve() 抛错，构造直接失败。 + 所以「构造通过」= 所有被查询的 stem 都恰好命中 1 个候选。 + 2. 键双向 diff：引擎 state_dict_keyname() 与产物 index 的张量名必须完全相等。 + 3. 逐键 shape 对账：blob 必须是 [out, row_bytes]，(block_size, type_size) 直接从 + gguf-py 的 GGML_QUANT_SIZES 取（**独立于 gguf.cpp 里那份常量表**）——两侧谁算 + 窄了/算宽了都会在下层的 load_no_sync 里炸，这里先炸出来，报错更好读。 + 4. 加载：load_model_state_dict_by_file 末尾的 check_parameters 对缺键/多键直接 + raise，等于框架替我们做 strict=False 的兜底审查（判据 1）。 + 5. 首次 forward：blob Linear 必须真的进了 linear_gguf 并返回（判据 3：没有静默 + 回落稠密 GEMM）。阶段 2 时这里期望的是抛「阶段 3 实现」占位，3.2 落地后 + 期望反过来：日志里出现带 M/N/K/ggml_type/row_bytes 的契约行，且 row_bytes + 用 gguf-py 的 (block_size, type_size) 独立重算相等。整模端到端（generate） + 由 scripts/gguf_routeb_stage3_check.py 覆盖：forward_raw 的 python 签名不暴 + 露 mamba_*_state_indices，GDN 模型走完 in_proj 后会在下游 conv1d 里因可选 + 入参为空抛 bad_optional_access —— 上游 API 缺口，与 GGUF 无关。 + +用法： + source /home/liuxd/InfiniLM/scripts/gguf_routeb_env.sh + /usr/bin/python3 scripts/gguf_routeb_stage2_check.py [--device cuda:0] [--no-forward] +退出码 0 = 全部 PASS。 +""" + +from __future__ import annotations + +import argparse +import collections +import ctypes +import json +import os +import re +import sys +import traceback + +_HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, _HERE) +sys.path.insert(0, os.path.join( + os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py")) + +DEFAULT_MODEL = "/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8" +BLOB_SUFFIX = "weight_bytes" +# 与 csrc/layers/quantization/gguf.cpp 里那条诊断日志的格式对应 +BLOB_RE = re.compile( + r"linear_gguf: 首个 blob 前向 (\S+) — M=(\d+) N=(\d+) K=(\d+) " + r"ggml_type=(\d+) row_bytes=(\d+)") +MAX_DECODE_M = 8 # kMaxDecodeM：<=8 走 gemv，>8 走 prefill（阶段 3.3 起不再是上限） +PROMPT_TOKENS = 3 # 下面 forward_raw 喂的 token 数，用来核对契约行的 M + +_PASS = 0 +_FAIL = 0 + + +def check(name, ok, detail=""): + global _PASS, _FAIL + if ok: + _PASS += 1 + print(" PASS %s" % name) + else: + _FAIL += 1 + print(" FAIL %s%s" % (name, ("\n %s" % detail) if detail else "")) + return ok + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model-path", default=DEFAULT_MODEL) + ap.add_argument("--device", default="cuda:0") + ap.add_argument("--no-forward", action="store_true", help="跳过首次 forward 判据") + a = ap.parse_args() + + import infinicore + from infinilm.cache import StaticKVCacheConfig + from infinilm.distributed import DistConfig + from infinilm.infer_engine import InferEngine + from infinilm.modeling_utils import load_model_state_dict_by_file + from safetensors import safe_open + from gguf.constants import GGML_QUANT_SIZES + + # ---------------------------------------------------------------- 0. config + print("\n== 0. 产物 config.json ==") + with open(os.path.join(a.model_path, "config.json")) as f: + cfg = json.load(f) + qc = cfg.get("quantization_config") or {} + check("quantization_config 在顶层且 quant_method=gguf", qc.get("quant_method") == "gguf", + "qc keys=%s" % sorted(qc)) + table = qc.get("ggml_types") or {} + check("类型表非空（%d 条）" % len(table), bool(table)) + bs_ts = {int(t): (int(v[0]), int(v[1])) for t, v in GGML_QUANT_SIZES.items()} + ids = sorted({v for v in table.values() if isinstance(v, int)}) + check("表内 type id 都能从 gguf-py 查出 (block_size, type_size)：%s" % ids, + all(i in bs_ts for i in ids), str([i for i in ids if i not in bs_ts])) + + with open(os.path.join(a.model_path, "model.safetensors.index.json")) as f: + weight_map = json.load(f)["weight_map"] + n_blob = sum(1 for v in table.values() if isinstance(v, int)) + print(" -> 类型表 %d 条：blob %d / 稠密 %d；产物 index %d 个张量；key_prefix='%s'" + % (len(table), n_blob, len(table) - n_blob, len(weight_map), qc.get("key_prefix"))) + # 溯源：表键有两种历史形态。新规则（§6.0 纠正 2）= 张量名原文（与产物 index 同名）； + # 旧规则 = 去前缀的相对名且 blob 归一成 .weight（与 index 不同名）。两者 C++ 都能 + # 命中（裁前缀时 key_prefix 缺失就取 ""，探键时 weight_bytes / weight 都探）， + # 但必须知道眼下这份产物是哪一种，不然对不上时会查错方向。 + n_ident = len(set(table) & set(weight_map)) + print(" -> 表键形态：%d/%d 条与产物张量名同名（新规则），其余 %d 条为相对名或前缀外键" + % (n_ident, len(table), len(table) - n_ident)) + + # ------------------------------------------------------------- 1. 构造引擎 + print("\n== 1. 用 GGUFBlockQuantization 构造引擎（device=%s）==" % a.device) + # infinicore.device("cuda:0", 0) 会报 “index should not be provided”，带冒号就不能再传 index + dev_spec = infinicore.device(a.device) if ":" in a.device else infinicore.device(a.device, 0) + try: + eng = InferEngine( + model_path=a.model_path, + device=dev_spec, + distributed_config=DistConfig(1), + cache_config=StaticKVCacheConfig(max_batch_size=1, max_cache_len=16), + ) + ok, err = True, "" + except Exception as e: # noqa: BLE001 + ok, err = False, "%s: %s" % (type(e).__name__, str(e)[:1200]) + check("构造通过（= 被查询的 stem 全部恰好命中 1 个候选，且无 TP/bias 违规）", ok, err) + if not ok: + print("\n构造都没过，后面全部跳过\n" + traceback.format_exc()) + return 1 + check("引擎确实走 GGUF 方案", + (eng.hf_config.get("quantization_config") or {}).get("quant_method") == "gguf") + + # --------------------------------------------------------- 2. 键双向 diff + print("\n== 2. 引擎参数键 vs 产物张量名 ==") + keys = list(eng.state_dict_keyname()) + extra = sorted(set(keys) - set(weight_map)) + missing = sorted(set(weight_map) - set(keys)) + check("产物有、引擎不要（多键 -> strict=False 下静默丢权重）", not extra, str(extra[:12])) + check("引擎要、产物没有（缺键 -> 保持随机初始化）", not missing, str(missing[:12])) + check("键数一致（引擎 %d / 产物 %d）" % (len(keys), len(weight_map)), + len(keys) == len(weight_map)) + + # ---------------------------------------------- 3. 逐键 dtype / shape 对账 + print("\n== 3. 逐键 shape 对账（blob 行字节独立重算）==") + sd_keys = set(keys) + meta = {} + for fn in sorted(set(weight_map.values())): + with safe_open(os.path.join(a.model_path, fn), framework="pt") as f: + for k in f.keys(): + if k in sd_keys: + meta[k] = (f.get_slice(k).get_dtype(), list(f.get_slice(k).get_shape())) + eng_sd = eng.state_dict()[0] + + # 照抄 C++ GGUFBlockQuantization::resolve() 的查表语义：表键 = 产物名裁掉 + # 已声明的 key_prefix（未声明则为 ""，即保留原样），探 stem+"weight_bytes" 与 + # stem+"weight" 两个候选。引擎侧的绝对键 = 模型参数路径，可能与表键不同形， + # 所以这里按候选集查而不是 table[k] 直查（命中数 != 1 算 FAIL，不让脚本 KeyError）。 + MOD_PREFIX = "model.language_model." + W_BLOB = "." + BLOB_SUFFIX + + def table_hits(k): + cands = {k, k[: -len(W_BLOB)] + ".weight" if k.endswith(W_BLOB) else k} + for base in list(cands): + if base.startswith(MOD_PREFIX): + cands.add(base[len(MOD_PREFIX):]) + declared = qc.get("key_prefix") or "" + for base in list(cands): + if declared and base.startswith(declared): + cands.add(base[len(declared):]) + return sorted(c for c in cands if c in table) + + bad_shape, bad_dtype, n_blob_eng, n_table_form = [], [], 0, collections.Counter() + for k in sorted(sd_keys & set(meta)): + e_shape = [int(x) for x in eng_sd[k].shape] + if e_shape != list(meta[k][1]): + bad_shape.append("%s: 引擎 %s vs 产物 %s" % (k, e_shape, meta[k][1])) + if k.endswith("." + BLOB_SUFFIX): + n_blob_eng += 1 + if "U8" not in str(eng_sd[k].dtype).upper() or meta[k][0] != "U8": + bad_dtype.append("%s: 引擎 %s / 产物 %s" % (k, eng_sd[k].dtype, meta[k][0])) + hits = table_hits(k) + if len(hits) != 1: + bad_shape.append("%s: 类型表命中 %d 个候选 %s（C++ 会抛或静默走稠密）" + % (k, len(hits), hits[:4])) + continue + n_table_form["与张量名同名" if hits[0] == k else "相对名/归一后缀"] += 1 + _bs, ts = bs_ts[int(table[hits[0]])] + if e_shape and ts and e_shape[-1] % ts: + bad_shape.append("%s: row_bytes=%d 不是 type_size %d 的整数倍" + % (k, e_shape[-1], ts)) + check("%d 个 blob 键在引擎侧与产物侧都是 U8" % n_blob_eng, not bad_dtype, + str(bad_dtype[:6])) + check("全部 %d 键 shape 逐字相等（blob 为 [out, row_bytes]）" % len(sd_keys), + not bad_shape, str(bad_shape[:8])) + print(" -> %d 个 blob 命中的表键形态：%s" + % (n_blob_eng, ", ".join("%s x%d" % kv for kv in n_table_form.most_common()) + or "无")) + + # ----------------------------------------------------------------- 4. 加载 + print("\n== 4. 加载（末尾 check_parameters 会对缺/多键抛错 = 判据 1）==") + try: + load_model_state_dict_by_file(eng, a.model_path, dtype=eng.dtype) + ok, err = True, "" + except Exception as e: # noqa: BLE001 + ok, err = False, "%s: %s" % (type(e).__name__, str(e)[:1200]) + check("%d 个条目全部装载完毕" % len(weight_map), ok, err) + + # ------------------------------------------------------- 5. 首次 forward + if a.no_forward: + print("\n== 5. 跳过（--no-forward）==") + else: + print("\n== 5. 首个 blob Linear 必须进 linear_gguf 并返回（判据 3：不静默回落稠密）==") + import torch + + def to_dev(t): + return infinicore.from_torch( + t.cuda(0) if a.device.startswith("cuda") else t) + + ids = to_dev(torch.tensor([[114, 5, 7]], dtype=torch.int32)) + # qwen3_5 是 mrope（position_id_axes=3），position_ids 的轴序在 C++ 侧 + # 只要求最后一维是 seq，这里按 [axes, seq] / [seq] 两种形状各试一次， + # 目的是越过入参校验走到第一个 Linear —— 判据只看那里抛的是什么。 + cands = [ + to_dev(torch.tensor([[0, 1, 2], [0, 1, 2], [0, 1, 2]], dtype=torch.int32)), + to_dev(torch.tensor([[0, 1, 2]], dtype=torch.int32)), + ] + # RankWorker 会把工作线程里的异常换个文案再抛一次（python 侧只看到 + # “RankWorker is closing”），真实抛出只落在 spdlog 里。实测 spdlog 走的是 + # **stdout**（把 2 单独分流到文件后 “linear_gguf” 那条 [error] 仍留在 + # stdout），所以 fd 1、2 都得用 memfd 接住（沙箱里 /tmp 只读）。 + def open_cap(): + try: + return os.memfd_create("stage2_log") + except AttributeError: + return os.open(os.path.join(_HERE, ".stage2_log.tmp"), + os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600) + + libc = ctypes.CDLL(None) + caps = {fd: open_cap() for fd in (1, 2)} + saved = {fd: os.dup(fd) for fd in caps} + msgs = [] + try: + # 先把手头的正常输出推完再换管道：否则 step 4 的 PASS 还躺在 python + # 的块缓冲里，换完才被 flush，会打到 memfd 里而不在日志文件中。 + sys.stdout.flush() + sys.stderr.flush() + for fd, mem in caps.items(): + os.dup2(mem, fd) + for pos in cands: + try: + eng.forward_raw(input_ids=ids, position_ids=pos) + msgs.append("<没抛异常：blob 被当成稠密权重跑了！>") + break + except Exception as e: # noqa: BLE001 + msgs.append("%s: %s" % (type(e).__name__, + str(e).strip().splitlines()[0][:200])) + finally: + libc.fflush(None) # C++ 侧重定向到文件时是块缓冲，不冲读不到 + sys.stdout.flush() + sys.stderr.flush() + for fd, mem in caps.items(): + os.fsync(mem) + os.dup2(saved[fd], fd) + os.close(saved[fd]) + captured = "" + for mem in caps.values(): + os.lseek(mem, 0, os.SEEK_SET) + captured += os.read(mem, 1 << 20).decode("utf-8", "replace") + os.close(mem) + haystack = "\n".join(msgs) + "\n" + captured + line = next((ln for ln in captured.splitlines() if "linear_gguf" in ln), "") + m = BLOB_RE.search(line) + if not m: + check("首个 blob Linear 进入 linear_gguf 并返回（未回落稠密）", False, + "python: %s\n 日志尾部: %s" % (" | ".join(msgs), + captured[-600:])) + else: + M, N, K, tid, row_bytes = [int(m.group(i)) for i in range(2, 7)] + check("首个 blob Linear 进入 linear_gguf 并返回（未回落稠密）", True) + # 只留 linear_gguf 之后的部分：spdlog 前缀占掉大半行，按整行截断会把张量名切掉 + print(" %s" % line[line.find("linear_gguf"):].strip()) + bs, ts = bs_ts[tid] + # 阶段 3.3 前这里评的是“M <= 8”（当时的 decode 护栏）；现在 M 的唯一 + # 契约是“等于本次喂进去的 token 数”，大了小了都算错。 + check("契约行自洽：M=%d 等于 prompt token 数 %d 且 row_bytes=%d == (K/%d)*%d" + % (M, PROMPT_TOKENS, row_bytes, bs, ts), + M == PROMPT_TOKENS and row_bytes == (K // bs) * ts, + "type=%d (block_size, type_size)=(%d,%d)" % (tid, bs, ts)) + + print("\n== 结果：%d PASS / %d FAIL ==" % (_PASS, _FAIL)) + return 0 if _FAIL == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_routeb_stage3_check.py b/scripts/gguf_routeb_stage3_check.py new file mode 100644 index 000000000..d443c4bda --- /dev/null +++ b/scripts/gguf_routeb_stage3_check.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +""" +InfiniLM 路线 B —— 阶段 3 端到端验收（执行方案 §7.1 判据 4/5） + +拿 mini8 产物在**量化形态**下真跑一遍 generate，逐条判据独立可失败： + + 1. PagedKVCacheConfig + attention_backend="paged-attn" 的引擎能构造并加载 121 条目。 + （必须 paged：Qwen3NextCausalConv1D::forward 取 mamba_metadata 的三个 + optional.value()，而 forward_raw 的 python 签名不暴露 + mamba_*_state_indices —— 上游 API 缺口，与 GGUF 无关，见 §7.2 备注。） + 2. **prefill 正例**：prompt 长度 12（> kMaxDecodeM=8）的 generate 必须跑完。阶段 3.3 + 之前这里是必抛「超过 decode kernel 的上限」，现在反过来：抛就算 FAIL。 + 3. 日志里出现「首个 blob 前向 …」契约行，且 **M 等于 prompt 长度**（证明整个 + 批量一次进了 kernel、没被拆开也没回落），row_bytes 用 gguf-py 的 + (block_size, type_size) 独立重算相等。 + 4. 贪心（top_k=1 / temperature=0）两次同 prompt 结果逐字相同 —— 说明 kernel + 没有 NaN/不确定行为（数值对不对是阶段 4 的比对，这里不比数值）。 + 5. token id 落在词表内。 + 6. **decode 回归**：prompt 长度 4（<= kMaxDecodeM）仍走 gemv、仍跑完 —— 撤护栏不 + 许把已经能用的短 prompt 路径弄坏。 + 7. （--count-blob-calls）把自己在 gdb 下重跑一遍，用断点命中次数证明 + 「每一步、每个 blob 模块」都进了 kernel：命中数 == 步数 × blob 条目数。 + 这条与路径无关（gemv/prefill 都过同一个 infiniopLinearGguf），少了就是有 + blob 静默回落稠密，多了就是有别的稠密 Linear 被误开。 + +用法： + source /home/liuxd/InfiniLM/scripts/gguf_routeb_env.sh + /usr/bin/python3 scripts/gguf_routeb_stage3_check.py [--new-tokens 8] [--count-blob-calls] +退出码 0 = 全部 PASS。 +""" + +from __future__ import annotations + +import argparse +import ctypes +import os +import re +import subprocess +import sys +import tempfile + +_HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join( + os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py")) + +DEFAULT_MODEL = "/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8" +BLOB_RE = re.compile( + r"linear_gguf: 首个 blob 前向 (\S+) — M=(\d+) N=(\d+) K=(\d+) " + r"ggml_type=(\d+) row_bytes=(\d+)") +MAX_DECODE_M = 8 # kMaxDecodeM：<=8 走 gemv，>8 走 prefill（两条路径同一个谓词） +PREFILL_M = 12 # > MAX_DECODE_M：阶段 3.3 的 prefill 正例（旧行为是必抛） +DECODE_M = 4 # <= MAX_DECODE_M：decode 回归用例 + +_PASS = 0 +_FAIL = 0 + + +def check(name, ok, detail=""): + global _PASS, _FAIL + if ok: + _PASS += 1 + print(" PASS %s" % name) + else: + _FAIL += 1 + print(" FAIL %s%s" % (name, ("\n %s" % detail) if detail else "")) + return ok + + +# --------------------------------------------------------------- C++ 日志捕获 +def _open_cap(): + try: + return os.memfd_create("stage3_log") + except AttributeError: + path = os.path.join(tempfile.gettempdir(), ".stage3_log.tmp") + try: + return os.open(path, os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600) + except OSError: + return os.open(os.path.join(_HERE, ".stage3_log.tmp"), + os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600) + + +class capture: + """把 fd 1/2 换到内存文件，用于读 spdlog 的输出。 + + RankWorker 会把工作线程里的异常换个文案再抛一次（python 侧只看到 + “RankWorker …”），真实抛出点只落在 spdlog 里；实测 spdlog 走 stdout， + 所以 1、2 两个 fd 都得接。 + """ + + def __enter__(self): + sys.stdout.flush() + sys.stderr.flush() + libc = ctypes.CDLL(None) + self._libc = libc + self.captured = "" + self._caps = {fd: _open_cap() for fd in (1, 2)} + self._saved = {fd: os.dup(fd) for fd in self._caps} + for fd, mem in self._caps.items(): + os.dup2(mem, fd) + return self + + def __exit__(self, *exc): + self._libc.fflush(None) # C++ 侧块缓冲，不冲就读不到 + sys.stdout.flush() + sys.stderr.flush() + for fd, mem in self._caps.items(): + try: + os.fsync(mem) + except OSError: + pass + os.lseek(mem, 0, os.SEEK_SET) + self.captured += os.read(mem, 1 << 22).decode("utf-8", "replace") + os.dup2(self._saved[fd], fd) + os.close(self._saved[fd]) + os.close(mem) + return False + + +# ------------------------------------------------------------------ gdb 计数 +def count_blob_calls(inner_argv): + """在 gdb 下重跑本脚本（inner_argv 已带 --route-b-inner），读断点命中次数。""" + script = os.path.join(tempfile.gettempdir(), "stage3_count.gdb") + try: + with open(script, "w") as f: + f.write("set pagination off\nset confirm off\n" + "set breakpoint pending on\n" + "break infiniopLinearGguf\ncommands\nsilent\ncontinue\nend\n" + "run\nprintf \"\\n===BPSTAT===\\n\"\ninfo breakpoints\n") + except OSError: + script = os.path.join(_HERE, ".stage3_count.gdb") + with open(script, "w") as f: + f.write("set pagination off\nset confirm off\n" + "set breakpoint pending on\n" + "break infiniopLinearGguf\ncommands\nsilent\ncontinue\nend\n" + "run\nprintf \"\\n===BPSTAT===\\n\"\ninfo breakpoints\n") + cmd = ["gdb", "-q", "-batch", "-x", script, "--args", sys.executable] + inner_argv + print(" -> %s" % " ".join(cmd[:8])) + p = subprocess.run(cmd, capture_output=True, text=True) + tail = p.stdout + p.stderr + m = re.search(r"breakpoint already hit (\d+) times", tail) + return int(m.group(1)) if m else None, tail + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model-path", default=DEFAULT_MODEL) + ap.add_argument("--new-tokens", type=int, default=8) + ap.add_argument("--num-blocks", type=int, default=16) + ap.add_argument("--block-size", type=int, default=256) + ap.add_argument("--count-blob-calls", action="store_true") + ap.add_argument("--route-b-inner", action="store_true", + help="内部用：gdb 子进程模式，只做前 6 条判据") + a, _unknown = ap.parse_known_args() + + import infinicore + from infinilm.cache import PagedKVCacheConfig + from infinilm.distributed import DistConfig + from infinilm.infer_engine import GenerationConfig, InferEngine + from infinilm.modeling_utils import load_model_state_dict_by_file + from gguf.constants import GGML_QUANT_SIZES + import json + + with open(os.path.join(a.model_path, "config.json")) as f: + cfg = json.load(f) + table = (cfg.get("quantization_config") or {}).get("ggml_types") or {} + n_blob = sum(1 for v in table.values() if isinstance(v, int)) + text_cfg = cfg.get("text_config") if isinstance(cfg.get("text_config"), dict) else cfg + vocab = int(text_cfg.get("vocab_size") or 0) + + def build(): + return InferEngine( + model_path=a.model_path, + device=infinicore.device("cuda:0"), + distributed_config=DistConfig(1), + cache_config=PagedKVCacheConfig(a.num_blocks, a.block_size, + max_batch_size=1), + attention_backend="paged-attn", + ) + + # ------------------------------------------------------------- 1. 构造加载 + print("\n== 1. paged 引擎构造 + 加载（blob %d / 稠密 %d）==" % (n_blob, + len(table) - n_blob)) + try: + eng = build() + ok, err = True, "" + except Exception as e: # noqa: BLE001 + ok, err = False, "%s: %s" % (type(e).__name__, str(e)[:1200]) + check("构造通过（PagedKVCacheConfig + paged-attn）", ok, err) + if not ok: + return 1 + check("has_mamba_cache 且 enable_paged_attn（GDN 模型只能走这条路）", + eng.has_mamba_cache and eng.enable_paged_attn) + try: + load_model_state_dict_by_file(eng, a.model_path, dtype=eng.dtype) + ok, err = True, "" + except Exception as e: # noqa: BLE001 + ok, err = False, "%s: %s" % (type(e).__name__, str(e)[:1200]) + check("权重装载完毕", ok, err) + if not ok: + return 1 + + def do_generate(tokens): + ids = infinicore.from_list([tokens], dtype=infinicore.int64) + out = eng.generate(ids, GenerationConfig( + max_new_tokens=a.new_tokens, temperature=0.0, top_k=1, top_p=1.0, + eos_token_id=None, stop_on_eos=False)) + return [int(x.to_numpy().reshape(-1)[0]) for x in out] + + # 成功的 generate 次数；每完成一次 = 1 次 prefill + (new_tokens-1) 次 decode + # = new_tokens 个前向步，每步每个 blob 各进 kernel 一次（判据 7 的期望值）。 + done_generates = 0 + + def one_generate(tokens): + nonlocal done_generates + toks = do_generate(tokens) + done_generates += 1 + return toks + + # ------------------------------------- 2/3/4/5. prefill 正例（prompt > decode 上限） + print("\n== 2-5. prefill：prompt=%d token（> kMaxDecodeM=%d）==" % (PREFILL_M, MAX_DECODE_M)) + pre_prompt = list(range(100, 100 + PREFILL_M)) + with capture() as cap: + try: + toks1 = one_generate(pre_prompt) + perr = "" + except BaseException as e: # noqa: BLE001 + toks1, perr = None, "%s: %s" % ( + type(e).__name__, str(e).strip().splitlines()[:1]) + log1 = cap.captured + check("prefill generate 走完 %d 步（M=%d 不再抛）" % (a.new_tokens, PREFILL_M), + toks1 is not None, perr + "\n 日志尾部: " + log1[-500:]) + if toks1 is None: + print("\n== 结果：%d PASS / %d FAIL ==" % (_PASS, _FAIL)) + return 1 + print(" tokens=%s" % toks1) + check("token id 落在词表 [0,%d) 内" % vocab, + not vocab or all(0 <= t < vocab for t in toks1)) + + toks2 = one_generate(pre_prompt) + check("贪心两次结果逐字相同", toks1 == toks2, "%s vs %s" % (toks1, toks2)) + + m = BLOB_RE.search(log1) + check("日志出现 blob 前向契约行（= blob 没被当稠密权重跑）", bool(m), + "捕获 %d 字节，未见 linear_gguf 行" % len(log1)) + if m: + key, M, N, K, tid, row_bytes = m.group(1), *[int(m.group(i)) for i in + range(2, 7)] + print(" %s — M=%d N=%d K=%d ggml_type=%d row_bytes=%d" + % (key, M, N, K, tid, row_bytes)) + bs, ts = GGML_QUANT_SIZES[tid] + # 契约行是进 kernel 的第一个 blob，而第一个 blob 就在 prompt 的 prefill 里。 + # M 必须等于 prompt 长度：小了就是上层把 prompt 拆碎了/没走 prefill。 + check("契约行 M=%d 等于 prompt 长度 %d（整批进 kernel）" % (M, PREFILL_M), + M == PREFILL_M, "M=%d" % M) + check("该批只能由 prefill 路径处理（M=%d > kMaxDecodeM=%d）" % (M, MAX_DECODE_M), + M > MAX_DECODE_M, "M=%d" % M) + check("契约行 row_bytes == (K/%d)*%d 自洽" % (bs, ts), + row_bytes == (K // int(bs)) * int(ts), + "row_bytes=%d 期望=%d" % (row_bytes, (K // int(bs)) * int(ts))) + + # ------------------------------------------- 6. decode 回归（短 prompt 仍可用） + print("\n== 6. decode 回归：prompt=%d token（<= %d，仍走 gemv）==" + % (DECODE_M, MAX_DECODE_M)) + dec_prompt = list(range(300, 300 + DECODE_M)) + try: + toks3 = one_generate(dec_prompt) + err3 = "" + except BaseException as e: # noqa: BLE001 + toks3, err3 = None, "%s: %s" % (type(e).__name__, str(e).strip()[:200]) + check("短 prompt 用例走完 %d 步（撤护栏未弄坏 gemv 路径）" % a.new_tokens, + toks3 is not None, err3) + if toks3 is not None: + print(" tokens=%s" % toks3) + + # --------------------------------------------------- 7. 断点命中数（可选） + if a.count_blob_calls and not a.route_b_inner: + print("\n== 7. gdb 断点计数：每步 × 每个 blob ==") + inner = [os.path.abspath(sys.argv[0])] + \ + [x for x in sys.argv[1:] if x != "--count-blob-calls"] + \ + ["--route-b-inner"] + n, tail = count_blob_calls(inner) + steps = re.search(r"INNER_STEPS=(\d+)", tail) + steps = int(steps.group(1)) if steps else None + expect = steps * n_blob if steps else None + check("infiniopLinearGguf 命中 %s 次 == 步数 %s × blob %d = %s" + % (n, steps, n_blob, expect), n is not None and n == expect, + "实际 %s / 期望 %s\n 子进程输出尾部: %s" + % (n, expect, tail[-600:])) + elif a.route_b_inner: + # 子进程里：把实际完成的前向步数报给外层。每次 generate = 1 次 prefill + + # (max_new_tokens-1) 次 decode = max_new_tokens 步；本脚本一共跑 3 次。 + print("INNER_STEPS=%d" % (done_generates * a.new_tokens)) + + print("\n== 结果：%d PASS / %d FAIL ==" % (_PASS, _FAIL)) + return 0 if _FAIL == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_routeb_tokenizer_check.py b/scripts/gguf_routeb_tokenizer_check.py new file mode 100755 index 000000000..1b3940edb --- /dev/null +++ b/scripts/gguf_routeb_tokenizer_check.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Build canonical input IDs with llama.cpp and compare the packaged tokenizer.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request + + +def load_cases(path: str, selected: set[str]) -> list[dict]: + cases = [] + with open(path, encoding="utf-8") as f: + for line in f: + if line.strip(): + item = json.loads(line) + if not selected or item["id"] in selected: + cases.append(item) + missing = selected - {x["id"] for x in cases} + if missing: + raise ValueError("unknown case ids: %s" % sorted(missing)) + return cases + + +def post_json(url: str, body: dict, timeout: int = 30) -> dict: + request = urllib.request.Request( + url, + data=json.dumps(body, ensure_ascii=False).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.load(response) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", "replace") + raise RuntimeError("HTTP %d: %s" % (exc.code, detail[:1000])) from exc + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--model-path", required=True) + ap.add_argument("--prompts", required=True) + ap.add_argument("--server", default="http://127.0.0.1:18080") + ap.add_argument("--case-ids", default="") + ap.add_argument("--out", required=True) + args = ap.parse_args() + + selected = {x for x in args.case_ids.split(",") if x} + cases = load_cases(args.prompts, selected) + + from transformers import AutoTokenizer + + common = {"local_files_only": True, "trust_remote_code": False} + tok_default = AutoTokenizer.from_pretrained(args.model_path, **common) + try: + tok_fixed = AutoTokenizer.from_pretrained( + args.model_path, fix_mistral_regex=True, **common) + fixed_error = None + except Exception as exc: # compatibility with older transformers + tok_fixed = None + fixed_error = "%s: %s" % (type(exc).__name__, exc) + + results = [] + default_ok = fixed_ok = True + for case in cases: + llama = post_json(args.server.rstrip("/") + "/tokenize", { + "content": case["prompt"], + "add_special": False, + "parse_special": True, + "with_pieces": False, + })["tokens"] + llama = [int(x) for x in llama] + local_default = [int(x) for x in tok_default.encode( + case["prompt"], add_special_tokens=False)] + local_fixed = None if tok_fixed is None else [int(x) for x in tok_fixed.encode( + case["prompt"], add_special_tokens=False)] + match_default = llama == local_default + match_fixed = local_fixed is not None and llama == local_fixed + default_ok &= match_default + fixed_ok &= match_fixed + results.append({ + **case, + "input_ids": llama, + "local_default_ids": local_default, + "local_fixed_ids": local_fixed, + "default_match": match_default, + "fixed_match": match_fixed, + }) + print("%-10s llama=%3d default=%s fixed=%s" % ( + case["id"], len(llama), match_default, + "NA" if local_fixed is None else str(match_fixed))) + + if default_ok: + selected_variant = "default" + elif fixed_ok: + selected_variant = "fix_mistral_regex=True" + else: + selected_variant = None + + output = { + "model_path": os.path.abspath(args.model_path), + "server": args.server, + "add_special": False, + "parse_special": True, + "selected_local_variant": selected_variant, + "default_all_match": default_ok, + "fixed_all_match": fixed_ok, + "fixed_load_error": fixed_error, + "cases": results, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w", encoding="utf-8") as f: + json.dump(output, f, ensure_ascii=False, indent=2) + print("RESULT default_all=%s fixed_all=%s selected=%s cases=%d" % ( + default_ok, fixed_ok, selected_variant, len(results))) + return 0 if selected_variant else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_routeb_typecensus.py b/scripts/gguf_routeb_typecensus.py new file mode 100644 index 000000000..5e496acd9 --- /dev/null +++ b/scripts/gguf_routeb_typecensus.py @@ -0,0 +1,55 @@ +import sys +import collections + +sys.path.insert(0, "/home/liuxd/llama.cpp/gguf-py") +from gguf import GGUFReader # noqa: E402 +from gguf.constants import GGMLQuantizationType as QType # noqa: E402 + +GGUF = "/home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf" +r = GGUFReader(GGUF) +T = {t.name: t for t in r.tensors} + + +def tn(name): + return QType(int(T[name].tensor_type)).name + + +per = collections.defaultdict(collections.Counter) +for i in range(64): + full = (i + 1) % 4 == 0 + names = ([f"blk.{i}.attn_q.weight", f"blk.{i}.attn_k.weight", + f"blk.{i}.attn_v.weight", f"blk.{i}.attn_output.weight"] if full + else [f"blk.{i}.attn_qkv.weight", f"blk.{i}.attn_gate.weight", + f"blk.{i}.ssm_out.weight"]) + names += [f"blk.{i}.ffn_gate.weight", f"blk.{i}.ffn_up.weight", f"blk.{i}.ffn_down.weight"] + for n in names: + per[n.split(".")[2]][tn(n)] += 1 + +print("=== 按张量角色的类型分布（64 层）===") +for k, v in sorted(per.items()): + print(f" {k:14s}", dict(v)) + +print("=== full-attn 层内 q/k/v 类型是否一致（决定融合 blob 能否共用一块 buffer）===") +bad = [(i, [tn(f"blk.{i}.attn_{x}.weight") for x in ("q", "k", "v")]) + for i in range(3, 64, 4)] +bad = [b for b in bad if len(set(b[1])) != 1] +print(f" 不一致层数 = {len(bad)} 样例 = {bad[:6]}") + +print("=== ffn gate/up 类型是否一致（决定 GateUp 融合 blob 能否共用一块 buffer）===") +bad2 = [(i, [tn(f"blk.{i}.ffn_{x}.weight") for x in ("gate", "up")]) for i in range(64)] +bad2 = [b for b in bad2 if len(set(b[1])) != 1] +print(f" 不一致层数 = {len(bad2)} 样例 = {bad2[:6]}") + +print("=== GDN 层 attn_qkv / attn_gate / ssm_out 抽样类型 ===") +for i in (0, 1, 2, 4, 62): + print(" ", i, {s: tn(f"blk.{i}.{s}.weight") for s in + ("attn_qkv", "attn_gate", "ssm_out", "ffn_gate", "ffn_down")}) + +print("=== 每个 Linear 的 (角色 -> 类型) 逐层矩阵，看同一角色跨层是否稳定 ===") +for role in ("attn_q", "attn_k", "attn_v", "attn_output", "ffn_gate", "ffn_up", "ffn_down"): + c = collections.Counter() + for i in range(64): + n = f"blk.{i}.{role}.weight" + if n in T: + c[tn(n)] += 1 + print(f" {role:12s}", dict(c)) diff --git a/scripts/gguf_to_infinilm.py b/scripts/gguf_to_infinilm.py new file mode 100644 index 000000000..6857b8ff9 --- /dev/null +++ b/scripts/gguf_to_infinilm.py @@ -0,0 +1,733 @@ +#!/usr/bin/env python3 +""" +InfiniLM 路线 B —— 阶段 1 打包器：GGUF -> InfiniLM 原生量化产物（执行方案 §5） + + Qwen3.8-27B-UD-Q6_K.gguf -> models/Qwen3.8-27B-GGUF-native/ + config.json + model-0000N-of-0000M.safetensors + index + +铁律（阶段 0 的教训写在这里，别再用第二套定义）： + * 键名 / shape / 哪些走 blob / 哪些稠密化，全部来自 `gguf_mapping.build_plan(REAL)` + + `apply_v1_exceptions()`。本文件**不得**出现第二张表。 + * 反量化一律调 `gguf.quants.dequantize`，禁止自己实现解码。 + * 置换只沿 dim0 整行/整元素搬，块内字节绝不动（§2.7 已证明字节级可行）。 + * 取向：gguf-py 的 `tensor.data` 已经是 [out, in]（量化张量是 [out, row_bytes]）， + 与 InfiniLM 参数同序 ⇒ 全程不转置数据。 + +用法： + source /home/liuxd/InfiniLM/scripts/gguf_routeb_env.sh + python3 scripts/gguf_to_infinilm.py [--dry-run] [--layers 4] [--verify all] +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sys +import time +from math import prod + +_HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, _HERE) +sys.path.insert(0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py")) + +import numpy as np # noqa: E402 + +import gguf_mapping as M # noqa: E402 +import gguf_transforms as X # noqa: E402 +from gguf import GGUFReader # noqa: E402 +from gguf.constants import GGML_QUANT_SIZES, GGMLQuantizationType as Q # noqa: E402 +from gguf.quants import dequantize # noqa: E402 + +DEFAULT_GGUF = "/home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf" +DEFAULT_OUT = "/home/liuxd/models/Qwen3.8-27B-GGUF-native" +DEFAULT_TOKENIZER = "/home/liuxd/models/Qwen3.8-27B-BF16" + +TYPE_NAME = {int(v.value): str(v.name) for v in Q} +TYPE_ID = {str(v.name): int(v.value) for v in Q} +UNQUANTIZED = ("F32", "F16", "BF16") + +# 分词器配置文件：词表本身从 GGUF 导出，这些附属文件优先从 --tokenizer-dir 复制。 +TOKENIZER_FILES = ("tokenizer_config.json", "chat_template.jinja", "generation_config.json", + "preprocessor_config.json", "video_preprocessor_config.json", + "special_tokens_map.json", "merges.txt", "vocab.json", "tokenizer.json") + +_GiB = 2 ** 30 + + +def log(msg: str) -> None: + print(msg, flush=True) + + +def blk_sizes(type_name: str) -> tuple[int, int]: + bs, ts = GGML_QUANT_SIZES[Q[type_name]] + return int(bs), int(ts) + + +# safetensors 报的是 GGML 风 dtype 名（BF16/U8），torch 报的是 bfloat16/uint8， +# 不归一就会把 947 个键全判成不符（实测踩过）。 +_DTYPE_ALIAS = {"BF16": "bfloat16", "F16": "float16", "F32": "float32", "U8": "uint8"} + + +def norm_dtype(s) -> str: + s = str(s) + return _DTYPE_ALIAS.get(s.upper() if s.isupper() else s, s.lower()) + + +# --------------------------------------------------------------------------- +# 源 -> 目标：单一实现 +# --------------------------------------------------------------------------- + +def dense_float32(src: np.ndarray, type_name: str, chunk_rows: int) -> np.ndarray: + """源张量的若干行 -> float32 [rows, in]。未量化类型只是换 dtype。""" + if type_name in UNQUANTIZED: + return np.asarray(src, dtype=np.float32) + if src.ndim != 2: + raise ValueError("量化源张量应是 [out, row_bytes]，实测 %s" % (src.shape,)) + rows = src.shape[0] + if rows == 0: + return np.zeros((0,), dtype=np.float32) + q = Q[type_name] + first = np.asarray(dequantize(src[:chunk_rows], q), dtype=np.float32) + if rows <= chunk_rows: + return first + # 预分配而不是 parts+concatenate：lm_head（248320×5120）峰值从 ~10 GB 降到 ~5 GB + out = np.empty((rows,) + first.shape[1:], dtype=np.float32) + out[:chunk_rows] = first + for i in range(chunk_rows, rows, chunk_rows): + out[i:i + chunk_rows] = np.asarray(dequantize(src[i:i + chunk_rows], q), + dtype=np.float32) + return out + + +def make_blob(e, t, dims, opt): + """逐字节路径：U8 [out, row_bytes]，只在需要时做整行置换。""" + bs, ts = blk_sizes(opt.types[t.name]) + n_out, n_in = int(e.shape[0]), int(e.shape[1]) + rb = M.row_bytes(n_in, bs, ts) + if int(t.data.shape[-1]) != rb: + raise ValueError("%s: 源行字节 %d != 映射表期望 %d" + % (e.gguf, int(t.data.shape[-1]), rb)) + arr = t.data + if e.slices: + s, ep = e.slices[0] + arr = arr[s:ep] + if int(arr.shape[0]) != n_out: + raise ValueError("%s: 取段后 %d 行 != 映射表 %d" % (e.gguf, arr.shape[0], n_out)) + if M.needs_vperm(e): + arr = X.apply_vperm(arr, e, dims, opt.vperm) + return torch_from(arr, np.uint8) + + +def entry_float32(e, t, dims, opt) -> np.ndarray: + """稠密化条目的 float32 值。单一实现：写盘（make_dense）与自检（比 BF16 位）共用。""" + tn = opt.types[t.name] + src = t.data + if e.slices: + s, ep = e.slices[0] + src = src[s:ep] + if tn in UNQUANTIZED: + arr = np.asarray(src, dtype=np.float32) + else: + arr = dense_float32(src, tn, opt.chunk_rows) + for tr in e.transforms: + if tr == M.T_ALOG: + arr = X.alog_from_ssm_a(arr) + elif tr in M.VPERM_TRANSFORMS: + arr = X.apply_vperm(arr, e, dims, opt.vperm) + elif tr in (M.T_DENSE, M.T_NONE): + continue + else: + raise ValueError("%s: 未知 transform %r" % (e.infinilm, tr)) + want = tuple(int(x) for x in e.shape) + if tuple(arr.shape) != want: + if arr.size != prod(want): + raise ValueError("%s: 变换后 shape %s != 映射表 %s" + % (e.infinilm, arr.shape, want)) + arr = arr.reshape(want) # §2.11 第 5 条：conv1d 补中间维 + return arr + + +def make_dense(e, t, dims, opt): + """稠密化路径：反量化 / 换 dtype -> float32 -> BF16（cast 交给 torch，不自实现）。""" + return torch_from(entry_float32(e, t, dims, opt), "bf16") + + +def torch_from(arr: np.ndarray, dtype): + import torch + t = torch.from_numpy(np.ascontiguousarray(arr)) + return t.to(torch.bfloat16) if dtype == "bf16" else t + + +def _is_baked_plus1_norm(name: str) -> bool: + """llama.cpp 转换时已对 norm.weight baked +1 的那些参数（conversion/qwen.py:394， + linear_attn.norm 除外）。与 modeling_utils 的 `_remap_qwen3_5` 加载期 +1 集合一一对应： + input/post_attention_layernorm、self_attn.q/k_norm、最终 model.norm 都以 'norm.weight' 结尾。""" + return name.endswith("norm.weight") and not name.endswith("linear_attn.norm.weight") + + +def build(e, t, dims, opt, dense_all: bool): + """一条映射条目 -> 一个待写盘的张量 + 名称。dense_all 用于 --emit-dense-ref。""" + e2 = e + if dense_all and e.blob: + e2 = _as_dense(e) + tens = make_blob(e2, t, dims, opt) if e2.blob else make_dense(e2, t, dims, opt) + # dense-ref 的 ssm_out 列序必须从 GGUF 的 tiled 换成 grouped，否则与 blob 路径（运行时 gather）语义不同， + # §8.3 的逐层 cos_sim 对拍就失去意义。稠密 BF16 可以随便换列（不像 blob 跨块要重量化），所以这里直接 permute。 + # vperm=none 时 blob 路径不做运行时 gather，denseref 也必须保持 GGUF 原生列序，二者才同构。 + if dense_all and e.act_vperm and opt.vperm != "none": + n_k, r, hd = dims.lin_k_heads, dims.v_per_k, dims.lin_v_dim + out_dim, in_dim = int(e.shape[0]), int(e.shape[1]) + if in_dim != n_k * r * hd: + raise ValueError("%s: in_dim %d != num_k_heads*num_v_per_k*head_dim = %d，无法按头分块置换列" + % (e.infinilm, in_dim, n_k * r * hd)) + # [out, in] 解释为 [out, r, n_k, hd]（tiled 序）-> 对调 1,2 轴 -> [out, n_k, r, hd]（grouped 序）-> flatten + tens = tens.view(out_dim, r, n_k, hd).transpose(1, 2).contiguous().view(out_dim, in_dim) + # dense-ref 版删掉了 quantization_config（见主写盘处），框架按普通 HF 模型加载； + # python 侧 `_remap_qwen3_5`（modeling_utils L808）对**非 gguf** 模型会把 norm 权重 +1 + # （HF 存 delta、C++ 用完整权重的约定）。而 dense-ref 的 norm 值是从 GGUF 原样搬来的 + # **已 baked +1 的完整权重**，再 +1 就变成 2+w（实测使块输入翻倍、级联污染 §8.3）。 + # 故 dense-ref 预存 (w-1)，让加载期 +1 恰好还原成 w，与 blob 路径（gguf=True 不 +1）同构。 + # 集合与 modeling_utils:799 `norm_weight_suffixes` 一致：linear_attn.norm 除外。 + if dense_all and _is_baked_plus1_norm(e.infinilm): + tens = tens - 1 + name = M.ckpt_name(e2) + return name, tens + + +_DENSE_CACHE: dict = {} + + +def _as_dense(e): + """blob 条目的“同样内容但稠密化”视图（只给 dense-ref 用，不改原表）。""" + key = (e.infinilm, e.vperm) + v = _DENSE_CACHE.get(key) + if v is None: + tr = tuple(x for x in e.transforms if x != M.T_NONE) + (M.T_DENSE,) + v = M.Entry(e.infinilm, e.gguf, e.shape, False, tr, e.types, e.slices, e.vperm, + "dense-ref " + (e.note or "")) + _DENSE_CACHE[key] = v + return v + + +# --------------------------------------------------------------------------- +# 维度：从 GGUF 元数据推导，并与映射表的 REAL 对账 +# --------------------------------------------------------------------------- + +def _dec(x) -> float: + """float32 元数据归回十进制字面量（1e-6 而非 9.999999974752427e-07）， + 让 config.json 与 HF 原始 config 逐字符一致。7 位有效数字对 float32 无损。""" + return float("%.7g" % float(x)) + + +def dims_from_gguf(reader) -> M.Dims: + """元数据键名沿用 llama.cpp 标准写法，与审计脚本 E 节实测同一批键。""" + g = lambda suffix, idx=0: X.gguf_meta(reader, suffix)[idx] # noqa: E731 + n_layers = int(g("block_count")) - int(g("nextn_predict_layers")) + inner = int(g("ssm.inner_size")) + state = int(g("ssm.state_size")) + head_dim = int(g("attention.key_length")) + dim_cnt = int(g("rope.dimension_count")) + sec = [int(x) for x in X.gguf_meta(reader, "rope.dimension_sections")] + vocab = len(X.gguf_meta(reader, "tokenizer.ggml.tokens")) + return M.Dims( + hidden=int(g("embedding_length")), + n_q_heads=int(g("attention.head_count")), + n_kv_heads=int(g("attention.head_count_kv")), + head_dim=head_dim, + ffn=int(g("feed_forward_length")), + lin_k_heads=int(g("ssm.group_count")), + lin_v_heads=inner // state, + lin_k_dim=state, + lin_v_dim=state, + conv_kernel=int(g("ssm.conv_kernel")), + vocab=vocab, + n_layers=n_layers, + interval=int(g("full_attention_interval")), + mrope_section=tuple(sec[:3]), # 丢掉尾 0：§2.11 第 4 条 + rope_theta=_dec(g("rope.freq_base")), + partial_rotary_factor=_dec(dim_cnt / head_dim), + rms_norm_eps=_dec(g("attention.layer_norm_rms_epsilon")), + max_position_embeddings=int(g("context_length")), + ) + + +def check_dims(d: M.Dims) -> None: + """元数据推导必须与映射表钉死的 REAL 一致，否则说明换了模型还硬套表。 + + float 字段用相对容差：GGUF 存的是 float32，1e-6 读回来是 + 9.999999974e-07，按 == 比会误报（实测本文件的 rms_norm_eps 就撞在这上面）。 + """ + diff = [] + for f in _DIM_FIELDS: + got, want = getattr(d, f.name), getattr(M.REAL, f.name) + if isinstance(want, float) or isinstance(got, float): + if not np.isclose(float(got), float(want), rtol=1e-6, atol=1e-12): + diff.append("%s: %r != %r" % (f.name, got, want)) + elif got != want: + diff.append("%s: %r != %r" % (f.name, got, want)) + if diff: + raise SystemExit("GGUF 元数据推导出的维度与 gguf_mapping.REAL 不符：%s\n" + "=> 先按新模型实测重做阶段 0，不要改打包器来迁就。" % diff) + log(" rms_norm_eps：GGUF float32 %r -> config 写 HF 十进制 %r" + % (float(d.rms_norm_eps), M.REAL.rms_norm_eps)) + + +from dataclasses import fields as _dc_fields # noqa: E402 +_DIM_FIELDS = [f for f in _dc_fields(M.Dims) if f.name != "architectures"] + + +# --------------------------------------------------------------------------- +# 分片写出 +# --------------------------------------------------------------------------- + +class ShardWriter: + def __init__(self, out_dir: str, max_bytes: int): + self.dir, self.max = out_dir, max_bytes + self.buf: dict[str, object] = {} + self.buf_bytes = 0 + self.shards: list[str] = [] + self.weight_map: dict[str, str] = {} + self.total = 0 + + def add(self, name: str, tens) -> None: + nbytes = int(tens.numel()) * int(tens.element_size()) + if self.buf and self.buf_bytes + nbytes > self.max: + self.flush() + self.buf[name] = tens + self.buf_bytes += nbytes + self.total += nbytes + + def flush(self) -> None: + if not self.buf: + return + self.shards.append("__pending__") + idx = len(self.shards) + fname = "model-%05d.safetensors" % idx + from safetensors.torch import save_file + save_file(self.buf, os.path.join(self.dir, fname), metadata={"format": "pt"}) + for k in self.buf: + self.weight_map[k] = fname + log(" 写出 %s（%.2f GiB，%d 个张量）" % (fname, self.buf_bytes / _GiB, len(self.buf))) + self.shards[-1] = fname + self.buf, self.buf_bytes = {}, 0 + + def finish(self) -> None: + self.flush() + n = len(self.shards) + renamed = {} + for i, f in enumerate(self.shards, 1): + new = "model-%05d-of-%05d.safetensors" % (i, n) + if f != new: + os.rename(os.path.join(self.dir, f), os.path.join(self.dir, new)) + renamed[f] = new + self.weight_map = {k: renamed[v] for k, v in self.weight_map.items()} + with open(os.path.join(self.dir, "model.safetensors.index.json"), "w") as fp: + json.dump({"metadata": {"total_size": self.total}, + "weight_map": self.weight_map}, fp, indent=1, sort_keys=True) + log(" 分片 %d 个，合计 %.3f GiB" % (n, self.total / _GiB)) + + +# --------------------------------------------------------------------------- +# 自检 +# --------------------------------------------------------------------------- + +def rows_hash(a) -> str: + """把 [rows, cols] 字节阵的**行多重集**压成一个摘要（排序后逐行喂 hash）。 + + 用途：置换过的 blob 不能直接与源逐字节比（那等于拿置换代码自证）， + 但可以无条件断言“产物行集 == 源行集”（置换只是整行搬，不允许改字节）。 + """ + import hashlib + a = np.ascontiguousarray(a) + v = a.view(np.void(a.shape[1] * a.dtype.itemsize)).ravel() + h = hashlib.sha256() + for x in np.sort(v): + h.update(x.tobytes()) + return h.hexdigest()[:16] + + +def dense_bits_check(e, t, dims, opt, prod_t) -> bool: + """BF16 条目的位级校验：逐行块算期望值并与产物对应行块比，峰值内存有界。 + + 上一版直接 `bf16_bits(整块 float32)`，在 lm_head（12.7 亿元素）上把进程 OOM kill 掉了。 + V 头置换 / A_log 是跨行或逐元素语义，不能切块，但这类条目都很小，走全量路径。 + """ + import torch + if M.needs_vperm(e): + return bool(np.array_equal(prod_t.view(torch.uint16).numpy(), + X.bf16_bits(entry_float32(e, t, dims, opt)))) + src = np.asarray(t.data) + if e.slices: + s, ep = e.slices[0] + src = src[s:ep] + tn = opt.types[t.name] + tail = tuple(int(x) for x in e.shape[1:]) + per_row = prod(tail) if tail else 1 + rows = max(1, int(_BIG_ELEMS // per_row)) + n = int(e.shape[0]) + if src.shape[0] != n: + return False + for i in range(0, n, rows): + blk = src[i:i + rows] + arr = (np.asarray(blk, dtype=np.float32) if tn in UNQUANTIZED + else dense_float32(blk, tn, opt.chunk_rows)) + exp = X.bf16_bits(arr.reshape((blk.shape[0],) + tail)) + got = prod_t[i:i + rows].view(torch.uint16).numpy() + if not np.array_equal(got, exp): + return False + return True + + +_BIG_ELEMS = 64 * 1024 * 1024 # 切块阈值：一次最多算 64M 元素（float32 峰值 256 MB） + + +def verify(out_dir: str, plan, tensors, dims, opt, sample) -> int: + """重读产物：全量比键/shape/dtype，分类抽样比字节。返回 FAIL 数。""" + import torch + from safetensors import safe_open + log("\n== 自检：重读产物 ==") + bs_files = sorted(f for f in os.listdir(out_dir) + if f.endswith(".safetensors") and not f.startswith(".")) + with open(os.path.join(out_dir, "model.safetensors.index.json")) as fp: + index = json.load(fp) + got: dict[str, tuple] = {} + handles = {} + for f in bs_files: + h = safe_open(os.path.join(out_dir, f), framework="pt") + handles[f] = h + for k in h.keys(): + t = h.get_slice(k) + got[k] = (tuple(int(x) for x in t.get_shape()), norm_dtype(t.get_dtype())) + fails = 0 + want = {} + for e in plan: + name = M.ckpt_name(e) + if e.blob: + bs, ts = blk_sizes(opt.types[e.gguf]) + shape, dt = (int(e.shape[0]), M.row_bytes(int(e.shape[1]), bs, ts)), "uint8" + else: + shape, dt = tuple(int(x) for x in e.shape), "bfloat16" + want[name] = (shape, dt, e) + missing = sorted(set(want) - set(got)) + extra = sorted(set(got) - set(want)) + for label, keys in (("缺键", missing), ("多键", extra)): + if keys: + fails += 1 + log(" FAIL %s %d 个：%s" % (label, len(keys), keys[:6])) + else: + log(" PASS 无%s" % label) + bad = [k for k in set(want) & set(got) if want[k][:2] != got[k]] + if bad: + fails += 1 + log(" FAIL shape/dtype 不符 %d 个：%s" % (len(bad), + [(k, want[k][:2], got[k]) for k in sorted(bad)[:4]])) + else: + log(" PASS 全部 %d 个键的 shape+dtype 与映射表一致" % len(want)) + + # config.json 的类型表必须与产物张量名**双向逐字相等**：阶段 2 的 C++ 就是拿 + # 这些名字查表决定 blob / 稠密（方案 §6.0 纠正 2），两边对不上会在运行期变成 + # “查不到 key”，那比 shape 错更难查。 + with open(os.path.join(out_dir, "config.json")) as fp: + cfg = json.load(fp) + qcfg = cfg.get("quantization_config") or {} + table = qcfg.get("ggml_types") or {} + if qcfg.get("quant_method") != "gguf": + fails += 1 + log(" FAIL config.json 顶层 quantization_config.quant_method != 'gguf'（或在 text_config 里）") + elif qcfg.get("key_prefix") != M.PREFIX: + fails += 1 + log(" FAIL config.json 缺 key_prefix=%r（阶段 2 C++ 用它裁表 key）" % M.PREFIX) + else: + log(" PASS quantization_config 在顶层，key_prefix=%r" % M.PREFIX) + for label, keys in (("类型表缺键", sorted(set(got) - set(table))), + ("类型表多键", sorted(set(table) - set(got)))): + if keys: + fails += 1 + log(" FAIL %s %d 个：%s" % (label, len(keys), keys[:6])) + else: + log(" PASS 无%s（%d 个 key 与张量名逐字相等）" % (label, len(table))) + + # 分类抽样（按名排序取首个，可复现）：三种字节路径必须有各自的代表， + # 纯随机抽 3 个会全部落在“未置换 memcpy”上，那样根本测不到置换与切片。 + def sel(pred): + return sorted(k for k, (_, _, e) in want.items() if pred(e)) + cats = [("blob 未置换", sel(lambda e: e.blob and not e.slices and not M.needs_vperm(e))), + ("blob V 置换", sel(lambda e: e.blob and not e.slices and M.needs_vperm(e))), + ("blob 融合切片", sel(lambda e: e.blob and e.slices)), + ("bf16 反量化", sel(lambda e: not e.blob and not e.slices and not M.needs_vperm(e))), + ("bf16 置换+alog", sel(lambda e: not e.blob and M.needs_vperm(e))), + ("bf16 融合切片", sel(lambda e: not e.blob and e.slices))] + picks = [c[1][0] for c in cats if c[1]] + if sample == "all": + picks = sorted(k for k, (_, _, e) in want.items() if e.blob) + log(" 抽样 %d 个：%s" % (len(picks), "全部 blob" if sample == "all" else + " ".join("%s=%s" % (c, len(v)) for c, v in cats))) + for k in picks: + shape, dt, e = want[k] + prod_t = handles[index["weight_map"][k]].get_tensor(k) + src = np.asarray(tensors[e.gguf].data) + if e.slices: + s, ep = e.slices[0] + src = src[s:ep] + ref = build(e, tensors[e.gguf], dims, opt, False)[1] + checks = [] + if tuple(int(x) for x in prod_t.shape) != tuple(int(x) for x in ref.shape): + checks.append(("shape", False)) + elif e.blob: + p = prod_t.numpy() + checks.append(("与重建一致", bool(torch.equal(prod_t, ref)))) + if M.needs_vperm(e): + checks.append(("行集与源相同", rows_hash(p) == rows_hash(src))) + else: + checks.append(("与 GGUF 源逐字节", np.array_equal(p, src))) + else: + # BF16：拿 numpy 的 RNE 位模式比，相当于独立验一次 torch 的 cast + 读写往返 + checks.append(("BF16 位与 numpy RNE 一致", + dense_bits_check(e, tensors[e.gguf], dims, opt, prod_t))) + ok = all(v for _, v in checks) + fails += 0 if ok else 1 + log(" %s %-52s %-16s %s" % ("PASS" if ok else "FAIL", k, + str(tuple(int(x) for x in prod_t.shape)), + "，".join("%s=%s" % (n, "Y" if v else "N") + for n, v in checks))) + return fails + + +# --------------------------------------------------------------------------- +# main +# --------------------------------------------------------------------------- + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--gguf", default=DEFAULT_GGUF) + ap.add_argument("--out", default=DEFAULT_OUT) + ap.add_argument("--tokenizer-dir", default=DEFAULT_TOKENIZER) + ap.add_argument("--dense-iq", action=argparse.BooleanOptionalAction, default=True, + help="v1 把 5 个 IQ4_NL/IQ4_XS 稠密化（阶段 6 上了码本 kernel 后 --no-dense-iq）") + ap.add_argument("--dense-embed", action=argparse.BooleanOptionalAction, default=True, + help="v1 恒为 True；--no-dense-embed 需要阶段 6 的 embedding kernel") + ap.add_argument("--vperm", choices=("inv", "fwd", "none"), default="inv", + help="V 头 tiled->grouped 方向；阶段 4 A/B 用（§2.7）") + ap.add_argument("--emit-dense-ref", metavar="PATH", default=None, + help="额外产出一份全反量化 BF16 版（阶段 4 自洽基准，不部署）") + ap.add_argument("--max-shard-gib", type=float, default=4.0) + ap.add_argument("--chunk-rows", type=int, default=8192, + help="反量化分块行数，限制峰值内存") + ap.add_argument("--layers", type=int, default=None, + help="只打前 N 层，并同步把 config 的 num_hidden_layers 改成 N" + "（产物可直接被框架构造 + 加载，阶段 2/3 用小模型验收用）") + ap.add_argument("--verify", choices=("off", "sample", "all"), default="sample") + ap.add_argument("--skip-pack", action="store_true", + help="不重写 23 GiB 权重，只做分词器导出 + 自检（迭代自检逻辑用）") + ap.add_argument("--dry-run", action="store_true", + help="全量校验取向/shape/字节数，不写盘（稠密化条目也只算 shape）") + a = ap.parse_args() + + if not a.dense_embed: + raise SystemExit("--no-dense-embed 需要阶段 6 的 embedding / lm_head 原生 kernel，" + "v1 没有它们就只能稠密化（§2.4）") + + t0 = time.time() + log("读取 GGUF 元数据：%s" % a.gguf) + reader = GGUFReader(a.gguf) + tensors = {t.name: t for t in reader.tensors} + dims = dims_from_gguf(reader) + check_dims(dims) + log(" 维度与映射表 REAL 一致：%d 层，hidden=%d，vocab=%d" + % (dims.n_layers, dims.hidden, dims.vocab)) + + # --layers 必须在 check_dims **之后**覆盖：维度照旧逐项校 REAL（防止换模型后硬套本表）， + # 但 config 的 num_hidden_layers / layer_types 要跟着改，否则截断产物与 config + # 不自洽，框架构造 64 层却只拿到 N 层权重（旧版本里这条表现为“不可加载”）。 + if a.layers is not None: + if not 0 < a.layers < dims.n_layers: + raise SystemExit("--layers 必须在 (0, %d) 之间，实际 %d" + % (dims.n_layers, a.layers)) + log(" --layers %d：num_hidden_layers %d -> %d，产物可加载" + % (a.layers, dims.n_layers, a.layers)) + dims.n_layers = a.layers + + opt = type("Opt", (), {})() + opt.vperm, opt.chunk_rows = a.vperm, a.chunk_rows + opt.types = {n: TYPE_NAME[int(t.tensor_type)] for n, t in tensors.items()} + plan = M.build_plan(dims) + n_exc = M.apply_v1_exceptions(plan, opt.types, enabled=a.dense_iq) + log(" 映射条目 %d，v1 稠密化例外命中 %d 个 IQ4" % (len(plan), n_exc)) + blob = [e for e in plan if e.blob] + log(" blob %d 个 / 稠密化 %d 个 / 丢弃 MTP 前缀 %s" + % (len(blob), len(plan) - len(blob), M.DROP_PREFIXES)) + + if a.dry_run: + log("\n== dry-run：逐条目校验取向与字节数（不写盘、不反量化）==") + blob_bytes = dense_bytes = 0 + for e in plan: + t = tensors[e.gguf] + n_out = (e.slices[0][1] - e.slices[0][0]) if e.slices else int(e.shape[0]) + if e.blob: + bs, ts = blk_sizes(opt.types[e.gguf]) + rb = M.row_bytes(int(e.shape[1]), bs, ts) + if int(t.data.shape[-1]) != rb: + raise ValueError("%s: 源行字节 %d != 期望 %d" + % (e.gguf, int(t.data.shape[-1]), rb)) + if int(t.data.shape[0]) < n_out: + raise ValueError("%s: 源 %d 行 < 条目需 %d 行" + % (e.gguf, t.data.shape[0], n_out)) + blob_bytes += n_out * rb + else: + n = prod(tuple(int(x) for x in e.shape)) + if not e.slices and prod(int(x) for x in t.shape) != n: + raise ValueError("%s: 源元素数 %s != 条目 shape %s" + % (e.gguf, t.shape, tuple(e.shape))) + dense_bytes += n * 2 + log(" PASS %d 个条目取向/字节数自洽：blob %.3f GiB + 稠密化 BF16 %.3f GiB" + " = 产物应占 %.3f GiB" + % (len(plan), blob_bytes / _GiB, dense_bytes / _GiB, + (blob_bytes + dense_bytes) / _GiB)) + return 0 + + os.makedirs(a.out, exist_ok=True) + w = ShardWriter(a.out, int(a.max_shard_gib * _GiB)) + ggml_types = {} + for e in plan: + ggml_types[M.type_table_key(M.ckpt_name(e))] = \ + TYPE_ID[opt.types[e.gguf]] if e.blob else "dense_bf16" + if a.skip_pack: + with open(os.path.join(a.out, "model.safetensors.index.json")) as fp: + w.total = json.load(fp)["metadata"]["total_size"] + log("\n== --skip-pack：沿用已有权重，仅重写 config.json / 分词器与自检 ==") + else: + log("\n== 写出 %s ==" % a.out) + for i, e in enumerate(plan): + t = tensors[e.gguf] + name, tens = build(e, t, dims, opt, False) + w.add(name, tens) + if (i + 1) % 100 == 0: + log(" ... %d/%d 条目（%.1f s）" % (i + 1, len(plan), time.time() - t0)) + w.finish() + + # config.json 两条路都要写：activation_vperm 这类语义元数据只能在这里刷新， + # 留在 else 里会让 --skip-pack 沿用旧 config（为了几个键重打包 7.2 GiB 不值）。 + # 规则由映射表派生（M.activation_vperm_rules），C++ 照单执行，不在两边各抄一份。 + rules = M.activation_vperm_rules(dims, plan) + if a.vperm == "none": + # --vperm none = 全链路不做任何 V 头置换：in_proj 不重排、out_proj 不 gather、 + # denseref 不列置换。config 必须同步清空规则，否则 C++ 照旧 gather。 + rules = [] + cfg = M.make_root_config(dims, ggml_types, rules) + with open(os.path.join(a.out, "config.json"), "w") as fp: + json.dump(cfg, fp, indent=1, sort_keys=True) + log(" config.json：%d 个 ggml_types 键（quantization_config 在顶层）+ 激活 V 头置换规则 %d 条：%s" + % (len(ggml_types), len(rules), + " ".join("%s=%dx%dx%d" % (r["suffix"], r["num_k_heads"], r["num_v_per_k"], + r["head_dim"]) for r in rules) or "无")) + + fails = export_tokenizer(reader, a.out, a.tokenizer_dir, dims) + + if not a.skip_pack: + with open(os.path.join(a.out, "pack_report.json"), "w") as fp: + json.dump({"gguf": os.path.abspath(a.gguf), + # 张量 data 区之和 != 文件大小（后者含元数据与对齐填充）， + # 两者都记下来，免得日后拿这个数去对 stat 产生误会 + "gguf_file_bytes": os.path.getsize(os.path.abspath(a.gguf)), + "gguf_tensor_data_bytes": sum(int(t.n_bytes) for t in reader.tensors), + "n_gguf_tensors": len(tensors), + "v1_dense_iq": bool(a.dense_iq), "vperm": a.vperm, + "n_entries": len(plan), "n_blob": len(blob), + "n_v1_exceptions": n_exc, + "blob_type_ids": sorted({TYPE_ID[opt.types[e.gguf]] for e in blob}), + "out_bytes": w.total, "shards": w.shards, + "seconds": round(time.time() - t0, 1)}, + fp, indent=1, sort_keys=True) + + if a.verify != "off": + fails += verify(a.out, plan, tensors, dims, opt, a.verify) + + if a.emit_dense_ref: + log("\n== 额外产出稠密基准版 %s ==" % a.emit_dense_ref) + os.makedirs(a.emit_dense_ref, exist_ok=True) + wr = ShardWriter(a.emit_dense_ref, int(a.max_shard_gib * _GiB)) + for e in plan: + name, tens = build(e, tensors[e.gguf], dims, opt, True) + wr.add(name, tens) + wr.finish() + ref_cfg = M.make_root_config(dims, {M.type_table_key(k): "dense_bf16" + for k in ggml_types}, rules) + # 稠密基准版不写 quantization_config：框架默认 NoneQuantization，C++ 里没有人 + # 执行置换。它的 ssm_out 列序在打包期已置换为 grouped（见 build() 里的 act_vperm 分支）， + # 与 blob 路径（运行时 gather）语义相同，可做逐层 cos_sim 对拍（§8.3）。 + del ref_cfg["quantization_config"] + with open(os.path.join(a.emit_dense_ref, "config.json"), "w") as fp: + json.dump(ref_cfg, fp, indent=1, sort_keys=True) + export_tokenizer(reader, a.emit_dense_ref, a.tokenizer_dir, dims) + + log("\n===== 完成：%.1f s，产物 %.3f GiB，自检 FAIL %d 处 =====" + % (time.time() - t0, w.total / _GiB, fails)) + return 1 if fails else 0 + + +def export_tokenizer(reader, out_dir: str, tokenizer_dir: str, dims) -> int: + """产物自带完整分词器。为什么不是简单 copy： + + 实测 `--tokenizer-dir`（models/Qwen3.8-27B-BF16）只有 vocab.json，**没有** + merges.txt / tokenizer.json，`AutoTokenizer.from_pretrained` 直接报 + "`vocab` and `merges` must be both be from memory or both filenames"。 + GGUF 内嵌完整 byte-level BPE（实测 248320 tokens / 247587 merges， + tokenizer.ggml.model=gpt2, pre=qwen35），词表与 embedding 行数同源，故以 GGUF 为准 + 写 vocab.json + merges.txt，其余配置文件从 tokenizer_dir 复制。 + """ + tokens = [str(t) for t in X.gguf_meta(reader, "tokenizer.ggml.tokens")] + merges = [str(m) for m in X.gguf_meta(reader, "tokenizer.ggml.merges")] + model = str(X.gguf_meta(reader, "tokenizer.ggml.model")[0]) + if len(tokens) != dims.vocab: + raise SystemExit("GGUF 词表 %d != config vocab_size %d，词表与 embedding 不同源" + % (len(tokens), dims.vocab)) + if model != "gpt2": + log(" 警告：tokenizer.ggml.model=%r 非 gpt2，vocab.json/merges.txt 写法需复核" % model) + with open(os.path.join(out_dir, "vocab.json"), "w", encoding="utf-8") as fp: + json.dump({t: i for i, t in enumerate(tokens)}, fp, ensure_ascii=False) + with open(os.path.join(out_dir, "merges.txt"), "w", encoding="utf-8") as fp: + fp.write("#version: 0.2\n" + "\n".join(merges) + "\n") + log(" 词表来自 GGUF：vocab %d / merges %d（model=%s）" % (len(tokens), len(merges), model)) + + have = os.listdir(tokenizer_dir) if tokenizer_dir and os.path.isdir(tokenizer_dir) else [] + if not have: + log(" 警告：分词器配置目录不存在：%s（只写了词表）" % tokenizer_dir) + copied = [] + for f in TOKENIZER_FILES: + dst = os.path.join(out_dir, f) + if f in have and not os.path.exists(dst): + shutil.copy2(os.path.join(tokenizer_dir, f), dst) + copied.append(f) + log(" 附属配置复制 %d 个：%s" % (len(copied), " ".join(sorted(copied)))) + if "tokenizer_config.json" not in copied + have: + raise SystemExit("产物缺 tokenizer_config.json：既没从 %s 复制到，也没导出兜底" + % tokenizer_dir) + return check_tokenizer(out_dir, dims) + + +def check_tokenizer(out_dir: str, dims) -> int: + """真装一次 AutoTokenizer 并做编解码往返（阶段 5 的前置条件，现在就能测）。""" + try: + from transformers import AutoTokenizer + except ImportError: + log(" SKIP 分词器自检：本环境无 transformers") + return 0 + try: + tk = AutoTokenizer.from_pretrained(out_dir) + n, cls = len(tk), type(tk).__name__ + s = "你好，世界 hello world 27B" + ids = tk.encode(s) + ok = n == dims.vocab and tk.decode(ids) == s + log(" %s 分词器 %s vocab=%d 往返=%s" % ("PASS" if ok else "FAIL", cls, n, + tk.decode(ids) == s)) + return 0 if ok else 1 + except Exception as exc: # noqa: BLE001 + log(" FAIL 分词器加载：%s: %s" % (type(exc).__name__, str(exc)[:200])) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gguf_transforms.py b/scripts/gguf_transforms.py new file mode 100644 index 000000000..b9f1c2bd8 --- /dev/null +++ b/scripts/gguf_transforms.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +""" +InfiniLM 路线 B —— 打包期变换（纯 numpy，不依赖 gguf-py / torch / InfiniCore）。 + +为什么单独一个文件：审计脚本 `gguf_routeb_audit.py` C 节要**证明**这些置换自等/可行， +打包器 `gguf_to_infinilm.py` 要**执行**同一份置换。两处各写一遍正是阶段 0 踩过坑 +（同一事实两份定义 -> 两套互相矛盾的预算数字），故这里只有一份实现。 + +置换方向约定（依据 llama.cpp conversion/qwen.py:571-605 与 §2.7）： + HF / InfiniLM 序 = grouped，索引 [k][v] + GGUF 序 = tiled ，索引 [v][k] （dst[v*n_k + k] = src[k*n_v_per_k + v]） + ⇒ llama.cpp 写入 = grouped -> tiled = reorder_v + ⇒ 本方案打包 = tiled -> grouped = reorder_v_inverse +方向本身仍属阶段 4 的 A/B 项（作用域已钉死，方向未闭环），故打包器暴露 +`--vperm {inv,fwd,none}` 三个取值，默认 inv。 +""" + +from __future__ import annotations + +import numpy as np + + +# --------------------------------------------------------------------------- +# V 头置换 +# --------------------------------------------------------------------------- + +def reorder_v(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: + """grouped -> tiled，与 llama.cpp `_reorder_v_heads` 同语义（沿 dim0 的整头/整元素置换）。 + + 支持任意尾部维度：1-D（A_log/dt_bias，hd=1）、2-D（权重行）、3-D（conv1d [C,1,K]）。 + """ + rest = t.shape[1:] + return (t.reshape((n_k, n_v_per_k, hd) + rest) + .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) + .reshape((n_k * n_v_per_k * hd,) + rest)) + + +def reorder_v_inverse(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: + """逆变换 = 两个轴参数对调后再调用一次。""" + rest = t.shape[1:] + return (t.reshape((n_v_per_k, n_k, hd) + rest) + .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) + .reshape((n_k * n_v_per_k * hd,) + rest)) + + +_VPERM = {"inv": reorder_v_inverse, "fwd": reorder_v, "none": None} + + +def vperm_head_dim(e, dims) -> int: + """一条映射条目里每个 value 头占多少元素。 + + `in_proj_a/b`、`A_log`、`dt_bias` 是 head_dim=1 的退化形式（每头一个标量）， + 其余（in_proj_v / in_proj_z / conv1d 的 V 段）是 lin_v_dim 个。判据用 shape 而不是 + 键名匹配，避免打包器里再写一张名字表。 + """ + n_heads = dims.lin_v_heads + rows = int(e.shape[0]) if e.vperm == "all" else dims.value_dim + if rows % n_heads: + raise ValueError("%s：作用域行数 %d 不能被 value 头数 %d 整除" + % (e.infinilm, rows, n_heads)) + return rows // n_heads + + +def apply_vperm(arr: np.ndarray, e, dims, direction: str = "inv") -> np.ndarray: + """按条目的作用域（all / v_tail）对 dim0 做 V 头置换。""" + fn = _VPERM[direction] + if fn is None: + return arr + n_k, hd = dims.lin_k_heads, vperm_head_dim(e, dims) + v_per_k = dims.lin_v_heads // n_k + if int(dims.lin_v_heads) % n_k: + raise ValueError("lin_v_heads %d 不能被 lin_k_heads %d 整除" + % (dims.lin_v_heads, n_k)) + if e.vperm == "v_tail": + n_v = n_k * v_per_k * hd + if arr.shape[0] < n_v: + raise ValueError("%s：dim0=%d 小于 value 段长度 %d" + % (e.infinilm, arr.shape[0], n_v)) + out = np.asarray(arr, dtype=arr.dtype) + return np.concatenate([out[:-n_v], fn(out[-n_v:], n_k, v_per_k, hd)], axis=0) + return fn(np.asarray(arr, dtype=arr.dtype), n_k, v_per_k, hd) + + +# --------------------------------------------------------------------------- +# 其它变换 +# --------------------------------------------------------------------------- + +def alog_from_ssm_a(a: np.ndarray) -> np.ndarray: + """A_log = log(-ssm_a)。 + + GGUF 存的是 `-exp(A_log)`（conversion/qwen.py:388），而 InfiniCore + fused_gated_delta_net_gating 自己算 -expf(A_log) ⇒ 它要 HF 约定。 + 实测本文件 48 个值全为负；出现非负值说明源不是这个约定，必须炸出来而不是静默 NaN。 + """ + a = np.asarray(a, dtype=np.float32) + if not np.all(a < 0): + raise ValueError("ssm_a 存在非负值（min=%g），无法取 log(-x)；" + "请核对 conversion/qwen.py 的 A_log 约定" % float(a.min())) + return np.log(-a) + + +def gguf_meta(reader, suffix: str): + """元数据键带架构前缀（qwen35.*），允许传短名；contents() 对单元素返回标量，统一成列表。""" + for key in ("qwen35.%s" % suffix, "general.%s" % suffix, suffix): + if key in reader.fields: + v = reader.fields[key].contents() + return v if isinstance(v, (list, tuple, np.ndarray)) else [v] + raise KeyError("GGUF 元数据缺少：%s（qwen35./general. 前缀均未命中）" % suffix) + + +def bf16_bits(x: np.ndarray) -> np.ndarray: + """float32 -> bfloat16 的位模式（uint16）。 + + 只做 round-to-nearest-even 的截断，与 torch 的 `.to(torch.bfloat16)` 等价； + 打包器实际写盘用 torch 做 cast，这里留给校验路径把 BF16 张量按位比回来。 + 全程 uint32 而不升 uint64：进位只丢失 bit32，不影响要取的 bit16..31， + 内存却减半（lm_head 这类亿级张量上 uint64 会直接 OOM）。 + """ + u = np.ascontiguousarray(x, dtype=np.float32).view(np.uint32) + bias = ((u >> np.uint32(16)) & np.uint32(1)) + np.uint32(0x7FFF) + return ((u + bias) >> np.uint32(16)).astype(np.uint16) From 563ca745f99b736e3f0d8617cffde3a81587509d Mon Sep 17 00:00:00 2001 From: xindongliu594 Date: Thu, 3 Sep 2026 20:32:03 +0800 Subject: [PATCH 2/5] chore: format GGUF Route B changes and fix benchmark setup --- csrc/engine/rank_worker.hpp | 2 +- .../layers/causal_lm_templates/text_model.hpp | 3 +- csrc/layers/linear/base_linear.cpp | 3 +- csrc/layers/linear/base_linear.hpp | 6 +- csrc/layers/linear/fused_linear.cpp | 40 +- .../layers/quantization/base_quantization.hpp | 2 +- csrc/layers/quantization/fp8.cpp | 17 +- csrc/layers/quantization/gguf.cpp | 98 ++-- csrc/layers/quantization/gguf.hpp | 2 +- csrc/layers/quantization/quantization.hpp | 2 +- csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp | 4 +- csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp | 10 +- .../qwen3_5/qwen3_5_fused_qkv_linear.cpp | 20 +- csrc/pybind11/engine/engine.hpp | 13 +- python/infinilm/modeling_utils.py | 33 +- scripts/gguf_mapping.py | 330 +++++++++--- scripts/gguf_routeb_audit.py | 485 ++++++++++++------ scripts/gguf_routeb_blocks_probe.cpp | 3 +- scripts/gguf_routeb_blocks_probe.cu | 27 +- scripts/gguf_routeb_blocks_ref.py | 356 +++++++++---- scripts/gguf_routeb_compare.py | 47 +- scripts/gguf_routeb_first_diff.py | 72 ++- scripts/gguf_routeb_first_diff_batch.py | 142 +++-- scripts/gguf_routeb_gemv_check.py | 181 +++++-- scripts/gguf_routeb_gemv_probe.cu | 18 +- scripts/gguf_routeb_head_precision.py | 69 ++- scripts/gguf_routeb_infinilm_ref.py | 45 +- scripts/gguf_routeb_infinilm_trace.py | 173 +++++-- scripts/gguf_routeb_llama_probe.py | 32 +- scripts/gguf_routeb_llama_ref.py | 52 +- scripts/gguf_routeb_llama_trace.py | 61 ++- scripts/gguf_routeb_probe_params.py | 1 + scripts/gguf_routeb_shape_contract.py | 231 ++++++--- scripts/gguf_routeb_stage2_check.py | 173 +++++-- scripts/gguf_routeb_stage3_check.py | 194 ++++--- scripts/gguf_routeb_tokenizer_check.py | 69 ++- scripts/gguf_routeb_typecensus.py | 52 +- scripts/gguf_to_infinilm.py | 450 +++++++++++----- scripts/gguf_transforms.py | 40 +- test/bench/backends/infinilm.py | 4 + 40 files changed, 2387 insertions(+), 1175 deletions(-) diff --git a/csrc/engine/rank_worker.hpp b/csrc/engine/rank_worker.hpp index d000331c9..6a5830c0f 100644 --- a/csrc/engine/rank_worker.hpp +++ b/csrc/engine/rank_worker.hpp @@ -10,8 +10,8 @@ #include "rank_barrier.hpp" #include -#include #include +#include #include #include #include diff --git a/csrc/layers/causal_lm_templates/text_model.hpp b/csrc/layers/causal_lm_templates/text_model.hpp index ffa0b70ac..a979b913c 100644 --- a/csrc/layers/causal_lm_templates/text_model.hpp +++ b/csrc/layers/causal_lm_templates/text_model.hpp @@ -235,8 +235,7 @@ class TextModel : public infinicore::nn::Module { if (dump_dir == nullptr || dump_dir[0] == '\0') { return; } - const char *dump_numel = - std::getenv("INFINILM_FINAL_PRENORM_DUMP_NUMEL"); + const char *dump_numel = std::getenv("INFINILM_FINAL_PRENORM_DUMP_NUMEL"); if (dump_numel != nullptr && dump_numel[0] != '\0' && hidden_states->numel() != std::strtoull(dump_numel, nullptr, 10)) { diff --git a/csrc/layers/linear/base_linear.cpp b/csrc/layers/linear/base_linear.cpp index 5c0ce0a8a..92cebfbdb 100644 --- a/csrc/layers/linear/base_linear.cpp +++ b/csrc/layers/linear/base_linear.cpp @@ -192,8 +192,7 @@ std::vector BaseLinear::init_fused_shards( desc.shape, desc.dtype, device_, desc.split_dim, 0, 1, 0); // key 里的 "shard." 前缀是量化类在 forward() 里还原拼接顺序的依据 this->register_parameter( - std::string(infinilm::quantization::GGUFBlockQuantization::SHARD_PREFIX) + - std::to_string(i) + "." + desc.name, + std::string(infinilm::quantization::GGUFBlockQuantization::SHARD_PREFIX) + std::to_string(i) + "." + desc.name, param); registered.push_back({sh.name + "." + desc.name, std::move(param)}); } diff --git a/csrc/layers/linear/base_linear.hpp b/csrc/layers/linear/base_linear.hpp index ad76110d2..ba1948200 100644 --- a/csrc/layers/linear/base_linear.hpp +++ b/csrc/layers/linear/base_linear.hpp @@ -57,9 +57,9 @@ class BaseLinear : public infinicore::nn::Module { // One shard of a fused linear, for schemes that cannot share a single fused // buffer (GGUF block quantization: row_bytes differs per shard type). struct FusedShard { - std::string name; // "q_proj" / "gate_proj" ... 注册到父模块时用 - size_t out_features; // 本 shard 的逻辑输出行数 - std::string stem; // "layers.0.self_attn.q_proj." 类型表查询用 + std::string name; // "q_proj" / "gate_proj" ... 注册到父模块时用 + size_t out_features; // 本 shard 的逻辑输出行数 + std::string stem; // "layers.0.self_attn.q_proj." 类型表查询用 }; // 为融合 Linear 逐 shard 各分配一块独立 buffer：本对象 parameters_ 里的 key 是 diff --git a/csrc/layers/linear/fused_linear.cpp b/csrc/layers/linear/fused_linear.cpp index b8c30ddaa..11aada775 100644 --- a/csrc/layers/linear/fused_linear.cpp +++ b/csrc/layers/linear/fused_linear.cpp @@ -32,16 +32,16 @@ QKVParallelLinear::QKVParallelLinear(size_t hidden_size, engine::distributed::RankInfo rank_info, const std::string &stem) : infinilm::nn::ColumnParallelLinear( - hidden_size, - calculate_out_feature_size(num_q_head, q_dim, num_k_head, k_dim, num_v_head, v_dim, rank_info), - quantization == nullptr ? std::make_shared() : quantization, - (q_bias || k_bias || v_bias), - dtype, - device, - rank_info.tp_rank, - rank_info.tp_size, - -1, - stem), + hidden_size, + calculate_out_feature_size(num_q_head, q_dim, num_k_head, k_dim, num_v_head, v_dim, rank_info), + quantization == nullptr ? std::make_shared() : quantization, + (q_bias || k_bias || v_bias), + dtype, + device, + rank_info.tp_rank, + rank_info.tp_size, + -1, + stem), q_dim_(q_dim), k_dim_(k_dim), v_dim_(v_dim), @@ -163,16 +163,16 @@ GateUpParallelLinear::GateUpParallelLinear(size_t hidden_size, size_t intermedia engine::distributed::RankInfo rank_info, const std::string &stem) : infinilm::nn::ColumnParallelLinear( - hidden_size, - intermediate_size * 2, - quantization == nullptr ? std::make_shared() : quantization, - gate_bias || up_bias, - dtype, - device, - rank_info.tp_rank, - rank_info.tp_size, - -1, - stem), + hidden_size, + intermediate_size * 2, + quantization == nullptr ? std::make_shared() : quantization, + gate_bias || up_bias, + dtype, + device, + rank_info.tp_rank, + rank_info.tp_size, + -1, + stem), gate_bias_(gate_bias), up_bias_(up_bias) { if (gate_bias_ != up_bias_) { diff --git a/csrc/layers/quantization/base_quantization.hpp b/csrc/layers/quantization/base_quantization.hpp index 8dc94823b..9bc70eed5 100644 --- a/csrc/layers/quantization/base_quantization.hpp +++ b/csrc/layers/quantization/base_quantization.hpp @@ -38,7 +38,7 @@ struct SplitParam { class BaseQuantization : public std::enable_shared_from_this { public: - explicit BaseQuantization(const nlohmann::json &quant_config) : quant_config_(quant_config){}; + explicit BaseQuantization(const nlohmann::json &quant_config) : quant_config_(quant_config) {}; virtual ~BaseQuantization() = default; const nlohmann::json &get_config() const { return quant_config_; } diff --git a/csrc/layers/quantization/fp8.cpp b/csrc/layers/quantization/fp8.cpp index 7d96513c6..475f5652e 100644 --- a/csrc/layers/quantization/fp8.cpp +++ b/csrc/layers/quantization/fp8.cpp @@ -20,20 +20,15 @@ std::vector FP8Quantization::get_param_layout( std::vector descs; // Weight: FP8 (E4M3) format - keep as F8, do NOT convert to BF16 - descs.push_back({"weight", {out_features, in_features}, - infinicore::DataType::F8, split_dim, tp_rank, tp_size}); + descs.push_back({"weight", {out_features, in_features}, infinicore::DataType::F8, split_dim, tp_rank, tp_size}); // Per-block weight scale (inverse): BF16, shape = [ceil(N/128), ceil(K/128)] size_t num_out_blocks = (out_features + BLOCK_SIZE - 1) / BLOCK_SIZE; size_t num_in_blocks = (in_features + BLOCK_SIZE - 1) / BLOCK_SIZE; - descs.push_back({"weight_scale_inv", {num_out_blocks, num_in_blocks}, - infinicore::DataType::F32, split_dim, tp_rank, tp_size}); + descs.push_back({"weight_scale_inv", {num_out_blocks, num_in_blocks}, infinicore::DataType::F32, split_dim, tp_rank, tp_size}); if (bias) { - descs.push_back({"bias", {out_features}, dtype, - split_dim >= 0 ? 0 : -1, - split_dim >= 0 ? tp_rank : 0, - split_dim >= 0 ? tp_size : 1}); + descs.push_back({"bias", {out_features}, dtype, split_dim >= 0 ? 0 : -1, split_dim >= 0 ? tp_rank : 0, split_dim >= 0 ? tp_size : 1}); } return descs; } @@ -67,7 +62,7 @@ infinicore::Tensor FP8Quantization::forward( // Get dimensions auto x_shape = x->shape(); size_t ndim = x_shape.size(); - size_t K = x_shape[ndim - 1]; // last dim is always feature dim + size_t K = x_shape[ndim - 1]; // last dim is always feature dim // M = product of all leading dims size_t M = 1; for (size_t i = 0; i < ndim - 1; i++) { @@ -162,9 +157,7 @@ std::shared_ptr FP8Quantization::process_weights_after_loading size_t num_in_blocks = (in_features + BLOCK_SIZE - 1) / BLOCK_SIZE; auto scale_shape = scale->shape(); - if (scale_shape.size() != 2 || - scale_shape[0] != num_out_blocks || - scale_shape[1] != num_in_blocks) { + if (scale_shape.size() != 2 || scale_shape[0] != num_out_blocks || scale_shape[1] != num_in_blocks) { throw std::runtime_error("FP8Quantization: weight_scale_inv shape mismatch"); } diff --git a/csrc/layers/quantization/gguf.cpp b/csrc/layers/quantization/gguf.cpp index 803593075..583a06856 100644 --- a/csrc/layers/quantization/gguf.cpp +++ b/csrc/layers/quantization/gguf.cpp @@ -105,8 +105,7 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) const std::string v = kv.value().get(); if (v != DENSE_MARK) { throw std::runtime_error( - "GGUFBlockQuantization: '" + name + "' 的取值 '" + v + "' 既不是整数 type id 也不是 \"" + - DENSE_MARK + "\""); + "GGUFBlockQuantization: '" + name + "' 的取值 '" + v + "' 既不是整数 type id 也不是 \"" + DENSE_MARK + "\""); } ++n_dense; } else { @@ -121,9 +120,7 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) } if (!ggml_block(id)) { throw std::runtime_error( - "GGUFBlockQuantization: '" + name + "' 是不支持的 ggml type id=" + - std::to_string(id) + "（当前支持 " + supported_types() + - "；其余类型必须在打包期稠密化，不能留到运行期猜）"); + "GGUFBlockQuantization: '" + name + "' 是不支持的 ggml type id=" + std::to_string(id) + "（当前支持 " + supported_types() + "；其余类型必须在打包期稠密化，不能留到运行期猜）"); } ++n_blob; } @@ -145,8 +142,7 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) { const auto &rules = quant_config_.at("activation_vperm"); if (!rules.is_array()) { - throw std::runtime_error("GGUFBlockQuantization: activation_vperm 必须是数组，实际是 " + - std::string(rules.type_name())); + throw std::runtime_error("GGUFBlockQuantization: activation_vperm 必须是数组，实际是 " + std::string(rules.type_name())); } for (const auto &j : rules) { if (!j.is_object()) { @@ -155,8 +151,7 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) ActVPerm r; for (const char *key : {"suffix", "num_k_heads", "num_v_per_k", "head_dim"}) { if (!j.contains(key)) { - throw std::runtime_error("GGUFBlockQuantization: activation_vperm 条目缺 '" + - std::string(key) + "'"); + throw std::runtime_error("GGUFBlockQuantization: activation_vperm 条目缺 '" + std::string(key) + "'"); } } r.suffix = j.at("suffix").get(); @@ -165,14 +160,11 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) r.hd = j.at("head_dim").get(); if (r.suffix.empty() || r.suffix.back() != '.' || !r.n_k || !r.r || !r.hd) { throw std::runtime_error( - "GGUFBlockQuantization: activation_vperm 条目不合法：suffix='" + r.suffix + - "' 需以 '.' 结尾，三个维度需为正（实际 " + std::to_string(r.n_k) + "/" + - std::to_string(r.r) + "/" + std::to_string(r.hd) + "）"); + "GGUFBlockQuantization: activation_vperm 条目不合法：suffix='" + r.suffix + "' 需以 '.' 结尾，三个维度需为正（实际 " + std::to_string(r.n_k) + "/" + std::to_string(r.r) + "/" + std::to_string(r.hd) + "）"); } if (std::any_of(vperm_.begin(), vperm_.end(), [&r](const ActVPerm &e) { return e.suffix == r.suffix; })) { - throw std::runtime_error("GGUFBlockQuantization: activation_vperm 里 '" + r.suffix + - "' 出现多次（同一条规则只能有一份）"); + throw std::runtime_error("GGUFBlockQuantization: activation_vperm 里 '" + r.suffix + "' 出现多次（同一条规则只能有一份）"); } vperm_.push_back(std::move(r)); } @@ -193,8 +185,7 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) if (!vs.empty()) { vs += ", "; } - vs += r.suffix + "=" + std::to_string(r.n_k) + "x" + std::to_string(r.r) + "x" + - std::to_string(r.hd); + vs += r.suffix + "=" + std::to_string(r.n_k) + "x" + std::to_string(r.r) + "x" + std::to_string(r.hd); } spdlog::info("GGUF block quantization: 激活 V 头置换规则 {} 条（grouped->tiled）：{}", vperm_.size(), vs.empty() ? "无" : vs); @@ -227,9 +218,7 @@ int64_t GGUFBlockQuantization::resolve(const std::string &stem, std::string *mat // 两种都必须是异常：任何「查不到就走稠密」的回落都会变成能加载、显存暴涨、结果错。 if (hits != 1) { throw std::runtime_error( - "GGUFBlockQuantization: stem '" + describe(stem) + "' 在类型表里命中 " + - std::to_string(hits) + " 个候选（期望恰好 1 个：'" + blob_key + "' 或 '" + dense_key + - "'）；表共 " + std::to_string(types_.size()) + " 条"); + "GGUFBlockQuantization: stem '" + describe(stem) + "' 在类型表里命中 " + std::to_string(hits) + " 个候选（期望恰好 1 个：'" + blob_key + "' 或 '" + dense_key + "'）；表共 " + std::to_string(types_.size()) + " 条"); } const auto &hit = blob_it != types_.end() ? *blob_it : *dense_it; if (matched_key) { @@ -249,13 +238,11 @@ size_t GGUFBlockQuantization::row_bytes(size_t in_features, int64_t type_id) con const GgmlBlock *b = ggml_block(type_id); if (!b) { throw std::runtime_error( - "GGUFBlockQuantization: 不支持的 ggml type id=" + std::to_string(type_id) + - "（当前支持 " + supported_types() + "）"); + "GGUFBlockQuantization: 不支持的 ggml type id=" + std::to_string(type_id) + "（当前支持 " + supported_types() + "）"); } if (in_features % b->block_size != 0) { throw std::runtime_error( - "GGUFBlockQuantization: in_features=" + std::to_string(in_features) + - " 不能被 " + b->name + " 的块大小 " + std::to_string(b->block_size) + " 整除"); + "GGUFBlockQuantization: in_features=" + std::to_string(in_features) + " 不能被 " + b->name + " 的块大小 " + std::to_string(b->block_size) + " 整除"); } return in_features / b->block_size * b->type_size; } @@ -263,8 +250,7 @@ size_t GGUFBlockQuantization::row_bytes(size_t in_features, int64_t type_id) con const GGUFBlockQuantization::ActVPerm *GGUFBlockQuantization::vperm_rule( const std::string &stem) const { for (const auto &r : vperm_) { - if (stem.size() >= r.suffix.size() && - stem.compare(stem.size() - r.suffix.size(), r.suffix.size(), r.suffix) == 0) { + if (stem.size() >= r.suffix.size() && stem.compare(stem.size() - r.suffix.size(), r.suffix.size(), r.suffix) == 0) { return &r; } } @@ -277,8 +263,7 @@ infinicore::Tensor GGUFBlockQuantization::gather_grouped_to_tiled( const size_t ndim = shape.size(); if (ndim < 2) { throw std::runtime_error( - "GGUFBlockQuantization: " + name + " 的激活 rank=" + std::to_string(ndim) + - "，至少要是 [..., in_features]"); + "GGUFBlockQuantization: " + name + " 的激活 rank=" + std::to_string(ndim) + "，至少要是 [..., in_features]"); } const size_t K = shape[ndim - 1]; const size_t want = rule.n_k * rule.r * rule.hd; @@ -286,9 +271,7 @@ infinicore::Tensor GGUFBlockQuantization::gather_grouped_to_tiled( // TP 会把 in 维切成没关头数不等的分片，套上整头置换就是静默错位； // 与 get_param_layout 里「暂不支持 tensor parallel」的护栏保持同一口径。 throw std::runtime_error( - "GGUFBlockQuantization: " + name + " 的激活末维 " + std::to_string(K) + - " != activation_vperm 的 num_k_heads*num_v_per_k*head_dim = " + std::to_string(want) + - "（切分后的分片不能套整头置换）"); + "GGUFBlockQuantization: " + name + " 的激活末维 " + std::to_string(K) + " != activation_vperm 的 num_k_heads*num_v_per_k*head_dim = " + std::to_string(want) + "（切分后的分片不能套整头置换）"); } // [..., n_k, r, hd] -> [..., r, n_k, hd]：把 grouped（k-major）的激活置换为 tiled（v-major）。 const size_t k_axis = ndim - 1; @@ -324,14 +307,11 @@ std::vector GGUFBlockQuantization::get_param_layout( if (stem.empty()) { throw std::runtime_error( - "GGUFBlockQuantization: 构造 Linear 时没有传 checkpoint stem（in=" + - std::to_string(in_features) + ", out=" + std::to_string(out_features) + - "）——方案 §6.1 列出的构造点必须全部补上 prefix/stem"); + "GGUFBlockQuantization: 构造 Linear 时没有传 checkpoint stem（in=" + std::to_string(in_features) + ", out=" + std::to_string(out_features) + "）——方案 §6.1 列出的构造点必须全部补上 prefix/stem"); } if (tp_size != 1 || tp_rank != 0) { throw std::runtime_error( - "GGUFBlockQuantization: 暂不支持 tensor parallel（blob 的 TP 切分留待多卡阶段）：" + - describe(stem)); + "GGUFBlockQuantization: 暂不支持 tensor parallel（blob 的 TP 切分留待多卡阶段）：" + describe(stem)); } if (bias) { throw std::runtime_error( @@ -343,8 +323,7 @@ std::vector GGUFBlockQuantization::get_param_layout( if (stem.back() != '.') { if (!has_group(stem)) { throw std::runtime_error( - "GGUFBlockQuantization: 融合组 stem '" + stem + - "' 在类型表里没有任何 '" + stem + "..*' 条目"); + "GGUFBlockQuantization: 融合组 stem '" + stem + "' 在类型表里没有任何 '" + stem + "..*' 条目"); } ++n_group_; return {}; @@ -378,11 +357,10 @@ infinicore::Tensor GGUFBlockQuantization::forward_shard( const std::string &table_key) const { if (suffix == DENSE_SUFFIX) { // 参数后缀是 get_param_layout 按 resolve() 结果选的，两者不一致 = 有地方改坏了 - //（blob 被当成 BF16 读就是「能加载、结果错」），宁可抛。 + // （blob 被当成 BF16 读就是「能加载、结果错」），宁可抛。 if (type_id != DENSE_BF16) { throw std::runtime_error( - "GGUFBlockQuantization: " + table_key + " 的参数后缀是 " + DENSE_SUFFIX + - "，但类型表给出的 ggml type id=" + std::to_string(type_id) + "（不一致）"); + "GGUFBlockQuantization: " + table_key + " 的参数后缀是 " + DENSE_SUFFIX + "，但类型表给出的 ggml type id=" + std::to_string(type_id) + "（不一致）"); } auto x = input->is_contiguous() ? input : input->contiguous(); auto w = weight->is_contiguous() ? weight : weight->contiguous(); @@ -393,9 +371,7 @@ infinicore::Tensor GGUFBlockQuantization::forward_shard( // 那等于把块字节当成 BF16 读，能跑完但结果是错的，宁可抛。 if (alpha != 1.0F) { throw std::runtime_error( - "linear_gguf: 不支持 alpha=" + std::to_string(alpha) + - "（GGUF blob 路径没有缩放权重，alpha!=1 说明上层期望与实现不符）：" + - table_key); + "linear_gguf: 不支持 alpha=" + std::to_string(alpha) + "（GGUF blob 路径没有缩放权重，alpha!=1 说明上层期望与实现不符）：" + table_key); } auto x = input->is_contiguous() ? input : input->contiguous(); auto w = weight->is_contiguous() ? weight : weight->contiguous(); @@ -416,11 +392,11 @@ infinicore::Tensor GGUFBlockQuantization::forward_shard( flat = flat->is_contiguous() ? flat : flat->contiguous(); const bool f32_decode_out = use_f32_decode_output(table_key, M); const auto out_dtype = f32_decode_out - ? infinicore::DataType::F32 - : input->dtype(); + ? infinicore::DataType::F32 + : input->dtype(); auto out = infinicore::Tensor::empty({M, N}, out_dtype, input->device()); // 只报第一个 blob 调用：端到端排障时区分「死在 blob 路径之前」与 - //「已在 kernel 里」，两者处置完全不同（前者是接线问题，后者是下游算子）。 + // 「已在 kernel 里」，两者处置完全不同（前者是接线问题，后者是下游算子）。 static std::atomic blob_calls{0}; if (blob_calls.fetch_add(1) == 0) { spdlog::info( @@ -442,8 +418,7 @@ infinicore::Tensor GGUFBlockQuantization::forward_shard( return out->view(out_shape); } throw std::runtime_error( - "GGUFBlockQuantization: " + table_key + " 的参数后缀 '" + suffix + - "' 既不是 " + DENSE_SUFFIX + " 也不是 " + BLOB_SUFFIX); + "GGUFBlockQuantization: " + table_key + " 的参数后缀 '" + suffix + "' 既不是 " + DENSE_SUFFIX + " 也不是 " + BLOB_SUFFIX); } infinicore::Tensor GGUFBlockQuantization::forward( @@ -478,7 +453,7 @@ infinicore::Tensor GGUFBlockQuantization::forward( throw std::runtime_error( "GGUFBlockQuantization: 融合组 '" + describe(stem) + "' 的 shard '" + describe(s) + "' 命中激活置换规则，但一根 input 同时服务于所有 shard，" - "无法按 shard 分别置换（实际产物里 out_proj 不是融合 Linear，走到这里=接线错）"); + "无法按 shard 分别置换（实际产物里 out_proj 不是融合 Linear，走到这里=接线错）"); } } } else if (rule) { @@ -496,9 +471,8 @@ infinicore::Tensor GGUFBlockQuantization::forward( if (shard_stems.empty()) { if (params.size() != 1) { throw std::runtime_error( - "GGUFBlockQuantization: " + describe(stem) + " 有 " + - std::to_string(params.size()) + " 个参数却没收到 shard_stems" - "（内部错误：BaseLinear::compute_linear 没有把 shard_stems_ 传下来）"); + "GGUFBlockQuantization: " + describe(stem) + " 有 " + std::to_string(params.size()) + " 个参数却没收到 shard_stems" + "（内部错误：BaseLinear::compute_linear 没有把 shard_stems_ 传下来）"); } const auto &kv = *params.begin(); std::string table_key; @@ -512,17 +486,14 @@ infinicore::Tensor GGUFBlockQuantization::forward( // 每个 shard 的 ggml 类型由 shard_stems[i] 查表（实测 q/k/v 不同类型，见 §7.2）。 if (shard_stems.size() != params.size()) { throw std::runtime_error( - "GGUFBlockQuantization: " + describe(stem) + " 有 " + - std::to_string(params.size()) + " 个 shard 参数但收到 " + - std::to_string(shard_stems.size()) + " 个 shard stem（内部错误：两者应在 " - "BaseLinear::init_fused_shards 的同一个循环里产生）"); + "GGUFBlockQuantization: " + describe(stem) + " 有 " + std::to_string(params.size()) + " 个 shard 参数但收到 " + std::to_string(shard_stems.size()) + " 个 shard stem（内部错误：两者应在 " + "BaseLinear::init_fused_shards 的同一个循环里产生）"); } std::vector> parts; for (const auto &kv : params) { if (kv.first.compare(0, std::string(SHARD_PREFIX).size(), SHARD_PREFIX) != 0) { throw std::runtime_error( - "GGUFBlockQuantization: 融合 Linear 的参数名 '" + kv.first + - "' 不是 " + SHARD_PREFIX + ". 形式（" + describe(stem) + "）"); + "GGUFBlockQuantization: 融合 Linear 的参数名 '" + kv.first + "' 不是 " + SHARD_PREFIX + ". 形式（" + describe(stem) + "）"); } const size_t dot = kv.first.find('.'); if (dot == std::string::npos) { @@ -533,8 +504,7 @@ infinicore::Tensor GGUFBlockQuantization::forward( dot - std::string(SHARD_PREFIX).size())); if (idx >= shard_stems.size()) { throw std::runtime_error( - "GGUFBlockQuantization: 参数名 '" + kv.first + "' 的 shard 下标越出 shard_stems（" + - describe(stem) + "）"); + "GGUFBlockQuantization: 参数名 '" + kv.first + "' 的 shard 下标越出 shard_stems（" + describe(stem) + "）"); } std::string table_key; const int64_t id = resolve(shard_stems[idx], &table_key); @@ -572,9 +542,7 @@ std::vector GGUFBlockQuantization::split_params( } if (result.size() != splits.size()) { throw std::runtime_error( - "GGUFBlockQuantization::split_params: " + std::to_string(splits.size()) + - " 个 shard 只匹配到 " + std::to_string(result.size()) + - " 个参数（GGUF 融合 Linear 应走 BaseLinear::init_fused_shards）"); + "GGUFBlockQuantization::split_params: " + std::to_string(splits.size()) + " 个 shard 只匹配到 " + std::to_string(result.size()) + " 个参数（GGUF 融合 Linear 应走 BaseLinear::init_fused_shards）"); } return result; } @@ -584,9 +552,7 @@ std::shared_ptr GGUFBlockQuantization::process_weights_after_l const infinicore::Device &, int) const { for (auto &kv : params) { - const bool is_blob = kv.first.size() >= strlen(BLOB_SUFFIX) && - kv.first.compare(kv.first.size() - strlen(BLOB_SUFFIX), - strlen(BLOB_SUFFIX), BLOB_SUFFIX) == 0; + const bool is_blob = kv.first.size() >= strlen(BLOB_SUFFIX) && kv.first.compare(kv.first.size() - strlen(BLOB_SUFFIX), strlen(BLOB_SUFFIX), BLOB_SUFFIX) == 0; if (!is_blob) { continue; } diff --git a/csrc/layers/quantization/gguf.hpp b/csrc/layers/quantization/gguf.hpp index 0a9eb475b..6af785587 100644 --- a/csrc/layers/quantization/gguf.hpp +++ b/csrc/layers/quantization/gguf.hpp @@ -65,7 +65,7 @@ class GGUFBlockQuantization : public BaseQuantization { const std::string &stem) const override; // 融合 Linear 的唯一入口：各 shard 的 ggml type id 只能由自己的 stem 查出来 - //（实测 q/k/v 同类型的 full-attn 层数 0/16），而组 stem 做不到。见 §7.2 子步骤 0。 + // （实测 q/k/v 同类型的 full-attn 层数 0/16），而组 stem 做不到。见 §7.2 子步骤 0。 infinicore::Tensor forward( const ParamsMap ¶ms, const infinicore::Tensor &input, diff --git a/csrc/layers/quantization/quantization.hpp b/csrc/layers/quantization/quantization.hpp index 1f935d13d..b75314e1b 100644 --- a/csrc/layers/quantization/quantization.hpp +++ b/csrc/layers/quantization/quantization.hpp @@ -4,11 +4,11 @@ #include "awq_marlin.hpp" #include "base_quantization.hpp" #include "compressed_tensors.hpp" +#include "fp8.hpp" #include "gguf.hpp" #include "gptq.hpp" #include "gptq_marlin.hpp" #include "gptq_qy.hpp" #include "mxfp4.hpp" #include "none_quantization.hpp" -#include "fp8.hpp" #include "quantization_scheme.hpp" diff --git a/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp b/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp index 2df7e2c79..cfbe9ab4e 100644 --- a/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp +++ b/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp @@ -98,8 +98,8 @@ std::tuple Qwen35DecoderLayer::forward(c const bool fp32_fused = fp32_fused_env != nullptr && fp32_fused_env[0] != '\0' && std::string(fp32_fused_env) != "0"; const bool mixed_gguf_f32 = residual - && hidden_states->dtype() == infinicore::DataType::F32 - && residual->dtype() == infinicore::DataType::BF16; + && hidden_states->dtype() == infinicore::DataType::F32 + && residual->dtype() == infinicore::DataType::BF16; if (mixed_gguf_f32) { auto y = infinicore::Tensor::empty( hidden_states->shape(), infinicore::DataType::BF16, hidden_states->device()); diff --git a/csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp b/csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp index c47bc7797..1105b2d24 100644 --- a/csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp +++ b/csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp @@ -18,9 +18,9 @@ Qwen35ForCausalLM::Qwen35ForCausalLM( const size_t hidden_size = model_config->get("hidden_size"); const size_t vocab_size = model_config->get("vocab_size"); const auto &dtype = model_config->get_dtype(); - fp32_lm_head_output_ = - model_config->get_config_json().value( - "lm_head_output_dtype", std::string()) == "float32"; + fp32_lm_head_output_ = model_config->get_config_json().value( + "lm_head_output_dtype", std::string()) + == "float32"; INFINICORE_NN_MODULE_INIT(model, model_config, device); INFINICORE_NN_MODULE_INIT( @@ -42,8 +42,8 @@ InfinilmModel::Output Qwen35ForCausalLM::forward( infinicore::Tensor logits; if (fp32_lm_head_output_) { auto hidden = hidden_states->is_contiguous() - ? hidden_states - : hidden_states->contiguous(); + ? hidden_states + : hidden_states->contiguous(); const size_t ndim = hidden->ndim(); auto output_shape = hidden->shape(); output_shape[ndim - 1] = lm_head_->out_features(); diff --git a/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.cpp b/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.cpp index eba12169f..87183f81c 100644 --- a/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.cpp +++ b/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.cpp @@ -17,16 +17,16 @@ Qwen35FusedQKVLinear::Qwen35FusedQKVLinear(size_t hidden_size, engine::distributed::RankInfo rank_info, const std::string &prefix) : infinilm::layers::linear::ColumnParallelLinear( - hidden_size, - num_q_head * head_dim * 2 + num_kv_head * head_dim * calculate_kv_replicas(num_kv_head, rank_info.tp_size) * 2, - quantization == nullptr ? std::make_shared() : quantization, - bias, - dtype, - device, - rank_info.tp_rank, - rank_info.tp_size, - -1, - prefix), + hidden_size, + num_q_head * head_dim * 2 + num_kv_head * head_dim * calculate_kv_replicas(num_kv_head, rank_info.tp_size) * 2, + quantization == nullptr ? std::make_shared() : quantization, + bias, + dtype, + device, + rank_info.tp_rank, + rank_info.tp_size, + -1, + prefix), head_dim_(head_dim), local_num_q_heads_(num_q_head / tp_size_), q_proj_out_size_(num_q_head * head_dim * 2 / tp_size_), diff --git a/csrc/pybind11/engine/engine.hpp b/csrc/pybind11/engine/engine.hpp index 38b50ab8a..5e8a6caea 100644 --- a/csrc/pybind11/engine/engine.hpp +++ b/csrc/pybind11/engine/engine.hpp @@ -122,18 +122,14 @@ inline void bind_infer_engine(py::module &m) { return state_dict_tp_all; }) .def("process_weights_after_loading", &InferEngine::process_weights_after_loading, "Process the weights after loading on all workers (e.g., for quantization)") - .def( - "forward", [](InferEngine &self, const InferEngine::Input &input) -> InferEngine::Output { + .def("forward", [](InferEngine &self, const InferEngine::Input &input) -> InferEngine::Output { // IMPORTANT: Release the GIL before calling forward() to allow other Python threads // to run concurrently during inference (which may block for a long time). // Do NOT remove this — without it, the GIL is held throughout inference and will // deadlock or stall any other Python thread (e.g., request handling, scheduling). py::gil_scoped_release release; - return self.forward(input); - }, - "Run inference on all ranks with arbitrary arguments") - .def( - "reset_cache", [](InferEngine &self, std::shared_ptr cfg) { self.reset_cache(cfg ? cfg.get() : nullptr); }, py::arg("cache_config") = py::none()) + return self.forward(input); }, "Run inference on all ranks with arbitrary arguments") + .def("reset_cache", [](InferEngine &self, std::shared_ptr cfg) { self.reset_cache(cfg ? cfg.get() : nullptr); }, py::arg("cache_config") = py::none()) .def("get_kv_cache", &InferEngine::get_kv_cache, "Get per-rank kv cache list") .def("get_cache_config", [](const InferEngine &self) -> std::shared_ptr { auto cfg = self.get_cache_config(); @@ -211,8 +207,7 @@ inline void bind_infer_engine(py::module &m) { } else if (key == "top_k") { input.top_k = py::cast(item.second); } else if (key == "suppressed_token_ids") { - input.suppressed_token_ids = - py::cast>>(item.second); + input.suppressed_token_ids = py::cast>>(item.second); } } diff --git a/python/infinilm/modeling_utils.py b/python/infinilm/modeling_utils.py index 0e3245ecd..8647cecab 100644 --- a/python/infinilm/modeling_utils.py +++ b/python/infinilm/modeling_utils.py @@ -56,10 +56,12 @@ def parse_dtype(dtype_str: str): } _FP8_DTYPES = tuple( - x for x in ( + x + for x in ( getattr(torch, "float8_e4m3fn", None), getattr(torch, "float8_e5m2", None), - ) if x is not None + ) + if x is not None ) @@ -266,7 +268,10 @@ def load_model_state_dict_by_file( # Convert FP8 block scales from BF16 to FP32 for CUTLASS GEMM for key in list(model_param.keys()): - if key.endswith("weight_scale_inv") and model_param[key].dtype == torch.bfloat16: + if ( + key.endswith("weight_scale_inv") + and model_param[key].dtype == torch.bfloat16 + ): model_param[key] = model_param[key].float() # --------------------------------------------------------- # @@ -786,7 +791,9 @@ def _remap_qwen3_5(state_dict, config): state_dict = drop_keys(state_dict, ["mtp."]) # Filter out visual encoder keys (not used in language-only mode) - state_dict = {k: v for k, v in state_dict.items() if not k.startswith("model.visual.")} + state_dict = { + k: v for k, v in state_dict.items() if not k.startswith("model.visual.") + } llm_config = config["text_config"] key_dim = llm_config["linear_key_head_dim"] * llm_config["linear_num_key_heads"] @@ -796,9 +803,7 @@ def _remap_qwen3_5(state_dict, config): # （conversion/qwen.py:393-394，除 linear_attn.norm 之外全部加），打包器按「不得再 # 加一次」原样搬运（scripts/gguf_mapping.py 顶部第 13 行）。这里再加一次就变成 # 2+w；融合 QKV 也已在打包期拆成 in_proj_q/k/v，不能按老键名再拆一遍。 - gguf = ( - (config.get("quantization_config") or {}).get("quant_method", "") == "gguf" - ) + gguf = (config.get("quantization_config") or {}).get("quant_method", "") == "gguf" norm_weight_suffixes = ( "input_layernorm.weight", @@ -954,14 +959,14 @@ def fuse_expert_group(expert_ids): b1_tensors.append(torch.cat([gate_bias, up_bias], dim=0)) b2_tensors.append(down_bias) - fused_dtype = w1_tensors[0].dtype if w1_tensors[0].dtype in _FP8_DTYPES else target_dtype + fused_dtype = ( + w1_tensors[0].dtype + if w1_tensors[0].dtype in _FP8_DTYPES + else target_dtype + ) fused = { - "w1": torch.stack(w1_tensors, dim=0) - .to(dtype=fused_dtype) - .contiguous(), - "w2": torch.stack(w2_tensors, dim=0) - .to(dtype=fused_dtype) - .contiguous(), + "w1": torch.stack(w1_tensors, dim=0).to(dtype=fused_dtype).contiguous(), + "w2": torch.stack(w2_tensors, dim=0).to(dtype=fused_dtype).contiguous(), } if has_all_bias: fused["b1"] = ( diff --git a/scripts/gguf_mapping.py b/scripts/gguf_mapping.py index 84c7fdd49..dfa54d513 100644 --- a/scripts/gguf_mapping.py +++ b/scripts/gguf_mapping.py @@ -28,15 +28,14 @@ import re from dataclasses import dataclass - # --------------------------------------------------------------------------- # transform 语义 # --------------------------------------------------------------------------- -T_NONE = "" # 原样搬运（blob 逐字节 / dense 仅换 dtype） -T_VROWS = "vrows" # 沿 out 维按 V 头分块整块搬回 grouped 序（blob 可行级置换） -T_VELEM = "velem" # 1-D、每头 1 个元素：T_VROWS 的 head_dim=1 退化形式（同一实现） -T_ALOG = "alog" # A_log = log(-ssm_a)，再置换 -T_DENSE = "dense" # 反量化为 BF16（框架不支持该参数走量化路径） +T_NONE = "" # 原样搬运（blob 逐字节 / dense 仅换 dtype） +T_VROWS = "vrows" # 沿 out 维按 V 头分块整块搬回 grouped 序（blob 可行级置换） +T_VELEM = "velem" # 1-D、每头 1 个元素：T_VROWS 的 head_dim=1 退化形式（同一实现） +T_ALOG = "alog" # A_log = log(-ssm_a)，再置换 +T_DENSE = "dense" # 反量化为 BF16（框架不支持该参数走量化路径） # V 头置换在 dim0 上的作用域： # all = 整个 dim0 都是 value 头（in_proj_v / in_proj_z / in_proj_a / in_proj_b / A_log / dt_bias） @@ -85,8 +84,11 @@ def apply_v1_exceptions(plan, gguf_types, enabled=True): if e.blob and gguf_types.get(e.gguf) in V1_IQUANT_DENSE: e.blob = False e.transforms = e.transforms + (T_DENSE,) - e.note = (e.note + "；" if e.note else "") + \ - "v1 稠密化例外（源 %s），阶段 6 上原生 kernel 后取消" % gguf_types[e.gguf] + e.note = ( + e.note + "；" if e.note else "" + ) + "v1 稠密化例外（源 %s），阶段 6 上原生 kernel 后取消" % gguf_types[ + e.gguf + ] n += 1 return n @@ -95,16 +97,16 @@ def apply_v1_exceptions(plan, gguf_types, enabled=True): class Entry: """一条 GGUF 张量 -> 一个 InfiniLM 参数。""" - infinilm: str # InfiniLM 参数名（含 model.language_model. 前缀） - gguf: str # GGUF 张量名 - shape: tuple # InfiniLM 期望 shape（未 TP 切分的全量），取向 [out, in] - blob: bool # True = 保留 GGUF 原始 block 字节（U8 [out, row_bytes]） + infinilm: str # InfiniLM 参数名（含 model.language_model. 前缀） + gguf: str # GGUF 张量名 + shape: tuple # InfiniLM 期望 shape（未 TP 切分的全量），取向 [out, in] + blob: bool # True = 保留 GGUF 原始 block 字节（U8 [out, row_bytes]） transforms: tuple = () - types: tuple = () # 允许的 GGUF 源类型名；() = 不限（由 contract 脚本报告实际值） - slices: tuple = () # 沿 out 维占用的 [start, end)；共用同一 gguf 的条目做覆盖校验 - vperm: str = VPERM_ALL # T_VROWS 的作用域（仅当 transforms 含 T_VROWS 时有意义） + types: tuple = () # 允许的 GGUF 源类型名；() = 不限（由 contract 脚本报告实际值） + slices: tuple = () # 沿 out 维占用的 [start, end)；共用同一 gguf 的条目做覆盖校验 + vperm: str = VPERM_ALL # T_VROWS 的作用域（仅当 transforms 含 T_VROWS 时有意义） # 该条目的权重需要置换的是**列（in 维）**而不是行：块量化沿 in 维分块 - #（Q4_K/Q5_K/Q6_K block_size=256），打包期置换列 = 跨块重排 = 必须重量化，做不到。 + # （Q4_K/Q5_K/Q6_K block_size=256），打包期置换列 = 跨块重排 = 必须重量化，做不到。 # 于是只能在运行时置换喂给它的输入激活，规则由 activation_vperm_rules() 导出进 config。 # 故意不放进 transforms：那个元组描述的是「打包期对字节做的事」，混进去会污染字节路径。 act_vperm: bool = False @@ -135,7 +137,7 @@ class Dims: # --- 派生量 --- @property - def q_rows(self) -> int: # q_proj 行数 = heads * head_dim * 2（q 与 gate 每头交错） + def q_rows(self) -> int: # q_proj 行数 = heads * head_dim * 2（q 与 gate 每头交错） return self.n_q_heads * self.head_dim * 2 @property @@ -155,7 +157,7 @@ def value_dim(self) -> int: return self.lin_v_heads * self.lin_v_dim @property - def qkv_rows(self) -> int: # q | k | v 融合（与 GGUF attn_qkv 一致） + def qkv_rows(self) -> int: # q | k | v 融合（与 GGUF attn_qkv 一致） return self.key_dim * 2 + self.value_dim @property @@ -168,17 +170,43 @@ def v_per_k(self) -> int: def layer_types(self) -> list: """与 C++ prepare_qwen3_5_model_config 的推导完全一致：(i+1) % interval == 0。""" - return ["full_attention" if (i + 1) % self.interval == 0 else "linear_attention" - for i in range(self.n_layers)] - + return [ + "full_attention" if (i + 1) % self.interval == 0 else "linear_attention" + for i in range(self.n_layers) + ] -REAL = Dims(hidden=5120, n_q_heads=24, n_kv_heads=4, head_dim=256, ffn=17408, - lin_k_heads=16, lin_v_heads=48, lin_k_dim=128, lin_v_dim=128, - conv_kernel=4, vocab=248320, n_layers=64, interval=4) -MINI = Dims(hidden=512, n_q_heads=2, n_kv_heads=1, head_dim=256, ffn=1024, - lin_k_heads=2, lin_v_heads=6, lin_k_dim=128, lin_v_dim=128, - conv_kernel=4, vocab=1024, n_layers=8, interval=4) +REAL = Dims( + hidden=5120, + n_q_heads=24, + n_kv_heads=4, + head_dim=256, + ffn=17408, + lin_k_heads=16, + lin_v_heads=48, + lin_k_dim=128, + lin_v_dim=128, + conv_kernel=4, + vocab=248320, + n_layers=64, + interval=4, +) + +MINI = Dims( + hidden=512, + n_q_heads=2, + n_kv_heads=1, + head_dim=256, + ffn=1024, + lin_k_heads=2, + lin_v_heads=6, + lin_k_dim=128, + lin_v_dim=128, + conv_kernel=4, + vocab=1024, + n_layers=8, + interval=4, +) PREFIX = "model.language_model." @@ -194,57 +222,172 @@ def layer_entries(d: Dims, i: int, role: str) -> list: G = f"blk.{i}." kd, vd = d.key_dim, d.value_dim out = [ - Entry(L + "input_layernorm.weight", G + "attn_norm.weight", (d.hidden,), - False, (T_DENSE,), note="GGUF 已 baked +1，打包不得再加"), - Entry(L + "post_attention_layernorm.weight", G + "post_attention_norm.weight", - (d.hidden,), False, (T_DENSE,), note="同上"), - Entry(L + "mlp.gate_proj.weight", G + "ffn_gate.weight", (d.ffn, d.hidden), True), + Entry( + L + "input_layernorm.weight", + G + "attn_norm.weight", + (d.hidden,), + False, + (T_DENSE,), + note="GGUF 已 baked +1，打包不得再加", + ), + Entry( + L + "post_attention_layernorm.weight", + G + "post_attention_norm.weight", + (d.hidden,), + False, + (T_DENSE,), + note="同上", + ), + Entry( + L + "mlp.gate_proj.weight", G + "ffn_gate.weight", (d.ffn, d.hidden), True + ), Entry(L + "mlp.up_proj.weight", G + "ffn_up.weight", (d.ffn, d.hidden), True), - Entry(L + "mlp.down_proj.weight", G + "ffn_down.weight", (d.hidden, d.ffn), True), + Entry( + L + "mlp.down_proj.weight", G + "ffn_down.weight", (d.hidden, d.ffn), True + ), ] if role == "full_attention": out += [ - Entry(L + "self_attn.q_proj.weight", G + "attn_q.weight", (d.q_rows, d.hidden), - True, (), note="行数含 q|gate 每头交错，与 Qwen35FusedQKVLinear 一致"), - Entry(L + "self_attn.k_proj.weight", G + "attn_k.weight", (d.kv_rows, d.hidden), True), - Entry(L + "self_attn.v_proj.weight", G + "attn_v.weight", (d.kv_rows, d.hidden), True), - Entry(L + "self_attn.o_proj.weight", G + "attn_output.weight", (d.hidden, d.o_in), True), - Entry(L + "self_attn.q_norm.weight", G + "attn_q_norm.weight", (d.head_dim,), - False, (T_DENSE,), note="GGUF 已 baked +1"), - Entry(L + "self_attn.k_norm.weight", G + "attn_k_norm.weight", (d.head_dim,), - False, (T_DENSE,), note="GGUF 已 baked +1"), + Entry( + L + "self_attn.q_proj.weight", + G + "attn_q.weight", + (d.q_rows, d.hidden), + True, + (), + note="行数含 q|gate 每头交错，与 Qwen35FusedQKVLinear 一致", + ), + Entry( + L + "self_attn.k_proj.weight", + G + "attn_k.weight", + (d.kv_rows, d.hidden), + True, + ), + Entry( + L + "self_attn.v_proj.weight", + G + "attn_v.weight", + (d.kv_rows, d.hidden), + True, + ), + Entry( + L + "self_attn.o_proj.weight", + G + "attn_output.weight", + (d.hidden, d.o_in), + True, + ), + Entry( + L + "self_attn.q_norm.weight", + G + "attn_q_norm.weight", + (d.head_dim,), + False, + (T_DENSE,), + note="GGUF 已 baked +1", + ), + Entry( + L + "self_attn.k_norm.weight", + G + "attn_k_norm.weight", + (d.head_dim,), + False, + (T_DENSE,), + note="GGUF 已 baked +1", + ), ] else: out += [ - Entry(L + "linear_attn.in_proj_q.weight", G + "attn_qkv.weight", (kd, d.hidden), - True, (), slices=((0, kd),), note="attn_qkv 行 [0:kd]"), - Entry(L + "linear_attn.in_proj_k.weight", G + "attn_qkv.weight", (kd, d.hidden), - True, (), slices=((kd, 2 * kd),), note="attn_qkv 行 [kd:2kd]"), - Entry(L + "linear_attn.in_proj_v.weight", G + "attn_qkv.weight", (vd, d.hidden), - True, (T_VROWS,), slices=((2 * kd, 2 * kd + vd),), - note="attn_qkv 行 [2kd:]，V 头 tiled->grouped"), - Entry(L + "linear_attn.in_proj_z.weight", G + "attn_gate.weight", (vd, d.hidden), - True, (T_VROWS,), note="qwen.py:583 行重排（head_v_dim）"), - Entry(L + "linear_attn.in_proj_a.weight", G + "ssm_alpha.weight", - (d.lin_v_heads, d.hidden), False, (T_DENSE, T_VROWS), - note="实测源为 Q8_0；框架该参数不走量化路径 -> 稠密化；" - "qwen.py:587 行重排 head_dim=1"), - Entry(L + "linear_attn.in_proj_b.weight", G + "ssm_beta.weight", - (d.lin_v_heads, d.hidden), False, (T_DENSE, T_VROWS), note="同上"), - Entry(L + "linear_attn.A_log", G + "ssm_a", (d.lin_v_heads,), - False, (T_ALOG, T_VROWS), note="GGUF 存的是 -exp(A_log)，需 log(-x) 反解"), - Entry(L + "linear_attn.dt_bias", G + "ssm_dt.bias", (d.lin_v_heads,), - False, (T_VELEM,), note="qwen.py:589 逐头置换，值不变"), - Entry(L + "linear_attn.conv1d.weight", G + "ssm_conv1d.weight", - (d.conv_channels, 1, d.conv_kernel), False, (T_DENSE, T_VROWS), - vperm=VPERM_TAIL, - note="GGUF 已 squeeze 成 [C,K] -> 补回中间维；仅末尾 V 通道段重排"), - Entry(L + "linear_attn.norm.weight", G + "ssm_norm.weight", (d.lin_v_dim,), - False, (T_DENSE,), note="不在 qwen.py 重排列表内；两侧都不加 1"), - Entry(L + "linear_attn.out_proj.weight", G + "ssm_out.weight", - (d.hidden, vd), True, (), act_vperm=True, - note="qwen.py:609 重排的是列(in 维)，blob 不能跨块置换 -> " - "运行时对输入激活做 grouped->tiled（见 config 的 activation_vperm）"), + Entry( + L + "linear_attn.in_proj_q.weight", + G + "attn_qkv.weight", + (kd, d.hidden), + True, + (), + slices=((0, kd),), + note="attn_qkv 行 [0:kd]", + ), + Entry( + L + "linear_attn.in_proj_k.weight", + G + "attn_qkv.weight", + (kd, d.hidden), + True, + (), + slices=((kd, 2 * kd),), + note="attn_qkv 行 [kd:2kd]", + ), + Entry( + L + "linear_attn.in_proj_v.weight", + G + "attn_qkv.weight", + (vd, d.hidden), + True, + (T_VROWS,), + slices=((2 * kd, 2 * kd + vd),), + note="attn_qkv 行 [2kd:]，V 头 tiled->grouped", + ), + Entry( + L + "linear_attn.in_proj_z.weight", + G + "attn_gate.weight", + (vd, d.hidden), + True, + (T_VROWS,), + note="qwen.py:583 行重排（head_v_dim）", + ), + Entry( + L + "linear_attn.in_proj_a.weight", + G + "ssm_alpha.weight", + (d.lin_v_heads, d.hidden), + False, + (T_DENSE, T_VROWS), + note="实测源为 Q8_0；框架该参数不走量化路径 -> 稠密化；" + "qwen.py:587 行重排 head_dim=1", + ), + Entry( + L + "linear_attn.in_proj_b.weight", + G + "ssm_beta.weight", + (d.lin_v_heads, d.hidden), + False, + (T_DENSE, T_VROWS), + note="同上", + ), + Entry( + L + "linear_attn.A_log", + G + "ssm_a", + (d.lin_v_heads,), + False, + (T_ALOG, T_VROWS), + note="GGUF 存的是 -exp(A_log)，需 log(-x) 反解", + ), + Entry( + L + "linear_attn.dt_bias", + G + "ssm_dt.bias", + (d.lin_v_heads,), + False, + (T_VELEM,), + note="qwen.py:589 逐头置换，值不变", + ), + Entry( + L + "linear_attn.conv1d.weight", + G + "ssm_conv1d.weight", + (d.conv_channels, 1, d.conv_kernel), + False, + (T_DENSE, T_VROWS), + vperm=VPERM_TAIL, + note="GGUF 已 squeeze 成 [C,K] -> 补回中间维；仅末尾 V 通道段重排", + ), + Entry( + L + "linear_attn.norm.weight", + G + "ssm_norm.weight", + (d.lin_v_dim,), + False, + (T_DENSE,), + note="不在 qwen.py 重排列表内；两侧都不加 1", + ), + Entry( + L + "linear_attn.out_proj.weight", + G + "ssm_out.weight", + (d.hidden, vd), + True, + (), + act_vperm=True, + note="qwen.py:609 重排的是列(in 维)，blob 不能跨块置换 -> " + "运行时对输入激活做 grouped->tiled（见 config 的 activation_vperm）", + ), ] return out @@ -252,16 +395,34 @@ def layer_entries(d: Dims, i: int, role: str) -> list: def build_plan(d: Dims) -> list: """全模型映射条目（含顶层）。""" entries = [ - Entry(PREFIX + "embed_tokens.weight", "token_embd.weight", (d.vocab, d.hidden), - False, (T_DENSE,), note="实测 GGUF 为 Q6_K -> 反量化"), + Entry( + PREFIX + "embed_tokens.weight", + "token_embd.weight", + (d.vocab, d.hidden), + False, + (T_DENSE,), + note="实测 GGUF 为 Q6_K -> 反量化", + ), ] for i, role in enumerate(d.layer_types()): entries += layer_entries(d, i, role) entries += [ - Entry(PREFIX + "norm.weight", "output_norm.weight", (d.hidden,), - False, (T_DENSE,), note="GGUF 已 baked +1"), - Entry("lm_head.weight", "output.weight", (d.vocab, d.hidden), - False, (T_DENSE,), note="实测 GGUF 为 Q8_0 -> 反量化"), + Entry( + PREFIX + "norm.weight", + "output_norm.weight", + (d.hidden,), + False, + (T_DENSE,), + note="GGUF 已 baked +1", + ), + Entry( + "lm_head.weight", + "output.weight", + (d.vocab, d.hidden), + False, + (T_DENSE,), + note="实测 GGUF 为 Q8_0 -> 反量化", + ), ] return entries @@ -274,7 +435,7 @@ def activation_vperm_suffix(e: "Entry") -> str: """ name = re.sub(r"^" + re.escape(PREFIX) + r"layers\.\d+\.", "", e.infinilm) if name.endswith(".weight"): - name = name[:-len(".weight")] + name = name[: -len(".weight")] return name + "." @@ -295,14 +456,17 @@ def activation_vperm_rules(d: "Dims", plan: list) -> list: continue in_dim = int(e.shape[1]) if in_dim != n_k * r * hd: - raise ValueError("%s: 条目 in 维 %d != num_k_heads*num_v_per_k*head_dim = %d，" - "无法按头分块置换" % (e.infinilm, in_dim, n_k * r * hd)) + raise ValueError( + "%s: 条目 in 维 %d != num_k_heads*num_v_per_k*head_dim = %d，" + "无法按头分块置换" % (e.infinilm, in_dim, n_k * r * hd) + ) suffix = activation_vperm_suffix(e) if suffix in seen: continue seen.add(suffix) - rules.append({"suffix": suffix, "num_k_heads": n_k, - "num_v_per_k": r, "head_dim": hd}) + rules.append( + {"suffix": suffix, "num_k_heads": n_k, "num_v_per_k": r, "head_dim": hd} + ) return rules @@ -333,7 +497,7 @@ def compress(shape: tuple) -> tuple: def ckpt_name(e: "Entry") -> str: """写进 safetensors（以及框架 state_dict）的参数名。""" if e.blob and e.infinilm.endswith(".weight"): - return e.infinilm[:-len(".weight")] + "." + BLOB_SUFFIX + return e.infinilm[: -len(".weight")] + "." + BLOB_SUFFIX return e.infinilm diff --git a/scripts/gguf_routeb_audit.py b/scripts/gguf_routeb_audit.py index 2a6b6c826..c82d050c6 100644 --- a/scripts/gguf_routeb_audit.py +++ b/scripts/gguf_routeb_audit.py @@ -19,9 +19,9 @@ from __future__ import annotations import argparse +import collections import os import sys -import collections import numpy as np @@ -29,12 +29,14 @@ sys.path.insert(0, os.path.join(_LLAMA_CPP, "gguf-py")) sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from gguf import GGUFReader # noqa: E402 -from gguf.constants import ( # noqa: E402 +import gguf.quants as gq # noqa: E402 +from gguf import GGUFReader # noqa: E402 +from gguf.constants import ( # noqa: E402 GGML_QUANT_SIZES, +) +from gguf.constants import ( # noqa: E402 GGMLQuantizationType as QType, ) -import gguf.quants as gq # noqa: E402 QK_K = 256 @@ -53,11 +55,14 @@ def check(name: str, ok: bool, detail: str = "") -> bool: # 输入统一为 uint8 blob，形状 [n_rows, row_bytes]；输出 float32 [n_rows, n_cols] # --------------------------------------------------------------------------- + def _rows_to_blocks(blob: np.ndarray, type_size: int) -> np.ndarray: """[n_rows, row_bytes] -> [n_blocks, type_size]，块沿 in 连续、按 out 行排列。""" assert blob.dtype == np.uint8 n_rows, row_bytes = blob.shape - assert row_bytes % type_size == 0, f"row_bytes={row_bytes} 不是 type_size={type_size} 的整数倍" + assert row_bytes % type_size == 0, ( + f"row_bytes={row_bytes} 不是 type_size={type_size} 的整数倍" + ) return blob.reshape(-1, type_size) @@ -67,10 +72,10 @@ def _f16(col: np.ndarray) -> np.ndarray: def decode_q8_0(blob: np.ndarray, n_cols: int) -> np.ndarray: """块 = d(f16,2B) + qs(int8,32B)，共 34B / 32 元素。""" - bs, ts = 32, 34 + ts = 34 b = _rows_to_blocks(blob, ts) - d = _f16(b[:, :2]) # [nb,1] - x = b[:, 2:ts].view(np.int8).astype(np.float32) # [nb,32] + d = _f16(b[:, :2]) # [nb,1] + x = b[:, 2:ts].view(np.int8).astype(np.float32) # [nb,32] return (d * x).reshape(blob.shape[0], n_cols) @@ -92,7 +97,7 @@ def _k_scale_min(scales: np.ndarray) -> tuple[np.ndarray, np.ndarray]: _o = _e % 32 K_QS_BYTE = (_g // 2) * 32 + _o K_QS_SHIFT = (_g % 2) * 4 -K_SCALE_IDX = _g # 每 32 元素一组 scale/min +K_SCALE_IDX = _g # 每 32 元素一组 scale/min # Q5_K 的第 5 bit：qh 字节 = o，位 = g K5_QH_BYTE = _o @@ -119,8 +124,11 @@ def decode_q4_k(blob: np.ndarray, n_cols: int) -> np.ndarray: dmin = _f16(b[:, 2:4]) sc, mn = _k_scale_min(b[:, 4:16]) qs = b[:, 16:ts] - q = ((qs[:, K_QS_BYTE] >> K_QS_SHIFT.astype(np.uint8)) - & np.uint8(0x0F)).reshape(b.shape[0], 8, 32).astype(np.float32) + q = ( + ((qs[:, K_QS_BYTE] >> K_QS_SHIFT.astype(np.uint8)) & np.uint8(0x0F)) + .reshape(b.shape[0], 8, 32) + .astype(np.float32) + ) d_eff = (d * sc.astype(np.float32)).reshape(b.shape[0], 8, 1) m_eff = (dmin * mn.astype(np.float32)).reshape(b.shape[0], 8, 1) return (d_eff * q - m_eff).reshape(blob.shape[0], n_cols) @@ -155,8 +163,11 @@ def decode_q6_k(blob: np.ndarray, n_cols: int) -> np.ndarray: d = _f16(b[:, 208:210]) lo = (ql[:, Q6_LO_BYTE] >> Q6_LO_SHIFT.astype(np.uint8)) & np.uint8(0x0F) hi = (qh[:, Q6_HI_BYTE] >> Q6_HI_SHIFT.astype(np.uint8)) & np.uint8(0x03) - q = ((lo | (hi << np.uint8(4))).astype(np.int16) - 32 - ).reshape(n, 16, 16).astype(np.float32) + q = ( + ((lo | (hi << np.uint8(4))).astype(np.int16) - 32) + .reshape(n, 16, 16) + .astype(np.float32) + ) step = (d * sc).reshape(n, 16, 1) return (step * q).reshape(blob.shape[0], n_cols) @@ -173,6 +184,7 @@ def decode_q6_k(blob: np.ndarray, n_cols: int) -> np.ndarray: # A. 容器 / 字节布局 / block 位运算 # --------------------------------------------------------------------------- + def pick_samples(tensors: dict[str, object], per_type: int = 3) -> list: """每种量化类型最多挑 per_type 个（按 (in,out) 形状去重），只解部分行以省时。""" by_type = collections.defaultdict(list) @@ -205,14 +217,14 @@ def section_a(reader) -> None: for t in samples: qt = QType(int(t.tensor_type)) bs, ts = GGML_QUANT_SIZES[int(qt)] - n_in, n_out = int(t.shape[0]), int(t.shape[1]) # GGML: ne[0]=in, ne[1]=out + n_in, n_out = int(t.shape[0]), int(t.shape[1]) # GGML: ne[0]=in, ne[1]=out row_bytes = n_in // bs * ts - blob = np.ascontiguousarray(t.data) # 解析器已给 [out, row_bytes] + blob = np.ascontiguousarray(t.data) # 解析器已给 [out, row_bytes] ok_shape = blob.shape == (n_out, row_bytes) dec = DECODERS[qt] - n_rows = min(64, n_out) # 只解前 n_rows 行，省时 + n_rows = min(64, n_out) # 只解前 n_rows 行，省时 ours = dec(blob[:n_rows], n_in) - ref_full = gq.dequantize(blob[:n_rows], qt) # 权威实现，输入为字节形状 + ref_full = gq.dequantize(blob[:n_rows], qt) # 权威实现，输入为字节形状 ref = np.asarray(ref_full, dtype=np.float32) exact = ours.shape == ref.shape and np.array_equal(ours, ref) # 单行独立性：逐行解码必须与整体解码一致（证明行是连续独立单元） @@ -226,22 +238,34 @@ def section_a(reader) -> None: check("A 汇总", all_ok) # 非量化张量的轴序（打包器是否需要转置的依据） conv = tensors["blk.0.ssm_conv1d.weight"] - check("F32 张量的 data 也是 C 序 [shape[1], shape[0]]（= HF 取向，打包器不转置）", - conv.data.shape == (int(conv.shape[1]), int(conv.shape[0])), - f"ne={list(map(int, conv.shape))} data={conv.data.shape} -> HF [10240,1,4]") + check( + "F32 张量的 data 也是 C 序 [shape[1], shape[0]]（= HF 取向，打包器不转置）", + conv.data.shape == (int(conv.shape[1]), int(conv.shape[0])), + f"ne={list(map(int, conv.shape))} data={conv.data.shape} -> HF [10240,1,4]", + ) norm = tensors["blk.0.attn_norm.weight"] - check("1-D norm 保持 dtype=float32且长度 = hidden", - norm.data.dtype == np.float32 and norm.data.shape == (5120,)) + check( + "1-D norm 保持 dtype=float32且长度 = hidden", + norm.data.dtype == np.float32 and norm.data.shape == (5120,), + ) # --------------------------------------------------------------------------- # B. 对齐事实 # --------------------------------------------------------------------------- + def section_b() -> None: print("\n== B. 对齐事实（kernel 的硬约束）==") facts = [] - for qt in (QType.Q8_0, QType.Q4_K, QType.Q5_K, QType.Q6_K, QType.IQ4_NL, QType.IQ4_XS): + for qt in ( + QType.Q8_0, + QType.Q4_K, + QType.Q5_K, + QType.Q6_K, + QType.IQ4_NL, + QType.IQ4_XS, + ): bs, ts = GGML_QUANT_SIZES[int(qt)] align_block = 2 if ts % 2 == 0 else 1 for n_in in (5120, 6144, 10240, 17408, 248320): @@ -252,17 +276,26 @@ def section_b() -> None: while a > 1 and rb % a: a //= 2 facts.append((qt.name, bs, ts, n_in, rb, a, align_block)) - print(f" {'type':8s} {'bs':>4s} {'ts':>4s} {'in':>7s} {'row_bytes':>10s} " - f"{'行对齐':>7s} {'块起始对齐':>10s}") + print( + f" {'type':8s} {'bs':>4s} {'ts':>4s} {'in':>7s} {'row_bytes':>10s} " + f"{'行对齐':>7s} {'块起始对齐':>10s}" + ) worst_row, worst_block = 16, 2 for name, bs, ts, n_in, rb, a, ab in facts: - print(f" {name:8s} {bs:4d} {ts:4d} {n_in:7d} {rb:10d} {str(a)+'B':>7s} {str(ab)+'B':>10s}") + print( + f" {name:8s} {bs:4d} {ts:4d} {n_in:7d} {rb:10d} {str(a) + 'B':>7s} {str(ab) + 'B':>10s}" + ) worst_row = min(worst_row, a) worst_block = min(worst_block, ab) - check("块起始地址仅保证 2B 对齐（Q6_K=210B / Q8_0=34B 非 4 倍数）", - worst_block == 2, f"min_block_align={worst_block}B") - check("Q6_K 在 in=5120/17408 时行 stride 仅 8B 对齐", - any(f[0] == "Q6_K" and f[5] == 8 for f in facts)) + check( + "块起始地址仅保证 2B 对齐（Q6_K=210B / Q8_0=34B 非 4 倍数）", + worst_block == 2, + f"min_block_align={worst_block}B", + ) + check( + "Q6_K 在 in=5120/17408 时行 stride 仅 8B 对齐", + any(f[0] == "Q6_K" and f[5] == 8 for f in facts), + ) print(" -> 结论：kernel 不得对单块起始地址做 >2B 向量化加载假设；容器不做 pad。") @@ -270,25 +303,30 @@ def section_b() -> None: # C. V 头重排（grouped <-> tiled） # --------------------------------------------------------------------------- + def reorder_v(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: """与 llama.cpp conversion/qwen.py::_reorder_v_heads 同语义（沿 dim0 的整头置换）。""" rest = t.shape[1:] - return (t.reshape((n_k, n_v_per_k, hd) + rest) - .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) - .reshape((n_k * n_v_per_k * hd,) + rest)) + return ( + t.reshape((n_k, n_v_per_k, hd) + rest) + .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) + .reshape((n_k * n_v_per_k * hd,) + rest) + ) def reorder_v_inverse(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: """逆变换 = 两个轴参数对调后再调用一次。""" rest = t.shape[1:] - return (t.reshape((n_v_per_k, n_k, hd) + rest) - .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) - .reshape((n_k * n_v_per_k * hd,) + rest)) + return ( + t.reshape((n_v_per_k, n_k, hd) + rest) + .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) + .reshape((n_k * n_v_per_k * hd,) + rest) + ) def section_c() -> None: print("\n== C. V 头重排（执行方案 §2.7）==") - n_k, n_v_per_k, hd = 16, 3, 128 # Qwen3.8: 16 key heads, 48 value heads + n_k, n_v_per_k, hd = 16, 3, 128 # Qwen3.8: 16 key heads, 48 value heads n_v = n_k * n_v_per_k rng = np.random.default_rng(0) @@ -296,52 +334,86 @@ def section_c() -> None: tiled = reorder_v(grouped, n_k, n_v_per_k, hd) back = reorder_v_inverse(tiled, n_k, n_v_per_k, hd) check("grouped -> tiled -> grouped 自等", np.array_equal(grouped, back)) - check("reorder_v 是整头搬运（每个 head 的 hd 行连续不被打散）", - all(np.array_equal(tiled[i * hd:(i + 1) * hd], - grouped[((i % n_k) * n_v_per_k + i // n_k) * hd - :((i % n_k) * n_v_per_k + i // n_k) * hd + hd]) - for i in range(n_v))) + check( + "reorder_v 是整头搬运（每个 head 的 hd 行连续不被打散）", + all( + np.array_equal( + tiled[i * hd : (i + 1) * hd], + grouped[ + ((i % n_k) * n_v_per_k + i // n_k) * hd : ( + (i % n_k) * n_v_per_k + i // n_k + ) + * hd + + hd + ], + ) + for i in range(n_v) + ), + ) # 槽位 j（value head 编号）-> 真实 k 头 的两种语义 - k_grouped = [j // n_v_per_k for j in range(n_v)] # InfiniCore kernel 的假设 - k_tiled = [j % n_k for j in range(n_v)] # GGUF(tiled) 的真实归属 - check("tiled 序直接喂给 `value_head_idx / value_heads_per_key_head` 会错配 k 头", - k_grouped != k_tiled, - f"错配槽位数={sum(a != b for a, b in zip(k_grouped, k_tiled))}/{n_v}") + k_grouped = [j // n_v_per_k for j in range(n_v)] # InfiniCore kernel 的假设 + k_tiled = [j % n_k for j in range(n_v)] # GGUF(tiled) 的真实归属 + check( + "tiled 序直接喂给 `value_head_idx / value_heads_per_key_head` 会错配 k 头", + k_grouped != k_tiled, + f"错配槽位数={sum(a != b for a, b in zip(k_grouped, k_tiled))}/{n_v}", + ) # 逆重排后回到 grouped 语义 - _ = np.repeat(np.arange(n_v), hd) # labels 仅用于形状参考 - check("逆变换后槽位归属恢复 grouped 语义", - np.array_equal( - reorder_v_inverse( - np.array([k * n_v_per_k + v for v in range(n_v_per_k) for k in range(n_k)]), - n_k, n_v_per_k, 1), - np.arange(n_v)), - "逆变换后 slot i 的 head 编号 = i，kernel 的 k = i // n_v_per_k 成立") - check("in_proj_a/b・A_log・dt_bias 的 head_dim=1 退化形式（逐元素置换）同样自等", - np.array_equal( - reorder_v_inverse(reorder_v(np.arange(n_v), n_k, n_v_per_k, 1), - n_k, n_v_per_k, 1), - np.arange(n_v))) - check("多维情形（如 conv1d 的 [channels, 1, kernel]）仅置换头维、尾部轴不动", - np.array_equal( - reorder_v_inverse(reorder_v(grouped[:, :1], n_k, n_v_per_k, hd), - n_k, n_v_per_k, hd), - grouped[:, :1])) + _ = np.repeat(np.arange(n_v), hd) # labels 仅用于形状参考 + check( + "逆变换后槽位归属恢复 grouped 语义", + np.array_equal( + reorder_v_inverse( + np.array( + [k * n_v_per_k + v for v in range(n_v_per_k) for k in range(n_k)] + ), + n_k, + n_v_per_k, + 1, + ), + np.arange(n_v), + ), + "逆变换后 slot i 的 head 编号 = i，kernel 的 k = i // n_v_per_k 成立", + ) + check( + "in_proj_a/b・A_log・dt_bias 的 head_dim=1 退化形式（逐元素置换）同样自等", + np.array_equal( + reorder_v_inverse( + reorder_v(np.arange(n_v), n_k, n_v_per_k, 1), n_k, n_v_per_k, 1 + ), + np.arange(n_v), + ), + ) + check( + "多维情形（如 conv1d 的 [channels, 1, kernel]）仅置换头维、尾部轴不动", + np.array_equal( + reorder_v_inverse( + reorder_v(grouped[:, :1], n_k, n_v_per_k, hd), n_k, n_v_per_k, hd + ), + grouped[:, :1], + ), + ) # 行置换对量化 blob 是「整块搬运」：以 Q6_K 为例验证字节级可置换性 bs, ts = GGML_QUANT_SIZES[int(QType.Q6_K)] row_bytes = 5120 // bs * ts blob = rng.integers(0, 256, size=(n_v, row_bytes), dtype=np.uint8) perm = np.arange(n_v)[::-1].copy() - check("量化 blob 的行置换 == 字节整行置换（无需重新量化）", - np.array_equal(blob[perm], np.ascontiguousarray(blob)[perm])) - print(" -> 结论：整行置换可字节级完成；ssm_out 的列(in 维)置换不可，改用运行时激活 gather。") + check( + "量化 blob 的行置换 == 字节整行置换（无需重新量化）", + np.array_equal(blob[perm], np.ascontiguousarray(blob)[perm]), + ) + print( + " -> 结论：整行置换可字节级完成；ssm_out 的列(in 维)置换不可，改用运行时激活 gather。" + ) # --------------------------------------------------------------------------- # D. 命名 / 形状契约 # --------------------------------------------------------------------------- + def gguf_meta(reader, suffix: str): """元数据键带架构前缀（qwen35.*），允许传短名；contents() 对单元素返回标量，统一成列表。""" for key in (f"qwen35.{suffix}", f"general.{suffix}", suffix): @@ -359,20 +431,46 @@ def section_d(reader) -> None: n_main = 64 full = [i for i in range(n_main) if (i + 1) % interval == 0] gdn = [i for i in range(n_main) if i not in full] - check("主模型层数 64（block_count 含 1 个 MTP 层）", - n_layer_gguf == n_main + 1, f"block_count={n_layer_gguf}") - check("full attention 层 = 3,7,...,63 共 16 层", - len(full) == 16 and full[0] == 3 and full[-1] == 63) + check( + "主模型层数 64（block_count 含 1 个 MTP 层）", + n_layer_gguf == n_main + 1, + f"block_count={n_layer_gguf}", + ) + check( + "full attention 层 = 3,7,...,63 共 16 层", + len(full) == 16 and full[0] == 3 and full[-1] == 63, + ) check("GDN 层 48 层", len(gdn) == 48) - need_full = ["attn_norm.weight", "post_attention_norm.weight", "attn_q.weight", - "attn_k.weight", "attn_v.weight", "attn_output.weight", - "attn_q_norm.weight", "attn_k_norm.weight", - "ffn_gate.weight", "ffn_up.weight", "ffn_down.weight"] - need_gdn = ["attn_norm.weight", "post_attention_norm.weight", "attn_qkv.weight", - "attn_gate.weight", "ssm_a", "ssm_alpha.weight", "ssm_beta.weight", - "ssm_conv1d.weight", "ssm_dt.bias", "ssm_norm.weight", "ssm_out.weight", - "ffn_gate.weight", "ffn_up.weight", "ffn_down.weight"] + need_full = [ + "attn_norm.weight", + "post_attention_norm.weight", + "attn_q.weight", + "attn_k.weight", + "attn_v.weight", + "attn_output.weight", + "attn_q_norm.weight", + "attn_k_norm.weight", + "ffn_gate.weight", + "ffn_up.weight", + "ffn_down.weight", + ] + need_gdn = [ + "attn_norm.weight", + "post_attention_norm.weight", + "attn_qkv.weight", + "attn_gate.weight", + "ssm_a", + "ssm_alpha.weight", + "ssm_beta.weight", + "ssm_conv1d.weight", + "ssm_dt.bias", + "ssm_norm.weight", + "ssm_out.weight", + "ffn_gate.weight", + "ffn_up.weight", + "ffn_down.weight", + ] missing = [] for i in full: missing += [f"blk.{i}.{r}" for r in need_full if f"blk.{i}.{r}" not in tensors] @@ -381,42 +479,70 @@ def section_d(reader) -> None: check("64 层全部所需张量存在", not missing, f"missing={missing[:6]}") shapes = { - "attn_q": (5120, 12288), "attn_k": (5120, 1024), "attn_v": (5120, 1024), - "attn_output": (6144, 5120), "attn_qkv": (5120, 10240), "attn_gate": (5120, 6144), - "ssm_out": (6144, 5120), "ffn_gate": (5120, 17408), "ffn_up": (5120, 17408), - "ffn_down": (17408, 5120), "ssm_conv1d": (4, 10240), + "attn_q": (5120, 12288), + "attn_k": (5120, 1024), + "attn_v": (5120, 1024), + "attn_output": (6144, 5120), + "attn_qkv": (5120, 10240), + "attn_gate": (5120, 6144), + "ssm_out": (6144, 5120), + "ffn_gate": (5120, 17408), + "ffn_up": (5120, 17408), + "ffn_down": (17408, 5120), + "ssm_conv1d": (4, 10240), } bad = [] for name, want in shapes.items(): - probe = {"attn_q": f"blk.{full[0]}.", "attn_k": f"blk.{full[0]}.", - "attn_v": f"blk.{full[0]}.", "attn_output": f"blk.{full[0]}.", - "attn_qkv": f"blk.{gdn[0]}.", "attn_gate": f"blk.{gdn[0]}.", - "ssm_out": f"blk.{gdn[0]}.", "ssm_conv1d": f"blk.{gdn[0]}.", - "ffn_gate": f"blk.{0}.", "ffn_up": f"blk.{0}.", "ffn_down": f"blk.{0}."}[name] + probe = { + "attn_q": f"blk.{full[0]}.", + "attn_k": f"blk.{full[0]}.", + "attn_v": f"blk.{full[0]}.", + "attn_output": f"blk.{full[0]}.", + "attn_qkv": f"blk.{gdn[0]}.", + "attn_gate": f"blk.{gdn[0]}.", + "ssm_out": f"blk.{gdn[0]}.", + "ssm_conv1d": f"blk.{gdn[0]}.", + "ffn_gate": f"blk.{0}.", + "ffn_up": f"blk.{0}.", + "ffn_down": f"blk.{0}.", + }[name] t = tensors.get(probe + name + ".weight") if t is None or (int(t.shape[0]), int(t.shape[1])) != want: bad.append((name, None if t is None else list(map(int, t.shape)))) check("代表张量 (in,out) 与映射表一致", not bad, f"bad={bad}") # attn_q 的 12288 = 24 * (256 q + 256 gate) 交错 - n_q, hd_q, n_kv, hd_k = 24, 256, 4, 256 - check("attn_q 行数 = n_q*head*2（q 与 gate 每头交错）", - shapes["attn_q"][1] == n_q * hd_q * 2) - check("Qwen35FusedQKVLinear 期望 out = 12288 + 1024 + 1024 = 14336", - 12288 + 1024 + 1024 == 14336) - check("GDN in_proj_qkv 行数 = q2048 + k2048 + v6144 = 10240", - 2048 + 2048 + 6144 == shapes["attn_qkv"][1]) - check("conv 通道 = 2*head_k*n_k + head_v*n_v = 10240", - 2 * 128 * 16 + 128 * 48 == 10240) + n_q, hd_q = 24, 256 + check( + "attn_q 行数 = n_q*head*2（q 与 gate 每头交错）", + shapes["attn_q"][1] == n_q * hd_q * 2, + ) + check( + "Qwen35FusedQKVLinear 期望 out = 12288 + 1024 + 1024 = 14336", + 12288 + 1024 + 1024 == 14336, + ) + check( + "GDN in_proj_qkv 行数 = q2048 + k2048 + v6144 = 10240", + 2048 + 2048 + 6144 == shapes["attn_qkv"][1], + ) + check( + "conv 通道 = 2*head_k*n_k + head_v*n_v = 10240", + 2 * 128 * 16 + 128 * 48 == 10240, + ) mtp = [n for n in tensors if n.startswith("blk.64.")] nextn = [n for n in tensors if ".nextn." in n] - check("MTP 丢弃规则 = 整块 blk.64.*（不止 .nextn.*，包含完整一层）", - len(mtp) == 15 and len(nextn) == 4, - f"blk.64.*={len(mtp)} 个（其中 .nextn.* 仅 {len(nextn)} 个）") + check( + "MTP 丢弃规则 = 整块 blk.64.*（不止 .nextn.*，包含完整一层）", + len(mtp) == 15 and len(nextn) == 4, + f"blk.64.*={len(mtp)} 个（其中 .nextn.* 仅 {len(nextn)} 个）", + ) max_blk = max(int(n.split(".")[1]) for n in tensors if n.startswith("blk.")) - check("块号集合 = 0..64（64 主层 + 1 MTP 层，无其它残留）", - max_blk == 64 and len({int(n.split(".")[1]) for n in tensors if n.startswith("blk.")}) == 65, - f"max_blk={max_blk}") + check( + "块号集合 = 0..64（64 主层 + 1 MTP 层，无其它残留）", + max_blk == 64 + and len({int(n.split(".")[1]) for n in tensors if n.startswith("blk.")}) == 65, + f"max_blk={max_blk}", + ) # --------------------------------------------------------------------------- @@ -429,68 +555,98 @@ def section_d(reader) -> None: def section_f(reader) -> None: print("\n== F. 打包器字节核算 ==") - GiB = 2 ** 30 + GiB = 2**30 tensors = {t.name: t for t in reader.tensors} # 核算必须由映射表驱动：之前本脚本自写一套分桶，把 7 个 IQ4 张量当成“反量化”、 # 把实为 Q8_0 的 ssm_alpha/ssm_beta（框架不能量化它们）当成 blob，两处失真共 # 高估 0.70 GiB。单一事实源 = gguf_mapping.build_plan(REAL)。 import gguf_mapping as M + plan = M.build_plan(M.REAL) _tn = {int(v.value): str(v.name) for v in QType} - M.apply_v1_exceptions(plan, {n: _tn[int(t.tensor_type)] for n, t in tensors.items()}) + M.apply_v1_exceptions( + plan, {n: _tn[int(t.tensor_type)] for n, t in tensors.items()} + ) blob_src = {e.gguf for e in plan if e.blob and e.gguf in tensors} dense_e = [e for e in plan if not e.blob] bucket = collections.Counter() cnt = collections.Counter() for n in blob_src: - bucket[f"U8 blob {QType(int(tensors[n].tensor_type)).name}"] += int(tensors[n].n_bytes) + bucket[f"U8 blob {QType(int(tensors[n].tensor_type)).name}"] += int( + tensors[n].n_bytes + ) cnt[f"U8 blob {QType(int(tensors[n].tensor_type)).name}"] += 1 - d_emb = sum(int(np.prod(e.shape)) * 2 for e in dense_e - if e.gguf in ("token_embd.weight", "output.weight")) - d_other = sum(int(np.prod(e.shape)) * 2 for e in dense_e - if e.gguf not in ("token_embd.weight", "output.weight")) + d_emb = sum( + int(np.prod(e.shape)) * 2 + for e in dense_e + if e.gguf in ("token_embd.weight", "output.weight") + ) + d_other = sum( + int(np.prod(e.shape)) * 2 + for e in dense_e + if e.gguf not in ("token_embd.weight", "output.weight") + ) bucket["BF16 稠密(emb/lm_head)"] = d_emb cnt["BF16 稠密(emb/lm_head)"] = 2 bucket["BF16 稠密(其余稠密化条目)"] = d_other cnt["BF16 稠密(其余稠密化条目)"] = len(dense_e) - 2 - bucket["丢弃(MTP)"] = sum(int(t.n_bytes) for n, t in tensors.items() - if n.startswith("blk.64.")) + bucket["丢弃(MTP)"] = sum( + int(t.n_bytes) for n, t in tensors.items() if n.startswith("blk.64.") + ) cnt["丢弃(MTP)"] = sum(1 for n in tensors if n.startswith("blk.64.")) total = sum(v / GiB for k, v in bucket.items() if k != "丢弃(MTP)") for k in sorted(bucket): print(f" {k:26s} {bucket[k] / GiB:8.3f} GiB ({cnt[k]:4d} 条目)") - print(f" {'-'*52}") + print(f" {'-' * 52}") print(f" v1 加载后权重合计 {total:8.3f} GiB") - check("v1 权重合计 ≤ 24.0 GiB（单卡 32607 MiB 可容纳权重+KV+激活）", - total <= 24.0, f"total={total:.3f} GiB") - check("MTP 丢弃量 < 0.4 GiB（不影响预算）", - bucket["丢弃(MTP)"] / GiB < 0.4, f"{bucket['丢弃(MTP)'] / GiB:.3f} GiB") - check("v1 blob 桶恰好只含阶段 3 实现的 4 种类型", - {k.replace("U8 blob ", "") for k in bucket if k.startswith("U8 blob ")} - == set(M.NATIVE_BLOB_TYPES), - f"{sorted(k for k in bucket if k.startswith('U8'))}") - check("blob 条目数与映射表一致", - sum(cnt[k] for k in bucket if k.startswith("U8")) - == len({e.gguf for e in plan if e.blob}), - f"{sum(cnt[k] for k in bucket if k.startswith('U8'))}") + check( + "v1 权重合计 ≤ 24.0 GiB（单卡 32607 MiB 可容纳权重+KV+激活）", + total <= 24.0, + f"total={total:.3f} GiB", + ) + check( + "MTP 丢弃量 < 0.4 GiB（不影响预算）", + bucket["丢弃(MTP)"] / GiB < 0.4, + f"{bucket['丢弃(MTP)'] / GiB:.3f} GiB", + ) + check( + "v1 blob 桶恰好只含阶段 3 实现的 4 种类型", + {k.replace("U8 blob ", "") for k in bucket if k.startswith("U8 blob ")} + == set(M.NATIVE_BLOB_TYPES), + f"{sorted(k for k in bucket if k.startswith('U8'))}", + ) + check( + "blob 条目数与映射表一致", + sum(cnt[k] for k in bucket if k.startswith("U8")) + == len({e.gguf for e in plan if e.blob}), + f"{sum(cnt[k] for k in bucket if k.startswith('U8'))}", + ) emb, out = tensors["token_embd.weight"], tensors["output.weight"] - check("token_embd / output 也是量化的（Q6_K / Q8_0），v1 必须反量化它们", - int(emb.tensor_type) == int(QType.Q6_K) and int(out.tensor_type) == int(QType.Q8_0), - f"emb={emb.tensor_type} out={out.tensor_type}") - check("emb/output 均为 [hidden, vocab] 且 vocab 与元数据一致", - list(map(int, emb.shape)) == list(map(int, out.shape)) == [5120, 248320] - and len(gguf_meta(reader, "tokenizer.ggml.tokens")) == 248320, - f"shape={list(map(int, emb.shape))}") - print(" -> 阶段 6 可选项：emb 走 Q6_K 行 gather-dequant、lm_head 走 linear_gguf(Q8_0)，" - f"可再省 ≈ {(d_emb - (emb.n_bytes + out.n_bytes)) / GiB:.2f} GiB") + check( + "token_embd / output 也是量化的（Q6_K / Q8_0），v1 必须反量化它们", + int(emb.tensor_type) == int(QType.Q6_K) + and int(out.tensor_type) == int(QType.Q8_0), + f"emb={emb.tensor_type} out={out.tensor_type}", + ) + check( + "emb/output 均为 [hidden, vocab] 且 vocab 与元数据一致", + list(map(int, emb.shape)) == list(map(int, out.shape)) == [5120, 248320] + and len(gguf_meta(reader, "tokenizer.ggml.tokens")) == 248320, + f"shape={list(map(int, emb.shape))}", + ) + print( + " -> 阶段 6 可选项：emb 走 Q6_K 行 gather-dequant、lm_head 走 linear_gguf(Q8_0)，" + f"可再省 ≈ {(d_emb - (emb.n_bytes + out.n_bytes)) / GiB:.2f} GiB" + ) # --------------------------------------------------------------------------- # E. 元数据 -> config.json # --------------------------------------------------------------------------- + def section_e(reader) -> None: print("\n== E. 元数据与 config.json 依据 ==") @@ -502,37 +658,54 @@ def kv(suffix, idx=0): base = float(kv("rope.freq_base")) eps = float(kv("attention.layer_norm_rms_epsilon")) head_dim = int(kv("attention.key_length")) - check("head_dim = key_length = value_length = 256", - head_dim == int(kv("attention.value_length")) == 256) - check("partial rotary: dimension_count=64, head_dim=256 -> factor 0.25", - dim_cnt == 64 and dim_cnt * 4 == head_dim, f"dimension_count={dim_cnt}") - check("mrope sections [11,11,10,0] 之和 = 32 = dimension_count/2", - sum(rope_secs) == dim_cnt // 2, f"sections={rope_secs}") + check( + "head_dim = key_length = value_length = 256", + head_dim == int(kv("attention.value_length")) == 256, + ) + check( + "partial rotary: dimension_count=64, head_dim=256 -> factor 0.25", + dim_cnt == 64 and dim_cnt * 4 == head_dim, + f"dimension_count={dim_cnt}", + ) + check( + "mrope sections [11,11,10,0] 之和 = 32 = dimension_count/2", + sum(rope_secs) == dim_cnt // 2, + f"sections={rope_secs}", + ) check("rope_theta = 1e7", base == 1e7, f"base={base}") - check("mtp 层数声明为 1（与 block_count=65 = 64+1 一致）", - int(kv("nextn_predict_layers")) == 1) + check( + "mtp 层数声明为 1（与 block_count=65 = 64+1 一致）", + int(kv("nextn_predict_layers")) == 1, + ) n_k = int(kv("ssm.group_count")) inner = int(kv("ssm.inner_size")) st = int(kv("ssm.state_size")) dt = int(kv("ssm.time_step_rank")) - check("ssm: inner 6144 / group 16 / state 128 / time_step_rank 48 / conv 4", - (inner, n_k, st, dt, int(kv("ssm.conv_kernel"))) == (6144, 16, 128, 48, 4)) - check("value heads = inner/state = 48 = time_step_rank（两路推导一致）", - inner // st == dt == 48, f"inner/state={inner // st} time_step_rank={dt}") - check("num_k_heads * state = 2048 = q/k 段长度", - n_k * st == 2048) + check( + "ssm: inner 6144 / group 16 / state 128 / time_step_rank 48 / conv 4", + (inner, n_k, st, dt, int(kv("ssm.conv_kernel"))) == (6144, 16, 128, 48, 4), + ) + check( + "value heads = inner/state = 48 = time_step_rank（两路推导一致）", + inner // st == dt == 48, + f"inner/state={inner // st} time_step_rank={dt}", + ) + check("num_k_heads * state = 2048 = q/k 段长度", n_k * st == 2048) vocab = len(gguf_meta(reader, "tokenizer.ggml.tokens")) - print(f" arch={gguf_meta(reader, 'architecture')[0]!r} " - f"name={gguf_meta(reader, 'name')[0]!r} rms_eps={eps:g} " - f"ctx={int(kv('context_length'))} vocab={vocab} " - f"heads={int(kv('attention.head_count'))}/{int(kv('attention.head_count_kv'))} " - f"hidden={int(kv('embedding_length'))} ffn={int(kv('feed_forward_length'))}") + print( + f" arch={gguf_meta(reader, 'architecture')[0]!r} " + f"name={gguf_meta(reader, 'name')[0]!r} rms_eps={eps:g} " + f"ctx={int(kv('context_length'))} vocab={vocab} " + f"heads={int(kv('attention.head_count'))}/{int(kv('attention.head_count_kv'))} " + f"hidden={int(kv('embedding_length'))} ffn={int(kv('feed_forward_length'))}" + ) def main() -> int: ap = argparse.ArgumentParser() - ap.add_argument("--gguf", default="/home/liuxd/models/Qwen3.8-27B-GGUF/" - "Qwen3.8-27B-UD-Q6_K.gguf") + ap.add_argument( + "--gguf", default="/home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf" + ) args = ap.parse_args() print(f"审计对象：{args.gguf}\n大小：{os.path.getsize(args.gguf):,} bytes") reader = GGUFReader(args.gguf) diff --git a/scripts/gguf_routeb_blocks_probe.cpp b/scripts/gguf_routeb_blocks_probe.cpp index d0f57aadc..797a8f2fe 100644 --- a/scripts/gguf_routeb_blocks_probe.cpp +++ b/scripts/gguf_routeb_blocks_probe.cpp @@ -68,8 +68,7 @@ int main(int argc, char **argv) { } const size_t n_f32 = f32.size() * sizeof(float); const size_t n_bf16 = bf16.size() * sizeof(uint16_t); - const bool ok = std::fwrite(f32.data(), 1, n_f32, o1) == n_f32 && - std::fwrite(bf16.data(), 1, n_bf16, o2) == n_bf16; + const bool ok = std::fwrite(f32.data(), 1, n_f32, o1) == n_f32 && std::fwrite(bf16.data(), 1, n_bf16, o2) == n_bf16; std::fclose(o1); std::fclose(o2); if (!ok) { diff --git a/scripts/gguf_routeb_blocks_probe.cu b/scripts/gguf_routeb_blocks_probe.cu index 9ada52ba7..e78f74841 100644 --- a/scripts/gguf_routeb_blocks_probe.cu +++ b/scripts/gguf_routeb_blocks_probe.cu @@ -18,26 +18,30 @@ __global__ void decode_f32_kernel(int32_t type, const uint8_t *blk, int64_t n_blocks, int32_t bytes, int32_t elems, float *out) { const int64_t i = (int64_t)blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n_blocks) return; + if (i >= n_blocks) { + return; + } ggml_blocks::decode_blocks(type, blk + (int64_t)i * bytes, 1, out + i * elems); } __global__ void decode_bf16_kernel(int32_t type, const uint8_t *blk, int64_t n_blocks, int32_t bytes, int32_t elems, uint16_t *out) { const int64_t i = (int64_t)blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n_blocks) return; + if (i >= n_blocks) { + return; + } ggml_blocks::decode_blocks_bf16(type, blk + (int64_t)i * bytes, 1, out + i * elems); } -#define CUDA_CHECK(call) \ - do { \ - cudaError_t err__ = (call); \ - if (err__ != cudaSuccess) { \ - std::fprintf(stderr, "probe cuda: %s failed: %s\n", #call, \ - cudaGetErrorString(err__)); \ - return 5; \ - } \ +#define CUDA_CHECK(call) \ + do { \ + cudaError_t err__ = (call); \ + if (err__ != cudaSuccess) { \ + std::fprintf(stderr, "probe cuda: %s failed: %s\n", #call, \ + cudaGetErrorString(err__)); \ + return 5; \ + } \ } while (0) int main(int argc, char **argv) { @@ -111,8 +115,7 @@ int main(int argc, char **argv) { } const size_t n_f32 = h_f32.size() * sizeof(float); const size_t n_bf16 = h_bf16.size() * sizeof(uint16_t); - const bool ok = std::fwrite(h_f32.data(), 1, n_f32, o1) == n_f32 && - std::fwrite(h_bf16.data(), 1, n_bf16, o2) == n_bf16; + const bool ok = std::fwrite(h_f32.data(), 1, n_f32, o1) == n_f32 && std::fwrite(h_bf16.data(), 1, n_bf16, o2) == n_bf16; std::fclose(o1); std::fclose(o2); if (!ok) { diff --git a/scripts/gguf_routeb_blocks_ref.py b/scripts/gguf_routeb_blocks_ref.py index db56508b6..aa0d09dd6 100644 --- a/scripts/gguf_routeb_blocks_ref.py +++ b/scripts/gguf_routeb_blocks_ref.py @@ -47,14 +47,15 @@ _INFINICORE = os.environ.get("INFINICORE_DIR", "/home/liuxd/InfiniCore") sys.path.insert(0, os.path.join(_LLAMA_CPP, "gguf-py")) -import gguf.quants as gq # noqa: E402 -from gguf.constants import GGML_QUANT_SIZES, GGMLQuantizationType as Q # noqa: E402 +import gguf.quants as gq # noqa: E402 +from gguf.constants import GGML_QUANT_SIZES # noqa: E402 +from gguf.constants import GGMLQuantizationType as Q # noqa: E402 HEADER_DIR = os.path.join(_INFINICORE, "src", "infiniop", "ops", "linear_gguf") PROBE_CPP = os.path.join(_HERE, "gguf_routeb_blocks_probe.cpp") PROBE_CU = os.path.join(_HERE, "gguf_routeb_blocks_probe.cu") -TYPES = (8, 12, 13, 14) # 与 pack_report.json 的 blob_type_ids 一致 +TYPES = (8, 12, 13, 14) # 与 pack_report.json 的 blob_type_ids 一致 QK_K, QK8_0 = 256, 32 TYPE_SIZE = {t: GGML_QUANT_SIZES[Q(t)][1] for t in TYPES} BLOCK_SIZE = {t: GGML_QUANT_SIZES[Q(t)][0] for t in TYPES} @@ -93,13 +94,18 @@ def half_to_float(bits): exp = (bits >> np.uint32(10)) & np.uint32(0x1F) mant = bits & np.uint32(0x3FF) out = np.zeros(bits.shape, np.uint32) - zneg = (exp == 0) & (mant == 0) # ±0：符号位必须留住，否则 -0.0 被写成正零 + zneg = (exp == 0) & (mant == 0) # ±0：符号位必须留住，否则 -0.0 被写成正零 out[zneg] = sign[zneg] norm = (exp != 0) & (exp != 31) - out[norm] = sign[norm] | ((exp[norm] + np.uint32(112)) << np.uint32(23)) | ( - mant[norm] << np.uint32(13)) + out[norm] = ( + sign[norm] + | ((exp[norm] + np.uint32(112)) << np.uint32(23)) + | (mant[norm] << np.uint32(13)) + ) special = exp == 31 - out[special] = sign[special] | np.uint32(0x7F800000) | (mant[special] << np.uint32(13)) + out[special] = ( + sign[special] | np.uint32(0x7F800000) | (mant[special] << np.uint32(13)) + ) sub = (exp == 0) & (mant != 0) if sub.any(): m = mant[sub].astype(np.int64) @@ -110,8 +116,9 @@ def half_to_float(bits): break m[need] <<= 1 e[need] -= 1 - out[sub] = (sign[sub].astype(np.int64) | ((e + 127) << 23) - | ((m & 0x3FF) << 13)).astype(np.uint32) + out[sub] = ( + sign[sub].astype(np.int64) | ((e + 127) << 23) | ((m & 0x3FF) << 13) + ).astype(np.uint32) return out.view(np.float32) @@ -145,7 +152,7 @@ def ref_q8_0(blk): nb = blk.shape[0] d = half_to_float(_u16_le(blk[:, 0], blk[:, 1])).reshape(nb, 1) q = blk[:, 2:34].view(np.int8).astype(np.float32) - return q * d # C: qs[j] * d + return q * d # C: qs[j] * d def ref_q4_K(blk): @@ -169,8 +176,8 @@ def ref_q5_K(blk): m_eff = (dmin[:, None] * m.astype(np.float32)).reshape(nb, 8, 1) qs = blk[:, 48:176].reshape(nb, 4, 32) qh = blk[:, 16:48][:, None, :] - lo_shift = (2 * np.arange(4)).reshape(4, 1) # u1 = 1 << 2g - hi_shift = lo_shift + 1 # u2 = 2 << 2g + lo_shift = (2 * np.arange(4)).reshape(4, 1) # u1 = 1 << 2g + hi_shift = lo_shift + 1 # u2 = 2 << 2g lo = (qs & 0xF) | (((qh >> lo_shift) & 1) << 4).astype(np.uint8) hi = (qs >> 4) | (((qh >> hi_shift) & 1) << 4).astype(np.uint8) q = np.stack([lo, hi], axis=2).reshape(nb, 8, 32).astype(np.float32) @@ -181,13 +188,13 @@ def ref_q6_K(blk): nb = blk.shape[0] d = half_to_float(_u16_le(blk[:, 208], blk[:, 209])) sc = blk[:, 192:208].view(np.int8).astype(np.float32) - d_eff = d[:, None] * sc # (nb,16) 先 d*sc，同 C 结合顺序 + d_eff = d[:, None] * sc # (nb,16) 先 d*sc，同 C 结合顺序 out = np.empty((nb, QK_K), np.float32) - l = np.arange(32) - isidx = l // 16 + lane = np.arange(32) + isidx = lane // 16 for c in (0, 1): - ql = blk[:, 64 * c:64 * c + 64] - qh = blk[:, 128 + 32 * c:128 + 32 * c + 32] + ql = blk[:, 64 * c : 64 * c + 64] + qh = blk[:, 128 + 32 * c : 128 + 32 * c + 32] base = 128 * c q1 = ((ql[:, 0:32] & 0xF) | (((qh >> 0) & 3) << 4)).astype(np.int32) - 32 q2 = ((ql[:, 32:64] & 0xF) | (((qh >> 2) & 3) << 4)).astype(np.int32) - 32 @@ -196,7 +203,7 @@ def ref_q6_K(blk): for part, (q, off) in enumerate(((q1, 0), (q2, 32), (q3, 64), (q4, 96))): # C 里每处理一个 128 元素段就 `sc += 8`，所以段 1 的 scale 下标整体偏移 8 s = d_eff[:, 8 * c + isidx + 2 * part] - out[:, base + off:base + off + 32] = s * q.astype(np.float32) + out[:, base + off : base + off + 32] = s * q.astype(np.float32) return out @@ -220,14 +227,23 @@ def check_half_decode(): finite = np.isfinite(truth) ok = np.array_equal(mine[finite].view(np.uint32), truth[finite].view(np.uint32)) n_nan = int(np.isnan(truth).sum()) - ok_nan = bool(np.array_equal(np.isnan(mine), np.isnan(truth)) - and np.array_equal(np.isinf(mine) & (mine > 0), np.isinf(truth) & (truth > 0))) + ok_nan = bool( + np.array_equal(np.isnan(mine), np.isnan(truth)) + and np.array_equal(np.isinf(mine) & (mine > 0), np.isinf(truth) & (truth > 0)) + ) neq = np.flatnonzero(mine.view(np.uint32) != truth.view(np.uint32)) - check("numpy 参考的 half_to_float：有限值逐位相同（%d 个）+ NaN 仍为 NaN（%d 个）" - % (int(finite.sum()), n_nan), ok and ok_nan, - "不同 %d 个，首个 0x%04X：%s vs %s" % (neq.size, int(h[neq[0]]) if neq.size else 0, - float(mine[neq[0]]) if neq.size else 0, - float(truth[neq[0]]) if neq.size else 0)) + check( + "numpy 参考的 half_to_float：有限值逐位相同（%d 个）+ NaN 仍为 NaN（%d 个）" + % (int(finite.sum()), n_nan), + ok and ok_nan, + "不同 %d 个，首个 0x%04X：%s vs %s" + % ( + neq.size, + int(h[neq[0]]) if neq.size else 0, + float(mine[neq[0]]) if neq.size else 0, + float(truth[neq[0]]) if neq.size else 0, + ), + ) # ------------------------------------------------- 差异度量（要求逐位相同） @@ -245,11 +261,20 @@ def bitwise_diff(a, b): first = "" if n_diff: i = int(np.flatnonzero(neq)[0]) - first = ("第 %d 个非有限值以外的元素 a=%s(0x%08X) b=%s(0x%08X)" - % (i, float(ua[i]), ua[i], float(ub[i]), ub[i])) + first = "第 %d 个非有限值以外的元素 a=%s(0x%08X) b=%s(0x%08X)" % ( + i, + float(ua[i]), + ua[i], + float(ub[i]), + ub[i], + ) elif n_bad: i = int(np.flatnonzero(bad)[0]) - first = "非有限值 a=%s b=%s @flat %d" % (fa.reshape(-1)[i], fb.reshape(-1)[i], i) + first = "非有限值 a=%s b=%s @flat %d" % ( + fa.reshape(-1)[i], + fb.reshape(-1)[i], + i, + ) return n_bad, n_diff, maxabs, first @@ -270,7 +295,9 @@ def __init__(self, path): qc = cfg["quantization_config"] table = qc["ggml_types"] self.prefix = qc.get("key_prefix") or "" - idx = json.load(open(os.path.join(path, "model.safetensors.index.json")))["weight_map"] + idx = json.load(open(os.path.join(path, "model.safetensors.index.json")))[ + "weight_map" + ] self.shards = {} for name in sorted(set(idx.values())): p = os.path.join(path, name) @@ -294,8 +321,10 @@ def norm(k): continue n = norm(k) if n in self.table_norm and self.table_norm[n] != v: - raise RuntimeError("归一化后 %s 撞键且 ggml type 不同（%d vs %d）" - % (n, self.table_norm[n], v)) + raise RuntimeError( + "归一化后 %s 撞键且 ggml type 不同（%d vs %d）" + % (n, self.table_norm[n], v) + ) self.table_norm[n] = v self.n_table_blob = len(self.table_norm) @@ -307,8 +336,10 @@ def lookup(tn): if len(cands) == 1: return cands[0], "suffix" if len(cands) > 1: - raise RuntimeError("张量 %s 在表里后缀命中 %d 个键，歧义：%s" - % (tn, len(cands), sorted(cands)[:5])) + raise RuntimeError( + "张量 %s 在表里后缀命中 %d 个键，歧义：%s" + % (tn, len(cands), sorted(cands)[:5]) + ) raise RuntimeError("张量 %s 在类型表里找不到对应条目" % tn) self.blobs = {} @@ -322,16 +353,25 @@ def lookup(tn): self.matched_table_keys.add(key) t = self.table_norm[key] if t not in TYPES: - raise RuntimeError("%s 的 ggml type %d 不在路线 B 支持的 %s 里" - % (tname, t, list(TYPES))) + raise RuntimeError( + "%s 的 ggml type %d 不在路线 B 支持的 %s 里" + % (tname, t, list(TYPES)) + ) shard = os.path.join(self.path, idx[tname]) base, hdr = self.shards[shard] e = hdr[tname] if e["dtype"] != "U8" or len(e["shape"]) != 2: - raise RuntimeError("%s 应为 U8 [rows, row_bytes]，实为 %s %s" - % (tname, e["dtype"], e["shape"])) - self.blobs[tname] = (t, shard, base + e["data_offsets"][0], - int(e["shape"][1]), int(e["shape"][0])) + raise RuntimeError( + "%s 应为 U8 [rows, row_bytes]，实为 %s %s" + % (tname, e["dtype"], e["shape"]) + ) + self.blobs[tname] = ( + t, + shard, + base + e["data_offsets"][0], + int(e["shape"][1]), + int(e["shape"][0]), + ) # 表里说自己是 blob、但产物里没有对应 weight_bytes 张量的条目（应为 0） self.orphan_table_keys = sorted(set(self.table_norm) - self.matched_table_keys) @@ -350,10 +390,14 @@ def sample(self, t, want, rng): _t, shard, base, row_bytes, nrows = self.blobs[name] bpr = row_bytes // ts if bpr * ts != row_bytes: - raise RuntimeError("%s 的 row_bytes=%d 不是 block_size %d 的整数倍" - % (name, row_bytes, ts)) + raise RuntimeError( + "%s 的 row_bytes=%d 不是 block_size %d 的整数倍" + % (name, row_bytes, ts) + ) rows_needed = int(np.ceil(per_name / bpr)) - rows = np.sort(rng.choice(nrows, size=min(rows_needed, nrows), replace=False)) + rows = np.sort( + rng.choice(nrows, size=min(rows_needed, nrows), replace=False) + ) if shard not in handles: handles[shard] = open(shard, "rb") fh = handles[shard] @@ -378,25 +422,29 @@ def sample(self, t, want, rng): def edge_blocks(t, rng, n_random=2048): """手造边界 block：全 0、全 FF、次正规 d、scale 极值，再加有限值随机块。""" ts = TYPE_SIZE[t] - rows = [np.zeros(ts, np.uint8), np.full(ts, 0xFF, np.uint8), - np.full(ts, 0x00, np.uint8), np.full(ts, 0x01, np.uint8)] + rows = [ + np.zeros(ts, np.uint8), + np.full(ts, 0xFF, np.uint8), + np.full(ts, 0x00, np.uint8), + np.full(ts, 0x01, np.uint8), + ] b = np.full(ts, 0xFF, np.uint8) b[:] = 0 - if t == 8: # d = 最小次正规 half，qs 极值 + if t == 8: # d = 最小次正规 half，qs 极值 b[0:2] = [0x01, 0x00] - b[2:] = 0x80 # int8 -128 + b[2:] = 0x80 # int8 -128 rows.append(b.copy()) - b[2:] = 0x7F # int8 +127 + b[2:] = 0x7F # int8 +127 rows.append(b.copy()) - elif t in (12, 13): # d / dmin 次正规，6-bit scale/min 全 63 + elif t in (12, 13): # d / dmin 次正规，6-bit scale/min 全 63 b[0:2] = [0x01, 0x00] - b[2:4] = [0xFF, 0x00] # dmin = 1023 * 2^-24 + b[2:4] = [0xFF, 0x00] # dmin = 1023 * 2^-24 b[4:16] = 0xFF rows.append(b.copy()) - b[0:2] = [0xFE, 0x7B] # d = 65534（最大有限 half） + b[0:2] = [0xFE, 0x7B] # d = 65534（最大有限 half） b[2:4] = [0x00, 0x00] rows.append(b.copy()) - else: # Q6_K：int8 scale = -128 / +127 + else: # Q6_K：int8 scale = -128 / +127 b[192:208] = 0x80 b[208:210] = [0x01, 0x00] rows.append(b.copy()) @@ -407,7 +455,7 @@ def edge_blocks(t, rng, n_random=2048): for _ in range(n_random): r = rng.integers(0, 256, ts, dtype=np.uint8) for off in _half_offsets(t): - h = int(rng.integers(0, 0x7BFF + 1)) # exp != 0x1F + h = int(rng.integers(0, 0x7BFF + 1)) # exp != 0x1F r[off], r[off + 1] = h & 0xFF, (h >> 8) & 0xFF rows.append(r) return np.stack(rows) @@ -446,25 +494,38 @@ def half_sweep_blocks(t, rng): # ------------------------------------------------------------ probe 编译/调用 def build_probe(src, out, compiler, extra=()): - cmd = [compiler, "-O2", "-std=c++17", "-I", HEADER_DIR, src, "-o", out] + list(extra) + cmd = [compiler, "-O2", "-std=c++17", "-I", HEADER_DIR, src, "-o", out] + list( + extra + ) p = subprocess.run(cmd, capture_output=True, text=True) if p.returncode != 0: - raise RuntimeError("编译失败：%s\n%s" % (" ".join(cmd), (p.stderr or p.stdout)[-4000:])) + raise RuntimeError( + "编译失败：%s\n%s" % (" ".join(cmd), (p.stderr or p.stdout)[-4000:]) + ) return out def run_probe(binary, t, blocks, workdir, tag): - ts, bs = TYPE_SIZE[t], BLOCK_SIZE[t] + bs = BLOCK_SIZE[t] inbin = os.path.join(workdir, "%s_t%d.in" % (tag, t)) f32bin = os.path.join(workdir, "%s_t%d.f32" % (tag, t)) bf16bin = os.path.join(workdir, "%s_t%d.bf16" % (tag, t)) np.ascontiguousarray(blocks).tofile(inbin) - p = subprocess.run([binary, str(t), str(blocks.shape[0]), inbin, f32bin, bf16bin], - capture_output=True, text=True) + p = subprocess.run( + [binary, str(t), str(blocks.shape[0]), inbin, f32bin, bf16bin], + capture_output=True, + text=True, + ) if p.returncode != 0: - raise RuntimeError("%s 失败（type=%d, rc=%d）：%s" - % (os.path.basename(binary), t, p.returncode, - (p.stderr or p.stdout).strip()[-2000:])) + raise RuntimeError( + "%s 失败（type=%d, rc=%d）：%s" + % ( + os.path.basename(binary), + t, + p.returncode, + (p.stderr or p.stdout).strip()[-2000:], + ) + ) f32 = np.fromfile(f32bin, np.float32).reshape(-1, bs) bf16 = np.fromfile(bf16bin, np.uint16).reshape(-1, bs) m = re.search(r"elems=(\d+)", p.stdout) @@ -473,8 +534,12 @@ def run_probe(binary, t, blocks, workdir, tag): def main(): ap = argparse.ArgumentParser() - ap.add_argument("--model-path", default="/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8") - ap.add_argument("--blocks", type=int, default=20000, help="每种类型取多少真实 block") + ap.add_argument( + "--model-path", default="/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8" + ) + ap.add_argument( + "--blocks", type=int, default=20000, help="每种类型取多少真实 block" + ) ap.add_argument("--workdir", default="/home/liuxd/tmp_routeb/blocks31") ap.add_argument("--cxx", default=os.environ.get("CXX", "g++")) ap.add_argument("--nvcc", default=os.environ.get("CUDACXX", "nvcc")) @@ -484,33 +549,54 @@ def main(): rng = np.random.default_rng(args.seed) os.makedirs(args.workdir, exist_ok=True) - print("产物：%s\n头文件：%s\n临时目录：%s\n每类型真实 block 目标：%d" - % (args.model_path, os.path.join(HEADER_DIR, "ggml_blocks.h"), args.workdir, - args.blocks)) + print( + "产物：%s\n头文件：%s\n临时目录：%s\n每类型真实 block 目标：%d" + % ( + args.model_path, + os.path.join(HEADER_DIR, "ggml_blocks.h"), + args.workdir, + args.blocks, + ) + ) print("\n[0] 参考实现自检") check_half_decode() art = Artifact(args.model_path) n_blob_total = len(art.blobs) - print("产物 blob 张量 %d 个（key_prefix=%r），按类型：%s" - % (n_blob_total, art.prefix, {t: len(art.type_names(t)) for t in TYPES})) - check("类型表 blob 条目与产物 weight_bytes 张量双向对平（表 %d / 张量 %d，孤儿 %d，" - "匹配形态 %s）" - % (art.n_table_blob, n_blob_total, len(art.orphan_table_keys), dict(art.match_form)), - art.n_table_blob == n_blob_total and not art.orphan_table_keys, - "孤儿键：%s" % art.orphan_table_keys[:5]) + print( + "产物 blob 张量 %d 个（key_prefix=%r），按类型：%s" + % (n_blob_total, art.prefix, {t: len(art.type_names(t)) for t in TYPES}) + ) + check( + "类型表 blob 条目与产物 weight_bytes 张量双向对平（表 %d / 张量 %d，孤儿 %d，" + "匹配形态 %s）" + % ( + art.n_table_blob, + n_blob_total, + len(art.orphan_table_keys), + dict(art.match_form), + ), + art.n_table_blob == n_blob_total and not art.orphan_table_keys, + "孤儿键：%s" % art.orphan_table_keys[:5], + ) print("\n[1] 编译 probe driver") - host_bin = build_probe(PROBE_CPP, os.path.join(args.workdir, "blocks_probe_host"), args.cxx) + host_bin = build_probe( + PROBE_CPP, os.path.join(args.workdir, "blocks_probe_host"), args.cxx + ) print(" host driver ok：%s" % host_bin) dev_bin = None if args.no_cuda: skip("cuda driver 编译", "--no-cuda") else: try: - dev_bin = build_probe(PROBE_CU, os.path.join(args.workdir, "blocks_probe_cuda"), - args.nvcc, extra=["-x", "cu"]) + dev_bin = build_probe( + PROBE_CU, + os.path.join(args.workdir, "blocks_probe_cuda"), + args.nvcc, + extra=["-x", "cu"], + ) print(" cuda driver ok：%s" % dev_bin) except Exception as e: print(" ! %s" % e) @@ -521,41 +607,60 @@ def main(): name = Q(t).name want = args.blocks blocks, touched = art.sample(t, want, rng) - if not check("%s 采到 %d 个真实 block（目标 %d，覆盖 %d 个张量）" - % (name, blocks.shape[0], want, len(touched)), - blocks.shape[0] >= min(want, 100)): + if not check( + "%s 采到 %d 个真实 block（目标 %d，覆盖 %d 个张量）" + % (name, blocks.shape[0], want, len(touched)), + blocks.shape[0] >= min(want, 100), + ): continue ref = REF[t](np.ascontiguousarray(blocks)) py = gguf_py_dequant(t, blocks) n_bad, n_diff, maxabs, first = bitwise_diff(ref, py) - check("%s numpy 参考 vs gguf-py（%d block 逐位相同）" - % (name, blocks.shape[0]), - n_diff == 0 and n_bad == 0, - "差异 %d/%d 元素，非有限 %d，max|Δ|=%.3g，首个：%s" - % (n_diff, ref.size, n_bad, maxabs, first)) + check( + "%s numpy 参考 vs gguf-py（%d block 逐位相同）" % (name, blocks.shape[0]), + n_diff == 0 and n_bad == 0, + "差异 %d/%d 元素，非有限 %d，max|Δ|=%.3g，首个：%s" + % (n_diff, ref.size, n_bad, maxabs, first), + ) try: h_f32, h_bf16, elems = run_probe(host_bin, t, blocks, args.workdir, "host") except Exception as e: check("%s host probe 运行" % name, False, str(e)) continue - check("%s 头的 block_elems 与 GGML_QUANT_SIZES 一致（%d == %d）" - % (name, elems, BLOCK_SIZE[t]), elems == BLOCK_SIZE[t]) + check( + "%s 头的 block_elems 与 GGML_QUANT_SIZES 一致（%d == %d）" + % (name, elems, BLOCK_SIZE[t]), + elems == BLOCK_SIZE[t], + ) n_bad, n_diff, maxabs, first = bitwise_diff(h_f32, ref) - check("%s 头(host) fp32 vs numpy 参考（%d 元素逐位相同）" - % (name, h_f32.size), n_diff == 0 and n_bad == 0, - "差异 %d，首个：%s" % (n_diff, first)) + check( + "%s 头(host) fp32 vs numpy 参考（%d 元素逐位相同）" % (name, h_f32.size), + n_diff == 0 and n_bad == 0, + "差异 %d，首个：%s" % (n_diff, first), + ) want_bf16 = float_to_bf16_bits(ref) same_own = np.array_equal(h_bf16, want_bf16) - check("%s 头(host) bf16 vs numpy RNE 舍入" % name, same_own, - "首个差异 %s" % (np.flatnonzero(h_bf16 != want_bf16)[:5],)) + check( + "%s 头(host) bf16 vs numpy RNE 舍入" % name, + same_own, + "首个差异 %s" % (np.flatnonzero(h_bf16 != want_bf16)[:5],), + ) try: import torch - tv = torch.from_numpy(np.ascontiguousarray(ref)).to(torch.bfloat16) \ - .view(torch.uint16).numpy() - check("%s 头(host) bf16 vs torch .to(bfloat16)" % name, np.array_equal(h_bf16, tv)) + + tv = ( + torch.from_numpy(np.ascontiguousarray(ref)) + .to(torch.bfloat16) + .view(torch.uint16) + .numpy() + ) + check( + "%s 头(host) bf16 vs torch .to(bfloat16)" % name, + np.array_equal(h_bf16, tv), + ) except Exception as e: skip("%s bf16 vs torch" % name, str(e).splitlines()[0][:80]) @@ -565,10 +670,14 @@ def main(): except Exception as e: check("%s cuda probe 运行" % name, False, str(e)) else: - check("%s 头(cuda) fp32 vs 头(host) 逐位相同" % name, - np.array_equal(d_f32.view(np.uint32), h_f32.view(np.uint32))) - check("%s 头(cuda) bf16 vs 头(host) 逐位相同" % name, - np.array_equal(d_bf16, h_bf16)) + check( + "%s 头(cuda) fp32 vs 头(host) 逐位相同" % name, + np.array_equal(d_f32.view(np.uint32), h_f32.view(np.uint32)), + ) + check( + "%s 头(cuda) bf16 vs 头(host) 逐位相同" % name, + np.array_equal(d_bf16, h_bf16), + ) eb = edge_blocks(t, rng) eref = REF[t](np.ascontiguousarray(eb)) @@ -577,35 +686,54 @@ def main(): n_bad_e = int((~np.isfinite(eref) | ~np.isfinite(epy)).sum()) h_e, _, _ = run_probe(host_bin, t, eb, args.workdir, "host_edge") _, n_diff_h, maxabs_h, first_h = bitwise_diff(h_e, eref) - check("%s 边界块（%d 个）numpy vs gguf-py 逐位相同" % (name, eb.shape[0]), - n_diff_e == 0, "差异 %d，非有限 %d，max|Δ|=%.3g，首个：%s" - % (n_diff_e, n_bad_e, maxabs_e, first_e)) - check("%s 边界块（%d 个）头(host) vs numpy 逐位相同" % (name, eb.shape[0]), - n_diff_h == 0, "差异 %d，max|Δ|=%.3g，首个：%s" % (n_diff_h, maxabs_h, first_h)) + check( + "%s 边界块（%d 个）numpy vs gguf-py 逐位相同" % (name, eb.shape[0]), + n_diff_e == 0, + "差异 %d，非有限 %d，max|Δ|=%.3g，首个：%s" + % (n_diff_e, n_bad_e, maxabs_e, first_e), + ) + check( + "%s 边界块（%d 个）头(host) vs numpy 逐位相同" % (name, eb.shape[0]), + n_diff_h == 0, + "差异 %d，max|Δ|=%.3g，首个：%s" % (n_diff_h, maxabs_h, first_h), + ) print("\n[3] half 字段全域扫描（每个字段 65536 个位模式）") for t in TYPES: name = Q(t).name sb, _marks = half_sweep_blocks(t, rng) - with np.errstate(all="ignore"): # 扫描里故意喂 inf/nan half，告警与判据无关 + with np.errstate(all="ignore"): # 扫描里故意喂 inf/nan half，告警与判据无关 sref = REF[t](np.ascontiguousarray(sb)) sh, _, _ = run_probe(host_bin, t, sb, args.workdir, "host_sweep") n_bad, n_diff, maxabs, first = bitwise_diff(sh, sref) - check("%s 头(host) vs numpy 参考：half 全域扫描 %d block 逐位相同" - % (name, sb.shape[0]), n_diff == 0, - "差异 %d/%d 元素，非有限 %d（inf/nan 乘出的正常现象），max|Δ|=%.3g，首个：%s" - % (n_diff, sh.size, n_bad, maxabs, first)) + check( + "%s 头(host) vs numpy 参考：half 全域扫描 %d block 逐位相同" + % (name, sb.shape[0]), + n_diff == 0, + "差异 %d/%d 元素，非有限 %d（inf/nan 乘出的正常现象），max|Δ|=%.3g，首个：%s" + % (n_diff, sh.size, n_bad, maxabs, first), + ) print("\n[4] 不支持的类型必须被头拒绝") inbin = os.path.join(args.workdir, "reject.in") np.zeros(TYPE_SIZE[8], np.uint8).tofile(inbin) - p = subprocess.run([host_bin, "10", "1", inbin, - os.path.join(args.workdir, "reject.f32"), - os.path.join(args.workdir, "reject.bf16")], - capture_output=True, text=True) - check("头对 ggml type 10（TQ1_0，非本头范围）返回拒绝", - p.returncode == 3 and "no decoder" in (p.stderr + p.stdout), - "rc=%d stderr=%s" % (p.returncode, (p.stderr or p.stdout).strip()[-200:])) + p = subprocess.run( + [ + host_bin, + "10", + "1", + inbin, + os.path.join(args.workdir, "reject.f32"), + os.path.join(args.workdir, "reject.bf16"), + ], + capture_output=True, + text=True, + ) + check( + "头对 ggml type 10（TQ1_0，非本头范围）返回拒绝", + p.returncode == 3 and "no decoder" in (p.stderr + p.stdout), + "rc=%d stderr=%s" % (p.returncode, (p.stderr or p.stdout).strip()[-200:]), + ) print("\n== 结果：%d PASS / %d FAIL / %d SKIP ==" % (_PASS, _FAIL, _SKIP)) print("临时目录：%s" % args.workdir) diff --git a/scripts/gguf_routeb_compare.py b/scripts/gguf_routeb_compare.py index 4d8f9d96d..8ba49e950 100755 --- a/scripts/gguf_routeb_compare.py +++ b/scripts/gguf_routeb_compare.py @@ -34,8 +34,10 @@ def main() -> int: lmap = {x: lmap[x] for x in case_ids} imap = {x: imap[x] for x in case_ids} elif set(lmap) != set(imap): - raise ValueError("case sets differ: llama-only=%s infini-only=%s" % ( - sorted(set(lmap) - set(imap)), sorted(set(imap) - set(lmap)))) + raise ValueError( + "case sets differ: llama-only=%s infini-only=%s" + % (sorted(set(lmap) - set(imap)), sorted(set(imap) - set(lmap))) + ) cases = [] exact = 0 @@ -47,7 +49,9 @@ def main() -> int: raise ValueError("input ids differ for %s" % case_id) lt = left["runs"][0]["tokens"] rt = right["runs"][0]["tokens"] - first_difference = next((i for i, (a, b) in enumerate(zip(lt, rt)) if a != b), None) + first_difference = next( + (i for i, (a, b) in enumerate(zip(lt, rt)) if a != b), None + ) if first_difference is None and len(lt) != len(rt): first_difference = min(len(lt), len(rt)) is_exact = lt == rt @@ -55,19 +59,24 @@ def main() -> int: same = sum(a == b for a, b in zip(lt, rt)) matched += same total += max(len(lt), len(rt)) - cases.append({ - "id": case_id, - "exact_sequence_match": is_exact, - "matched_tokens": same, - "total_tokens": max(len(lt), len(rt)), - "first_difference": first_difference, - "llama_tokens": lt, - "infinilm_tokens": rt, - "llama_first_top_logprobs": left["runs"][0].get( - "first_token_top_logprobs", []), - }) - print("%-10s exact=%s first_diff=%s llama=%s infini=%s" % ( - case_id, is_exact, first_difference, lt, rt)) + cases.append( + { + "id": case_id, + "exact_sequence_match": is_exact, + "matched_tokens": same, + "total_tokens": max(len(lt), len(rt)), + "first_difference": first_difference, + "llama_tokens": lt, + "infinilm_tokens": rt, + "llama_first_top_logprobs": left["runs"][0].get( + "first_token_top_logprobs", [] + ), + } + ) + print( + "%-10s exact=%s first_diff=%s llama=%s infini=%s" + % (case_id, is_exact, first_difference, lt, rt) + ) result = { "cases": cases, @@ -82,8 +91,10 @@ def main() -> int: os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) with open(args.out, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False, indent=2) - print("RESULT exact=%d/%d token_match=%d/%d all_exact=%s" % ( - exact, len(cases), matched, total, result["all_exact"])) + print( + "RESULT exact=%d/%d token_match=%d/%d all_exact=%s" + % (exact, len(cases), matched, total, result["all_exact"]) + ) return 0 if result["all_exact"] else 1 diff --git a/scripts/gguf_routeb_first_diff.py b/scripts/gguf_routeb_first_diff.py index b13084e4a..8bd96990c 100755 --- a/scripts/gguf_routeb_first_diff.py +++ b/scripts/gguf_routeb_first_diff.py @@ -6,7 +6,6 @@ import argparse import ctypes import json -import math import os import sys import time @@ -42,8 +41,8 @@ def main() -> int: ap.add_argument("--out", required=True) args = ap.parse_args() - import numpy as np import infinicore + import numpy as np from infinilm.cache import PagedKVCacheConfig from infinilm.distributed import DistConfig from infinilm.infer_engine import InferEngine @@ -82,8 +81,7 @@ def main() -> int: "stream": False, "samplers": ["top_k", "temperature"], } - llama_response = post_json( - args.server.rstrip("/") + "/completion", llama_body) + llama_response = post_json(args.server.rstrip("/") + "/completion", llama_body) llama_probs = llama_response["completion_probabilities"][0]["top_logprobs"] load_started = time.time() @@ -92,7 +90,8 @@ def main() -> int: device=infinicore.device("cuda:0"), distributed_config=DistConfig(1), cache_config=PagedKVCacheConfig( - args.num_blocks, args.block_size, max_batch_size=1), + args.num_blocks, args.block_size, max_batch_size=1 + ), attention_backend="paged-attn", ) load_model_state_dict_by_file(engine, args.model_path, dtype=engine.dtype) @@ -103,14 +102,18 @@ def main() -> int: if engine.position_id_axes > 1: positions = [positions for _ in range(engine.position_id_axes)] tensors = { - "input_ids": infinicore.from_list([prefix], dtype=infinicore.int64).view([1, length]), + "input_ids": infinicore.from_list([prefix], dtype=infinicore.int64).view( + [1, length] + ), "position_ids": infinicore.from_list(positions, dtype=infinicore.int64), "past_kv_lengths": infinicore.from_list([0], dtype=infinicore.int32), "total_kv_lengths": infinicore.from_list([length], dtype=infinicore.int32), "input_offsets": infinicore.from_list([0, length], dtype=infinicore.int32), "cu_seqlens": infinicore.from_list([0, length], dtype=infinicore.int32), "block_tables": infinicore.from_list([[0]], dtype=infinicore.int32), - "slot_mapping": infinicore.from_list(list(range(length)), dtype=infinicore.int64), + "slot_mapping": infinicore.from_list( + list(range(length)), dtype=infinicore.int64 + ), "mamba_init_state_indices": infinicore.from_list([0], dtype=infinicore.int32), "mamba_final_state_indices": infinicore.from_list([1], dtype=infinicore.int32), } @@ -140,20 +143,29 @@ def main() -> int: bits = np.ctypeslib.as_array(bits_type.from_address(cpu_logits.data_ptr())).copy() all_logits = (bits.astype(np.uint32) << 16).view(np.float32).reshape(logits_shape) logits = all_logits.reshape(-1, logits_shape[-1])[-1] - order = np.argpartition(logits, -args.top_k)[-args.top_k:] + order = np.argpartition(logits, -args.top_k)[-args.top_k :] order = order[np.argsort(logits[order])[::-1]] max_logit = float(logits[order[0]]) - infini_top = [{"id": int(i), "logit": float(logits[i]), - "delta_from_top": float(logits[i] - max_logit)} for i in order] + infini_top = [ + { + "id": int(i), + "logit": float(logits[i]), + "delta_from_top": float(logits[i] - max_logit), + } + for i in order + ] llama_map = {int(x["id"]): float(x["logprob"]) for x in llama_probs} infini_map = {int(x["id"]): float(x["delta_from_top"]) for x in infini_top} candidate_ids = sorted(set(llama_map) | set(infini_map)) - candidate_table = [{ - "id": token_id, - "llama_logprob": llama_map.get(token_id), - "infini_delta_from_top": infini_map.get(token_id), - } for token_id in candidate_ids] + candidate_table = [ + { + "id": token_id, + "llama_logprob": llama_map.get(token_id), + "infini_delta_from_top": infini_map.get(token_id), + } + for token_id in candidate_ids + ] result = { "case_id": args.case_id, @@ -173,15 +185,27 @@ def main() -> int: os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) with open(args.out, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False, indent=2) - print("CASE=%s diff=%d prefix_len=%d llama=%d infini=%d" % ( - args.case_id, first_diff, len(prefix), result["llama_selected"], - result["infinilm_selected"])) - print("LLAMA_TOP5 %s" % [(x["id"], round(x["logprob"], 6)) - for x in llama_probs[:5]]) - print("INFINI_TOP5 %s" % [(x["id"], round(x["delta_from_top"], 6)) - for x in infini_top[:5]]) - print("FINITE=%s SHAPE=%s LOAD=%.3fs" % ( - result["infinilm_logits_finite"], result["infinilm_logits_shape"], load_s)) + print( + "CASE=%s diff=%d prefix_len=%d llama=%d infini=%d" + % ( + args.case_id, + first_diff, + len(prefix), + result["llama_selected"], + result["infinilm_selected"], + ) + ) + print( + "LLAMA_TOP5 %s" % [(x["id"], round(x["logprob"], 6)) for x in llama_probs[:5]] + ) + print( + "INFINI_TOP5 %s" + % [(x["id"], round(x["delta_from_top"], 6)) for x in infini_top[:5]] + ) + print( + "FINITE=%s SHAPE=%s LOAD=%.3fs" + % (result["infinilm_logits_finite"], result["infinilm_logits_shape"], load_s) + ) return 0 diff --git a/scripts/gguf_routeb_first_diff_batch.py b/scripts/gguf_routeb_first_diff_batch.py index c9f6a7d66..40284e1c7 100644 --- a/scripts/gguf_routeb_first_diff_batch.py +++ b/scripts/gguf_routeb_first_diff_batch.py @@ -13,8 +13,11 @@ def post_json(url: str, body: dict, timeout: int = 180) -> dict: req = urllib.request.Request( - url, data=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}, method="POST") + url, + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) with urllib.request.urlopen(req, timeout=timeout) as response: return json.load(response) @@ -31,8 +34,8 @@ def main() -> int: ap.add_argument("--out", required=True) args = ap.parse_args() - import numpy as np import infinicore + import numpy as np from infinilm.cache import PagedKVCacheConfig from infinilm.distributed import DistConfig from infinilm.infer_engine import InferEngine @@ -47,11 +50,14 @@ def main() -> int: started = time.time() engine = InferEngine( - model_path=args.model_path, device=infinicore.device("cuda:0"), + model_path=args.model_path, + device=infinicore.device("cuda:0"), distributed_config=DistConfig(1), cache_config=PagedKVCacheConfig( - args.num_blocks, args.block_size, max_batch_size=1), - attention_backend="paged-attn") + args.num_blocks, args.block_size, max_batch_size=1 + ), + attention_backend="paged-attn", + ) load_model_state_dict_by_file(engine, args.model_path, dtype=engine.dtype) load_s = time.time() - started @@ -63,13 +69,23 @@ def main() -> int: assert common == item["infinilm_tokens"][:first_diff] prefix = [int(x) for x in inputs[case_id]["input_ids"] + common] body = { - "prompt": prefix, "n_predict": 1, "temperature": 0.0, - "top_k": 1, "top_p": 1.0, "min_p": 0.0, - "typical_p": 1.0, "repeat_penalty": 1.0, - "repeat_last_n": 0, "presence_penalty": 0.0, - "frequency_penalty": 0.0, "seed": 1, "ignore_eos": True, - "cache_prompt": False, "return_tokens": True, - "n_probs": args.top_k, "stream": False, + "prompt": prefix, + "n_predict": 1, + "temperature": 0.0, + "top_k": 1, + "top_p": 1.0, + "min_p": 0.0, + "typical_p": 1.0, + "repeat_penalty": 1.0, + "repeat_last_n": 0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "seed": 1, + "ignore_eos": True, + "cache_prompt": False, + "return_tokens": True, + "n_probs": args.top_k, + "stream": False, "samplers": ["top_k", "temperature"], } llama = post_json(args.server.rstrip("/") + "/completion", body) @@ -80,26 +96,41 @@ def main() -> int: if engine.position_id_axes > 1: positions = [positions for _ in range(engine.position_id_axes)] tensors = { - "input_ids": infinicore.from_list([prefix], dtype=infinicore.int64).view([1, length]), + "input_ids": infinicore.from_list([prefix], dtype=infinicore.int64).view( + [1, length] + ), "position_ids": infinicore.from_list(positions, dtype=infinicore.int64), "past_kv_lengths": infinicore.from_list([0], dtype=infinicore.int32), "total_kv_lengths": infinicore.from_list([length], dtype=infinicore.int32), "input_offsets": infinicore.from_list([0, length], dtype=infinicore.int32), "cu_seqlens": infinicore.from_list([0, length], dtype=infinicore.int32), "block_tables": infinicore.from_list([[0]], dtype=infinicore.int32), - "slot_mapping": infinicore.from_list(list(range(length)), dtype=infinicore.int64), - "mamba_init_state_indices": infinicore.from_list([0], dtype=infinicore.int32), - "mamba_final_state_indices": infinicore.from_list([1], dtype=infinicore.int32), + "slot_mapping": infinicore.from_list( + list(range(length)), dtype=infinicore.int64 + ), + "mamba_init_state_indices": infinicore.from_list( + [0], dtype=infinicore.int32 + ), + "mamba_final_state_indices": infinicore.from_list( + [1], dtype=infinicore.int32 + ), } cpp_input = engine._build_input( - tensors["input_ids"], position_ids=tensors["position_ids"], + tensors["input_ids"], + position_ids=tensors["position_ids"], past_kv_lengths=tensors["past_kv_lengths"], total_kv_lengths=tensors["total_kv_lengths"], - input_offsets=tensors["input_offsets"], cu_seqlens=tensors["cu_seqlens"], - block_tables=tensors["block_tables"], slot_mapping=tensors["slot_mapping"], + input_offsets=tensors["input_offsets"], + cu_seqlens=tensors["cu_seqlens"], + block_tables=tensors["block_tables"], + slot_mapping=tensors["slot_mapping"], mamba_init_state_indices=tensors["mamba_init_state_indices"], mamba_final_state_indices=tensors["mamba_final_state_indices"], - sample_all_positions=False, temperature=0.0, top_k=1, top_p=1.0) + sample_all_positions=False, + temperature=0.0, + top_k=1, + top_p=1.0, + ) output = _infinilm.InferEngine.forward(engine, cpp_input) raw = infinicore.Tensor(output.logits) shape = list(raw.shape) @@ -110,50 +141,75 @@ def main() -> int: bits = np.ctypeslib.as_array(bits_type.from_address(cpu.data_ptr())).copy() logits = (bits.astype(np.uint32) << 16).view(np.float32).reshape(shape) logits = logits.reshape(-1, shape[-1])[-1] - order = np.argpartition(logits, -args.top_k)[-args.top_k:] + order = np.argpartition(logits, -args.top_k)[-args.top_k :] order = order[np.argsort(logits[order], kind="stable")[::-1]] top_logit = float(logits[order[0]]) - infini_top = [{"id": int(i), "logit": float(logits[i]), - "delta_from_top": float(logits[i] - top_logit)} for i in order] + infini_top = [ + { + "id": int(i), + "logit": float(logits[i]), + "delta_from_top": float(logits[i] - top_logit), + } + for i in order + ] llama_map = {int(x["id"]): float(x["logprob"]) for x in llama_probs} infini_map = {x["id"]: x["delta_from_top"] for x in infini_top} llama_selected = int(llama["tokens"][0]) infini_selected = int(order[0]) candidate_ids = sorted(set(llama_map) | set(infini_map)) - candidate_table = [{ - "id": token_id, "llama_logprob": llama_map.get(token_id), - "infini_delta_from_top": infini_map.get(token_id), - "infini_logit": float(logits[token_id]), - } for token_id in candidate_ids] + candidate_table = [ + { + "id": token_id, + "llama_logprob": llama_map.get(token_id), + "infini_delta_from_top": infini_map.get(token_id), + "infini_logit": float(logits[token_id]), + } + for token_id in candidate_ids + ] selected_logits = { "llama_token_infini_logit": float(logits[llama_selected]), "infini_token_infini_logit": float(logits[infini_selected]), - "infini_margin_selected_minus_llama": - float(logits[infini_selected] - logits[llama_selected]), - "llama_margin_selected_minus_infini": - float(llama_map[llama_selected] - llama_map.get(infini_selected, float("nan"))), + "infini_margin_selected_minus_llama": float( + logits[infini_selected] - logits[llama_selected] + ), + "llama_margin_selected_minus_infini": float( + llama_map[llama_selected] - llama_map.get(infini_selected, float("nan")) + ), } result = { - "case_id": case_id, "first_difference": first_diff, - "prefix_length": len(prefix), "llama_selected": llama_selected, + "case_id": case_id, + "first_difference": first_diff, + "prefix_length": len(prefix), + "llama_selected": llama_selected, "infinilm_selected": infini_selected, - "llama_top_logprobs": llama_probs, "infinilm_top_logits": infini_top, - "selected_pair": selected_logits, "candidate_table": candidate_table, + "llama_top_logprobs": llama_probs, + "infinilm_top_logits": infini_top, + "selected_pair": selected_logits, + "candidate_table": candidate_table, "infinilm_logits_shape": shape, "infinilm_logits_finite": bool(np.isfinite(logits).all()), } results.append(result) - print("%-10s diff=%2d llama=%6d infini=%6d llama_margin=%+.6f infini_margin=%+.6f" % ( - case_id, first_diff, llama_selected, infini_selected, - selected_logits["llama_margin_selected_minus_infini"], - selected_logits["infini_margin_selected_minus_llama"])) + print( + "%-10s diff=%2d llama=%6d infini=%6d llama_margin=%+.6f infini_margin=%+.6f" + % ( + case_id, + first_diff, + llama_selected, + infini_selected, + selected_logits["llama_margin_selected_minus_infini"], + selected_logits["infini_margin_selected_minus_llama"], + ) + ) report = {"load_s": round(load_s, 4), "case_count": len(results), "cases": results} os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) with open(args.out, "w", encoding="utf-8") as f: json.dump(report, f, ensure_ascii=False, indent=2) - print("RESULT cases=%d finite=%s load=%.3fs" % ( - len(results), all(x["infinilm_logits_finite"] for x in results), load_s)) + print( + "RESULT cases=%d finite=%s load=%.3fs" + % (len(results), all(x["infinilm_logits_finite"] for x in results), load_s) + ) return 0 diff --git a/scripts/gguf_routeb_gemv_check.py b/scripts/gguf_routeb_gemv_check.py index dcb3457c4..876cb14f8 100644 --- a/scripts/gguf_routeb_gemv_check.py +++ b/scripts/gguf_routeb_gemv_check.py @@ -49,14 +49,23 @@ _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -import gguf_routeb_blocks_ref as bref # noqa: E402 -from gguf_routeb_blocks_ref import (Artifact, BLOCK_SIZE, REF, # noqa: E402 - TYPE_SIZE, TYPES, check, skip) +import gguf_routeb_blocks_ref as bref # noqa: E402 +from gguf_routeb_blocks_ref import ( # noqa: E402 + BLOCK_SIZE, + REF, + TYPE_SIZE, + TYPES, + Artifact, + check, + skip, +) GEMV_DIR = os.path.join(bref.HEADER_DIR, "nvidia") PROBE_SRC = os.path.join(_HERE, "gguf_routeb_gemv_probe.cu") -MAX_M = 8 # kMaxDecodeM：M <= 8 走 gemv，M > 8 走 prefill -PREFILL_MS = "9,16,32,64,256,1024" # 9 = prefill 下边界（§1.2 第 3 条含 16/32/64/256/1024） +MAX_M = 8 # kMaxDecodeM：M <= 8 走 gemv，M > 8 走 prefill +PREFILL_MS = ( + "9,16,32,64,256,1024" # 9 = prefill 下边界（§1.2 第 3 条含 16/32/64/256/1024） +) PATH_RE = re.compile(r"path=(\w+)") T_NAME = {8: "Q8_0", 12: "Q4_K", 13: "Q5_K", 14: "Q6_K"} @@ -67,8 +76,9 @@ def reset_counters(): # --------------------------------------------------------------- bf16 位模式 def bf16_to_f32(bits): - return (np.asarray(bits, np.uint16).astype(np.uint32) - << np.uint32(16)).view(np.float32) + return (np.asarray(bits, np.uint16).astype(np.uint32) << np.uint32(16)).view( + np.float32 + ) def f32_to_bf16(x): @@ -95,8 +105,9 @@ def pick_rows(art, t, want_rows, rng, which=0): _t, shard, base, row_bytes, nrows = art.blobs[name] blocks_per_row = row_bytes // ts if blocks_per_row * ts != row_bytes or blocks_per_row < 1: - raise RuntimeError("%s 的 row_bytes=%d 不是 block_size %d 的整数倍" - % (name, row_bytes, ts)) + raise RuntimeError( + "%s 的 row_bytes=%d 不是 block_size %d 的整数倍" % (name, row_bytes, ts) + ) rows = min(want_rows, nrows) r0 = int(rng.integers(0, nrows - rows + 1)) with open(shard, "rb") as fh: @@ -108,9 +119,9 @@ def pick_rows(art, t, want_rows, rng, which=0): def dense_weights(t, W, rows, K): """numpy 参考反量化：W[rows, row_bytes] -> float32 [rows, K]。""" - ts, bs = TYPE_SIZE[t], BLOCK_SIZE[t] + ts = TYPE_SIZE[t] blocks = W.reshape(-1, ts) - dec = REF[t](blocks) # (n_blocks, bs) float32，3.1 已证逐位正确 + dec = REF[t](blocks) # (n_blocks, bs) float32，3.1 已证逐位正确 return dec.reshape(rows, K) @@ -159,30 +170,43 @@ def check_type_one(binary, art, t, name, W, K, r0, Ms, rng, workdir): Wf32 = dense_weights(t, W, n, K) # 稠密 BF16 权重 @ x 这条基准：先把反量化结果舍到 bf16 再算，同 §1.2 第 2 条口径 Wdense = bf16_to_f32(f32_to_bf16(Wf32)) - print(" %s：%s（起始行 %d），%d 行 x K=%d，row_bytes=%d" - % (T_NAME.get(t, t), name, r0, n, K, W.shape[1])) + print( + " %s：%s（起始行 %d），%d 行 x K=%d，row_bytes=%d" + % (T_NAME.get(t, t), name, r0, n, K, W.shape[1]) + ) for m in Ms: A = (rng.standard_normal((m, K)) * 0.5).astype(np.float32) Abits = f32_to_bf16(A) - Af = bf16_to_f32(Abits) # kernel 看到的就是这份值 + Af = bf16_to_f32(Abits) # kernel 看到的就是这份值 p, cbin = run_gemv(binary, t, Abits, W, K, workdir, "gemv") - if not check("%s M=%d：kernel 退出码 0" % (T_NAME.get(t, t), m), p.returncode == 0, - "rc=%d %s" % (p.returncode, (p.stderr or p.stdout).strip()[-600:])): + if not check( + "%s M=%d：kernel 退出码 0" % (T_NAME.get(t, t), m), + p.returncode == 0, + "rc=%d %s" % (p.returncode, (p.stderr or p.stdout).strip()[-600:]), + ): continue # 路由判据：probe 报的 path 必须等于算子在该 M 上会选的路径。数值过了但 # 路走错了同样不可接受（那意味着门测的不是发布路径）。 want_path = "gemv" if m <= MAX_M else "prefill" pm = PATH_RE.search(p.stdout or "") got_path = pm.group(1) if pm else "?" - check("%s M=%d：走 %s 路径（与算子 calculate 的谓词一致）" % (T_NAME.get(t, t), m, want_path), - got_path == want_path, "probe 报 path=%s" % got_path) + check( + "%s M=%d：走 %s 路径（与算子 calculate 的谓词一致）" + % (T_NAME.get(t, t), m, want_path), + got_path == want_path, + "probe 报 path=%s" % got_path, + ) got = bf16_to_f32(np.fromfile(cbin, np.uint16).reshape(m, n)) assert got.shape == (m, n) - ref = (Af @ Wdense.T).astype(np.float32) # §1.2 第 2 条口径的基准 - ref_exact = (Af @ Wf32.T).astype(np.float32) # 不先把权重舍到 bf16 + ref = (Af @ Wdense.T).astype(np.float32) # §1.2 第 2 条口径的基准 + ref_exact = (Af @ Wf32.T).astype(np.float32) # 不先把权重舍到 bf16 c = cos_sim(got, ref) - check("%s M=%d：cos_sim(kernel, 稠密 BF16 权重 @ x) > 0.999" % (T_NAME.get(t, t), m), - c > 0.999, "cos_sim=%.8f" % c) + check( + "%s M=%d：cos_sim(kernel, 稠密 BF16 权重 @ x) > 0.999" + % (T_NAME.get(t, t), m), + c > 0.999, + "cos_sim=%.8f" % c, + ) # 观测量（不作判据）：bf16 位相同率、最大绝对/相对偏差、vs 未舍入基准的 cos_sim same = float(np.mean(f32_to_bf16(got) == f32_to_bf16(ref))) dg = got.astype(np.float64) - ref.astype(np.float64) @@ -190,12 +214,24 @@ def check_type_one(binary, art, t, name, W, K, r0, Ms, rng, workdir): # 相对偏差只在“有意义的元素”上算（|ref| >= 最大幅值的 1%），否则会被近零 # 元素除出几十倍的假大数，那种数字没有判读价值。 sig = np.abs(ref.astype(np.float64)) >= 0.01 * float(np.max(np.abs(ref))) - rel = float(np.max(np.abs(dg[sig]) / np.abs(ref.astype(np.float64)[sig]))) if sig.any() else 0.0 - print(" 观测：cos_sim(kernel, 稠密 BF16 权重)=%.10f" - " cos_sim(kernel, 未舍入基准)=%.10f bf16 逐位相同率=%.4f" - " max|Δ|=%.3e max 相对偏差(|ref|≥最大幅值1%% 的子集)=%.3e %s" - % (c, cos_sim(got, ref_exact), same, absd, rel, - p.stdout.strip().split("ok")[-1].strip())) + rel = ( + float(np.max(np.abs(dg[sig]) / np.abs(ref.astype(np.float64)[sig]))) + if sig.any() + else 0.0 + ) + print( + " 观测：cos_sim(kernel, 稠密 BF16 权重)=%.10f" + " cos_sim(kernel, 未舍入基准)=%.10f bf16 逐位相同率=%.4f" + " max|Δ|=%.3e max 相对偏差(|ref|≥最大幅值1%% 的子集)=%.3e %s" + % ( + c, + cos_sim(got, ref_exact), + same, + absd, + rel, + p.stdout.strip().split("ok")[-1].strip(), + ) + ) def check_rejections(binary, art, workdir): @@ -214,28 +250,47 @@ def check_rejections(binary, art, workdir): Abits = f32_to_bf16(A) p, _ = run_gemv(binary, 10, Abits, W, K, workdir, "rej") - check("未知 ggml type 10 被拒（rc=3，不启动 kernel）", p.returncode == 3, - "rc=%d %s" % (p.returncode, p.stderr.strip()[-300:])) + check( + "未知 ggml type 10 被拒（rc=3，不启动 kernel）", + p.returncode == 3, + "rc=%d %s" % (p.returncode, p.stderr.strip()[-300:]), + ) - bad_k = K + (BLOCK_SIZE[name_ok] - 1) # 不再是整数个 block + bad_k = K + (BLOCK_SIZE[name_ok] - 1) # 不再是整数个 block A_bad = f32_to_bf16(np.zeros((1, bad_k), np.float32)) p, _ = run_gemv(binary, name_ok, A_bad, W, bad_k, workdir, "rej") - check("K 不是 block 元素数整数倍被拒（rc=3）", p.returncode == 3, - "rc=%d %s" % (p.returncode, p.stderr.strip()[-300:])) + check( + "K 不是 block 元素数整数倍被拒（rc=3）", + p.returncode == 3, + "rc=%d %s" % (p.returncode, p.stderr.strip()[-300:]), + ) # 同一条约束在 prefill 路径上也必须成立（两条路径各自有谓词，不能只查 gemv） A_bad_p = f32_to_bf16(np.zeros((MAX_M + 1, bad_k), np.float32)) p, _ = run_gemv(binary, name_ok, A_bad_p, W, bad_k, workdir, "rej") - check("prefill 路径同样拒掉不整除的 K（rc=3）", p.returncode == 3, - "rc=%d %s" % (p.returncode, p.stderr.strip()[-300:])) + check( + "prefill 路径同样拒掉不整除的 K（rc=3）", + p.returncode == 3, + "rc=%d %s" % (p.returncode, p.stderr.strip()[-300:]), + ) def main(): ap = argparse.ArgumentParser() - ap.add_argument("--model-path", default="/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8") - ap.add_argument("--rows", type=int, default=200, help="每个张量取多少行权重（不是 64 的整数倍才能盖住 tile 余数）") - ap.add_argument("--ms", default="1,8," + PREFILL_MS, - help="逗号分隔；<=8 走 gemv，>8 走 prefill（两条路径同一份门）") + ap.add_argument( + "--model-path", default="/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8" + ) + ap.add_argument( + "--rows", + type=int, + default=200, + help="每个张量取多少行权重（不是 64 的整数倍才能盖住 tile 余数）", + ) + ap.add_argument( + "--ms", + default="1,8," + PREFILL_MS, + help="逗号分隔；<=8 走 gemv，>8 走 prefill（两条路径同一份门）", + ) ap.add_argument("--workdir", default="/home/liuxd/tmp_routeb/gemv32") ap.add_argument("--nvcc", default=os.environ.get("CUDACXX", "nvcc")) ap.add_argument("--skip-build", action="store_true") @@ -247,11 +302,18 @@ def main(): rng = np.random.default_rng(args.seed) os.makedirs(args.workdir, exist_ok=True) Ms = [int(x) for x in args.ms.split(",") if x.strip()] - print("产物：%s\n被测：\n %s\n %s\n %s\n临时目录：%s\n每种类型权重行数：%d，M 取 %s" - % (args.model_path, - os.path.join(GEMV_DIR, "linear_gguf_gemv.cuh"), - os.path.join(GEMV_DIR, "linear_gguf_dequant.cuh"), - PROBE_SRC, args.workdir, args.rows, Ms)) + print( + "产物：%s\n被测：\n %s\n %s\n %s\n临时目录：%s\n每种类型权重行数：%d，M 取 %s" + % ( + args.model_path, + os.path.join(GEMV_DIR, "linear_gguf_gemv.cuh"), + os.path.join(GEMV_DIR, "linear_gguf_dequant.cuh"), + PROBE_SRC, + args.workdir, + args.rows, + Ms, + ) + ) binary = os.path.join(args.workdir, "gemv_probe") print("\n[1] 编译两条路径的驱动（prefill 需要 -lcublas）") @@ -259,18 +321,28 @@ def main(): skip("编译", "--skip-build") else: try: - bref.build_probe(PROBE_SRC, binary, args.nvcc, - extra=["-I", GEMV_DIR, "-lcublas"]) - check("nvcc 编译 %s 通过（含两个 kernel 头 + cublas）" - % os.path.basename(PROBE_SRC), True) - except Exception as exc: # noqa: BLE001 - check("nvcc 编译 %s 通过" % os.path.basename(PROBE_SRC), False, str(exc)[-2000:]) + bref.build_probe( + PROBE_SRC, binary, args.nvcc, extra=["-I", GEMV_DIR, "-lcublas"] + ) + check( + "nvcc 编译 %s 通过（含两个 kernel 头 + cublas）" + % os.path.basename(PROBE_SRC), + True, + ) + except Exception as exc: # noqa: BLE001 + check( + "nvcc 编译 %s 通过" % os.path.basename(PROBE_SRC), + False, + str(exc)[-2000:], + ) return 1 art = Artifact(args.model_path) print("\n[2] 真实权重对 numpy 稠密基准（gemv + prefill，判据：cos_sim > 0.999）") - print("产物 blob 张量 %d 个（key_prefix=%r），按类型：%s" - % (len(art.blobs), art.prefix, {t: len(art.type_names(t)) for t in TYPES})) + print( + "产物 blob 张量 %d 个（key_prefix=%r），按类型：%s" + % (len(art.blobs), art.prefix, {t: len(art.type_names(t)) for t in TYPES}) + ) for t in TYPES: n_avail = len(art.type_names(t)) # 张量本来就少（Q4_K 全模型只有 4 个，且都在 layer 1）时全取，否则首/中/尾 @@ -284,7 +356,10 @@ def main(): print("\n[3] 非法输入必须被拒（不许静默出结果）") check_rejections(binary, art, args.workdir) - print("\n== 结果：%d PASS / %d FAIL / %d SKIP ==" % (bref._PASS, bref._FAIL, bref._SKIP)) + print( + "\n== 结果：%d PASS / %d FAIL / %d SKIP ==" + % (bref._PASS, bref._FAIL, bref._SKIP) + ) print("临时目录：%s" % args.workdir) return 0 if bref._FAIL == 0 else 1 diff --git a/scripts/gguf_routeb_gemv_probe.cu b/scripts/gguf_routeb_gemv_probe.cu index 9ed747829..f5e9e7248 100644 --- a/scripts/gguf_routeb_gemv_probe.cu +++ b/scripts/gguf_routeb_gemv_probe.cu @@ -20,19 +20,19 @@ #include #include -#include #include +#include #include "linear_gguf_dequant.cuh" -#define CUDA_CHECK(call) \ - do { \ - cudaError_t err__ = (call); \ - if (err__ != cudaSuccess) { \ - std::fprintf(stderr, "gemv probe: %s failed: %s\n", #call, \ - cudaGetErrorString(err__)); \ - return 5; \ - } \ +#define CUDA_CHECK(call) \ + do { \ + cudaError_t err__ = (call); \ + if (err__ != cudaSuccess) { \ + std::fprintf(stderr, "gemv probe: %s failed: %s\n", #call, \ + cudaGetErrorString(err__)); \ + return 5; \ + } \ } while (0) static std::vector read_all(const char *path, size_t want) { diff --git a/scripts/gguf_routeb_head_precision.py b/scripts/gguf_routeb_head_precision.py index 6c36d67f5..941ad0fc4 100644 --- a/scripts/gguf_routeb_head_precision.py +++ b/scripts/gguf_routeb_head_precision.py @@ -6,8 +6,9 @@ import os import sys -sys.path.insert(0, os.path.join( - os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py")) +sys.path.insert( + 0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py") +) def main(): @@ -28,13 +29,17 @@ def main(): compared = {x["id"]: x for x in json.load(f)["cases"]} with open(args.infinilm_trace, encoding="utf-8") as f: traced = json.load(f)["cases"] - with open(os.path.join(args.model_path, "model.safetensors.index.json"), - encoding="utf-8") as f: + with open( + os.path.join(args.model_path, "model.safetensors.index.json"), encoding="utf-8" + ) as f: weight_map = json.load(f)["weight_map"] from safetensors import safe_open + native_shard = safe_open( os.path.join(args.model_path, weight_map["lm_head.weight"]), - framework="pt", device="cpu") + framework="pt", + device="cpu", + ) native_head = native_shard.get_slice("lm_head.weight") reader = GGUFReader(args.gguf, "r") output = next(t for t in reader.tensors if t.name == "output.weight") @@ -50,45 +55,61 @@ def main(): hidden = (bits.astype(np.uint32) << 16).view(np.float32) rows = [] for token_id in (llama_token, infini_token): - raw_row = output.data[token_id:token_id + 1] + raw_row = output.data[token_id : token_id + 1] row = np.asarray( dequantize(raw_row, GGMLQuantizationType(int(output.tensor_type))), - dtype=np.float32).reshape(-1) + dtype=np.float32, + ).reshape(-1) rows.append(row) logits = [float(np.dot(hidden, row)) for row in rows] native_rows = [ - native_head[token_id:token_id + 1].float().numpy().reshape(-1) + native_head[token_id : token_id + 1].float().numpy().reshape(-1) for token_id in (llama_token, infini_token) ] native_logits = [float(np.dot(hidden, row)) for row in native_rows] result = { - "case_id": case["case_id"], "first_difference": diff, - "llama_token": llama_token, "infinilm_token": infini_token, + "case_id": case["case_id"], + "first_difference": diff, + "llama_token": llama_token, + "infinilm_token": infini_token, "llama_token_fp32_logit": logits[0], "infinilm_token_fp32_logit": logits[1], "fp32_margin_llama_minus_infinilm": logits[0] - logits[1], "fp32_winner": llama_token if logits[0] > logits[1] else infini_token, - "bf16_weight_fp32_margin_llama_minus_infinilm": - native_logits[0] - native_logits[1], - "bf16_weight_fp32_winner": - llama_token if native_logits[0] > native_logits[1] else infini_token, + "bf16_weight_fp32_margin_llama_minus_infinilm": native_logits[0] + - native_logits[1], + "bf16_weight_fp32_winner": llama_token + if native_logits[0] > native_logits[1] + else infini_token, "hidden_shape": step["hidden_shape"], } results.append(result) - print("%-10s llama=%6d infini=%6d gguf_f32=%+.8f bf16w_f32=%+.8f winner=%d" % ( - result["case_id"], llama_token, infini_token, - result["fp32_margin_llama_minus_infinilm"], - result["bf16_weight_fp32_margin_llama_minus_infinilm"], - result["bf16_weight_fp32_winner"]), - flush=True) + print( + "%-10s llama=%6d infini=%6d gguf_f32=%+.8f bf16w_f32=%+.8f winner=%d" + % ( + result["case_id"], + llama_token, + infini_token, + result["fp32_margin_llama_minus_infinilm"], + result["bf16_weight_fp32_margin_llama_minus_infinilm"], + result["bf16_weight_fp32_winner"], + ), + flush=True, + ) report = {"gguf_lm_head_type": type_name, "cases": results} os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) with open(args.out, "w", encoding="utf-8") as f: json.dump(report, f, ensure_ascii=False, indent=2) - print("RESULT gguf_f32_llama_wins=%d/%d bf16_weight_f32_llama_wins=%d/%d" % ( - sum(x["fp32_winner"] == x["llama_token"] for x in results), len(results), - sum(x["bf16_weight_fp32_winner"] == x["llama_token"] for x in results), len(results)), - flush=True) + print( + "RESULT gguf_f32_llama_wins=%d/%d bf16_weight_f32_llama_wins=%d/%d" + % ( + sum(x["fp32_winner"] == x["llama_token"] for x in results), + len(results), + sum(x["bf16_weight_fp32_winner"] == x["llama_token"] for x in results), + len(results), + ), + flush=True, + ) if __name__ == "__main__": diff --git a/scripts/gguf_routeb_infinilm_ref.py b/scripts/gguf_routeb_infinilm_ref.py index 1b25baa98..522a080dc 100755 --- a/scripts/gguf_routeb_infinilm_ref.py +++ b/scripts/gguf_routeb_infinilm_ref.py @@ -25,8 +25,8 @@ def main() -> int: ap.add_argument("--out", required=True) args = ap.parse_args() - import numpy as np import infinicore + import numpy as np from infinilm.cache import PagedKVCacheConfig from infinilm.distributed import DistConfig from infinilm.infer_engine import GenerationConfig, InferEngine @@ -48,7 +48,8 @@ def main() -> int: device=infinicore.device("cuda:0"), distributed_config=DistConfig(1), cache_config=PagedKVCacheConfig( - args.num_blocks, args.block_size, max_batch_size=1), + args.num_blocks, args.block_size, max_batch_size=1 + ), attention_backend="paged-attn", ) load_model_state_dict_by_file(engine, args.model_path, dtype=engine.dtype) @@ -61,7 +62,8 @@ def main() -> int: runs = [] for repeat in range(args.repeats): prompt = infinicore.from_list( - [[int(x) for x in case["input_ids"]]], dtype=infinicore.int64) + [[int(x) for x in case["input_ids"]]], dtype=infinicore.int64 + ) config = GenerationConfig( max_new_tokens=args.new_tokens, temperature=0.0, @@ -74,25 +76,32 @@ def main() -> int: run_started = time.time() generated = engine.generate(prompt, config) tokens = [int(np.asarray(x.to_numpy()).reshape(-1)[0]) for x in generated] - runs.append({ - "repeat": repeat, - "tokens": tokens, - "elapsed_s": round(time.time() - run_started, 4), - }) + runs.append( + { + "repeat": repeat, + "tokens": tokens, + "elapsed_s": round(time.time() - run_started, 4), + } + ) deterministic = all(x["tokens"] == runs[0]["tokens"] for x in runs[1:]) exact_length = all(len(x["tokens"]) == args.new_tokens for x in runs) ok = deterministic and exact_length all_ok &= ok - outputs.append({ - "id": case["id"], - "prompt": case["prompt"], - "input_ids": case["input_ids"], - "deterministic": deterministic, - "exact_length": exact_length, - "runs": runs, - }) - print("%-10s deterministic=%s length=%s tokens=%s" % ( - case["id"], deterministic, exact_length, runs[0]["tokens"]), flush=True) + outputs.append( + { + "id": case["id"], + "prompt": case["prompt"], + "input_ids": case["input_ids"], + "deterministic": deterministic, + "exact_length": exact_length, + "runs": runs, + } + ) + print( + "%-10s deterministic=%s length=%s tokens=%s" + % (case["id"], deterministic, exact_length, runs[0]["tokens"]), + flush=True, + ) result = { "engine": "InfiniLM", diff --git a/scripts/gguf_routeb_infinilm_trace.py b/scripts/gguf_routeb_infinilm_trace.py index aab053a7c..e71665ba1 100644 --- a/scripts/gguf_routeb_infinilm_trace.py +++ b/scripts/gguf_routeb_infinilm_trace.py @@ -22,28 +22,45 @@ def main(): ap.add_argument("--num-blocks", type=int, default=64) ap.add_argument("--block-size", type=int, default=256) ap.add_argument( - "--stop-at-first-diff", action="store_true", - help="Stop each case immediately after its known first-difference step.") + "--stop-at-first-diff", + action="store_true", + help="Stop each case immediately after its known first-difference step.", + ) ap.add_argument( "--prenorm-dump-root", - help="Optional root for per-case pre-final-RMSNorm binary dumps.") + help="Optional root for per-case pre-final-RMSNorm binary dumps.", + ) + ap.add_argument( + "--case-id", + action="append", + help="Optionally trace only the named case; repeat for multiple cases.", + ) ap.add_argument( - "--case-id", action="append", - help="Optionally trace only the named case; repeat for multiple cases.") + "--operator-dump-layer", + type=int, + help="Override the per-case layer selected for generic operator dumps.", + ) ap.add_argument( - "--operator-dump-layer", type=int, - help="Override the per-case layer selected for generic operator dumps.") - ap.add_argument("--gdn-dump-layer", type=int, - help="Enable GDN intermediate dumps for this layer.") - ap.add_argument("--gdn-dump-seq-len", type=int, default=1, - help="Sequence length for GDN intermediate dumps (default: 1).") + "--gdn-dump-layer", + type=int, + help="Enable GDN intermediate dumps for this layer.", + ) + ap.add_argument( + "--gdn-dump-seq-len", + type=int, + default=1, + help="Sequence length for GDN intermediate dumps (default: 1).", + ) ap.add_argument("--out", required=True) - ap.add_argument("--allow-token-mismatch", action="store_true", - help="Diagnostic only: keep output even if selected tokens differ from expected.") + ap.add_argument( + "--allow-token-mismatch", + action="store_true", + help="Diagnostic only: keep output even if selected tokens differ from expected.", + ) args = ap.parse_args() - import numpy as np import infinicore + import numpy as np from infinilm.cache import PagedKVCacheConfig from infinilm.distributed import DistConfig from infinilm.infer_engine import InferEngine @@ -53,8 +70,9 @@ def main(): with open(args.inputs, encoding="utf-8") as f: inputs = {x["id"]: x for x in json.load(f)["cases"]} with open(args.compare, encoding="utf-8") as f: - divergent = [x for x in json.load(f)["cases"] - if x["first_difference"] is not None] + divergent = [ + x for x in json.load(f)["cases"] if x["first_difference"] is not None + ] if args.case_id: selected = set(args.case_id) divergent = [x for x in divergent if x["id"] in selected] @@ -66,19 +84,22 @@ def main(): with open(args.expected_results, encoding="utf-8") as f: current = json.load(f) expected_by_id = { - x["id"]: [int(t) for t in x["runs"][0]["tokens"]] - for x in current["cases"] + x["id"]: [int(t) for t in x["runs"][0]["tokens"]] for x in current["cases"] } if len(divergent) >= max(2, args.num_blocks // 4): raise ValueError("not enough independent Mamba cache rows") started = time.time() cache_config = PagedKVCacheConfig( - args.num_blocks, args.block_size, max_batch_size=1) + args.num_blocks, args.block_size, max_batch_size=1 + ) engine = InferEngine( - model_path=args.model_path, device=infinicore.device("cuda:0"), - distributed_config=DistConfig(1), cache_config=cache_config, - attention_backend="paged-attn") + model_path=args.model_path, + device=infinicore.device("cuda:0"), + distributed_config=DistConfig(1), + cache_config=cache_config, + attention_backend="paged-attn", + ) load_model_state_dict_by_file(engine, args.model_path, dtype=engine.dtype) load_s = time.time() - started results = [] @@ -141,51 +162,90 @@ def main(): slot_base = kv_block * args.block_size slot_mapping = [slot_base + i for i in range(past, total)] tensors = { - "input_ids": infinicore.from_list([current], dtype=infinicore.int64).view([1, seq_len]), + "input_ids": infinicore.from_list( + [current], dtype=infinicore.int64 + ).view([1, seq_len]), "position_ids": infinicore.from_list(positions, dtype=infinicore.int64), "past_kv_lengths": infinicore.from_list([past], dtype=infinicore.int32), - "total_kv_lengths": infinicore.from_list([total], dtype=infinicore.int32), - "input_offsets": infinicore.from_list([0, seq_len], dtype=infinicore.int32), + "total_kv_lengths": infinicore.from_list( + [total], dtype=infinicore.int32 + ), + "input_offsets": infinicore.from_list( + [0, seq_len], dtype=infinicore.int32 + ), "cu_seqlens": infinicore.from_list([0, total], dtype=infinicore.int32), - "block_tables": infinicore.from_list([[kv_block]], dtype=infinicore.int32), - "slot_mapping": infinicore.from_list(slot_mapping, dtype=infinicore.int64), + "block_tables": infinicore.from_list( + [[kv_block]], dtype=infinicore.int32 + ), + "slot_mapping": infinicore.from_list( + slot_mapping, dtype=infinicore.int64 + ), "mamba_init_state_indices": infinicore.from_list( - [0 if step == 0 else mamba_row], dtype=infinicore.int32), + [0 if step == 0 else mamba_row], dtype=infinicore.int32 + ), "mamba_final_state_indices": infinicore.from_list( - [mamba_row], dtype=infinicore.int32), + [mamba_row], dtype=infinicore.int32 + ), } cpp_input = engine._build_input( - tensors["input_ids"], position_ids=tensors["position_ids"], + tensors["input_ids"], + position_ids=tensors["position_ids"], past_kv_lengths=tensors["past_kv_lengths"], total_kv_lengths=tensors["total_kv_lengths"], - input_offsets=tensors["input_offsets"], cu_seqlens=tensors["cu_seqlens"], - block_tables=tensors["block_tables"], slot_mapping=tensors["slot_mapping"], + input_offsets=tensors["input_offsets"], + cu_seqlens=tensors["cu_seqlens"], + block_tables=tensors["block_tables"], + slot_mapping=tensors["slot_mapping"], mamba_init_state_indices=tensors["mamba_init_state_indices"], mamba_final_state_indices=tensors["mamba_final_state_indices"], - sample_all_positions=False, temperature=0.0, top_k=1, top_p=1.0) + sample_all_positions=False, + temperature=0.0, + top_k=1, + top_p=1.0, + ) output = _infinilm.InferEngine.forward(engine, cpp_input) - token = int(np.asarray(infinicore.Tensor(output.output_ids).to_numpy()).reshape(-1)[0]) + token = int( + np.asarray(infinicore.Tensor(output.output_ids).to_numpy()).reshape(-1)[ + 0 + ] + ) raw = infinicore.Tensor(output.logits) shape = list(raw.shape) cpu = raw.to(infinicore.device("cpu", 0)) if cpu.dtype == infinicore.bfloat16: bits_type = ctypes.c_uint16 * cpu.numel() - bits = np.ctypeslib.as_array(bits_type.from_address(cpu.data_ptr())).copy() + bits = np.ctypeslib.as_array( + bits_type.from_address(cpu.data_ptr()) + ).copy() logits = (bits.astype(np.uint32) << 16).view(np.float32).reshape(shape) elif cpu.dtype == infinicore.float32: - logits = np.ctypeslib.as_array( - (ctypes.c_float * cpu.numel()).from_address(cpu.data_ptr()) - ).copy().reshape(shape) + logits = ( + np.ctypeslib.as_array( + (ctypes.c_float * cpu.numel()).from_address(cpu.data_ptr()) + ) + .copy() + .reshape(shape) + ) else: raise TypeError("expected BF16 or F32 logits, got %s" % cpu.dtype) logits = logits.reshape(-1, shape[-1])[-1] - order = np.argpartition(logits, -args.top_k)[-args.top_k:] + order = np.argpartition(logits, -args.top_k)[-args.top_k :] order = order[np.argsort(logits[order], kind="stable")[::-1]] top_logit = float(logits[order[0]]) - candidates = [{"id": int(i), "logit": float(logits[i]), - "delta_from_top": float(logits[i] - top_logit)} for i in order] - step_result = {"step": step, "selected": token, - "logits_shape": shape, "top_logits": candidates} + candidates = [ + { + "id": int(i), + "logit": float(logits[i]), + "delta_from_top": float(logits[i] - top_logit), + } + for i in order + ] + step_result = { + "step": step, + "selected": token, + "logits_shape": shape, + "top_logits": candidates, + } if step == first_diff: hidden = infinicore.Tensor(output.hidden_states) hidden_shape = list(hidden.shape) @@ -194,17 +254,22 @@ def main(): if hidden_cpu.dtype == infinicore.bfloat16: hidden_bits_type = ctypes.c_uint16 * hidden_cpu.numel() hidden_bits = np.ctypeslib.as_array( - hidden_bits_type.from_address(hidden_cpu.data_ptr())).copy() + hidden_bits_type.from_address(hidden_cpu.data_ptr()) + ).copy() step_result["hidden_dtype"] = "bfloat16" step_result["hidden_bf16_bits"] = [int(x) for x in hidden_bits] elif hidden_cpu.dtype == infinicore.float32: hidden_values = np.ctypeslib.as_array( (ctypes.c_float * hidden_cpu.numel()).from_address( - hidden_cpu.data_ptr())).copy() + hidden_cpu.data_ptr() + ) + ).copy() step_result["hidden_dtype"] = "float32" step_result["hidden_f32"] = [float(x) for x in hidden_values] else: - raise TypeError("expected BF16 or F32 hidden state, got %s" % hidden_cpu.dtype) + raise TypeError( + "expected BF16 or F32 hidden state, got %s" % hidden_cpu.dtype + ) steps.append(step_result) generated.append(token) current = [token] @@ -212,14 +277,22 @@ def main(): expected = expected[:case_new_tokens] stable = generated == expected if not stable and not args.allow_token_mismatch: - raise RuntimeError("%s trace changed: %s != %s" % (case_id, generated, expected)) + raise RuntimeError( + "%s trace changed: %s != %s" % (case_id, generated, expected) + ) results.append({"case_id": case_id, "tokens": generated, "steps": steps}) - print("%-10s tokens=%d stable=%s" % (case_id, len(generated), stable), flush=True) + print( + "%-10s tokens=%d stable=%s" % (case_id, len(generated), stable), flush=True + ) os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) with open(args.out, "w", encoding="utf-8") as f: - json.dump({"load_s": round(load_s, 4), "cases": results}, - f, ensure_ascii=False, indent=2) + json.dump( + {"load_s": round(load_s, 4), "cases": results}, + f, + ensure_ascii=False, + indent=2, + ) print("RESULT cases=%d load=%.3fs" % (len(results), load_s), flush=True) diff --git a/scripts/gguf_routeb_llama_probe.py b/scripts/gguf_routeb_llama_probe.py index 1407abe08..f467c52bb 100644 --- a/scripts/gguf_routeb_llama_probe.py +++ b/scripts/gguf_routeb_llama_probe.py @@ -24,22 +24,36 @@ def main(): diff = item["first_difference"] prefix = inputs[case_id]["input_ids"] + item["llama_tokens"][:diff] body = { - "prompt": prefix, "n_predict": 1, "temperature": 0.0, - "top_k": 1, "top_p": 1.0, "min_p": 0.0, "typical_p": 1.0, - "repeat_penalty": 1.0, "repeat_last_n": 0, - "presence_penalty": 0.0, "frequency_penalty": 0.0, - "seed": 1, "ignore_eos": True, "cache_prompt": False, - "return_tokens": True, "n_probs": 100, "stream": False, + "prompt": prefix, + "n_predict": 1, + "temperature": 0.0, + "top_k": 1, + "top_p": 1.0, + "min_p": 0.0, + "typical_p": 1.0, + "repeat_penalty": 1.0, + "repeat_last_n": 0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "seed": 1, + "ignore_eos": True, + "cache_prompt": False, + "return_tokens": True, + "n_probs": 100, + "stream": False, "samplers": ["top_k", "temperature"], } req = urllib.request.Request( args.server.rstrip("/") + "/completion", data=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}, method="POST") + headers={"Content-Type": "application/json"}, + method="POST", + ) with urllib.request.urlopen(req, timeout=180) as response: raw = json.load(response) - results.append({"case_id": case_id, "diff": diff, "prefix": prefix, - "response": raw}) + results.append( + {"case_id": case_id, "diff": diff, "prefix": prefix, "response": raw} + ) probs = raw.get("completion_probabilities", []) print(case_id, "tokens=", raw.get("tokens"), "prob_entry=", probs[:1]) with open(args.out, "w", encoding="utf-8") as f: diff --git a/scripts/gguf_routeb_llama_ref.py b/scripts/gguf_routeb_llama_ref.py index 5df2bf84b..e5df59714 100755 --- a/scripts/gguf_routeb_llama_ref.py +++ b/scripts/gguf_routeb_llama_ref.py @@ -67,33 +67,43 @@ def main() -> int: } started = time.time() response = post_json( - args.server.rstrip("/") + "/completion", body, args.timeout) + args.server.rstrip("/") + "/completion", body, args.timeout + ) tokens = [int(x) for x in response.get("tokens", [])] probabilities = response.get("completion_probabilities", []) - runs.append({ - "repeat": repeat, - "tokens": tokens, - "content": response.get("content", ""), - "first_token_top_logprobs": ( - probabilities[0].get("top_logprobs", []) if probabilities else []), - "elapsed_s": round(time.time() - started, 4), - "tokens_evaluated": response.get("tokens_evaluated"), - "tokens_predicted": response.get("tokens_predicted"), - }) + runs.append( + { + "repeat": repeat, + "tokens": tokens, + "content": response.get("content", ""), + "first_token_top_logprobs": ( + probabilities[0].get("top_logprobs", []) + if probabilities + else [] + ), + "elapsed_s": round(time.time() - started, 4), + "tokens_evaluated": response.get("tokens_evaluated"), + "tokens_predicted": response.get("tokens_predicted"), + } + ) deterministic = all(x["tokens"] == runs[0]["tokens"] for x in runs[1:]) exact_length = all(len(x["tokens"]) == args.new_tokens for x in runs) ok = deterministic and exact_length all_ok &= ok - outputs.append({ - "id": case["id"], - "prompt": case["prompt"], - "input_ids": case["input_ids"], - "deterministic": deterministic, - "exact_length": exact_length, - "runs": runs, - }) - print("%-10s deterministic=%s length=%s tokens=%s" % ( - case["id"], deterministic, exact_length, runs[0]["tokens"])) + outputs.append( + { + "id": case["id"], + "prompt": case["prompt"], + "input_ids": case["input_ids"], + "deterministic": deterministic, + "exact_length": exact_length, + "runs": runs, + } + ) + print( + "%-10s deterministic=%s length=%s tokens=%s" + % (case["id"], deterministic, exact_length, runs[0]["tokens"]) + ) result = { "engine": "llama.cpp", diff --git a/scripts/gguf_routeb_llama_trace.py b/scripts/gguf_routeb_llama_trace.py index 778b1b347..15cca09af 100644 --- a/scripts/gguf_routeb_llama_trace.py +++ b/scripts/gguf_routeb_llama_trace.py @@ -19,38 +19,63 @@ def main(): with open(args.inputs, encoding="utf-8") as f: inputs = {x["id"]: x for x in json.load(f)["cases"]} with open(args.compare, encoding="utf-8") as f: - divergent = [x for x in json.load(f)["cases"] - if x["first_difference"] is not None] + divergent = [ + x for x in json.load(f)["cases"] if x["first_difference"] is not None + ] results = [] for item in divergent: case_id = item["id"] body = { "prompt": inputs[case_id]["input_ids"], - "n_predict": args.new_tokens, "temperature": 0.0, - "top_k": 1, "top_p": 1.0, "min_p": 0.0, "typical_p": 1.0, - "repeat_penalty": 1.0, "repeat_last_n": 0, - "presence_penalty": 0.0, "frequency_penalty": 0.0, - "seed": 1, "ignore_eos": True, "cache_prompt": False, - "return_tokens": True, "n_probs": args.n_probs, - "stream": False, "samplers": ["top_k", "temperature"], + "n_predict": args.new_tokens, + "temperature": 0.0, + "top_k": 1, + "top_p": 1.0, + "min_p": 0.0, + "typical_p": 1.0, + "repeat_penalty": 1.0, + "repeat_last_n": 0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "seed": 1, + "ignore_eos": True, + "cache_prompt": False, + "return_tokens": True, + "n_probs": args.n_probs, + "stream": False, + "samplers": ["top_k", "temperature"], } req = urllib.request.Request( args.server.rstrip("/") + "/completion", data=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}, method="POST") + headers={"Content-Type": "application/json"}, + method="POST", + ) with urllib.request.urlopen(req, timeout=300) as response: raw = json.load(response) tokens = [int(x) for x in raw.get("tokens", [])] expected = [int(x) for x in item["llama_tokens"]] if tokens != expected: - raise RuntimeError("%s rerun changed: %s != %s" % (case_id, tokens, expected)) - results.append({ - "case_id": case_id, "tokens": tokens, - "completion_probabilities": raw.get("completion_probabilities", []), - }) - print("%-10s tokens=%d probabilities=%d stable=%s" % ( - case_id, len(tokens), len(results[-1]["completion_probabilities"]), tokens == expected), - flush=True) + raise RuntimeError( + "%s rerun changed: %s != %s" % (case_id, tokens, expected) + ) + results.append( + { + "case_id": case_id, + "tokens": tokens, + "completion_probabilities": raw.get("completion_probabilities", []), + } + ) + print( + "%-10s tokens=%d probabilities=%d stable=%s" + % ( + case_id, + len(tokens), + len(results[-1]["completion_probabilities"]), + tokens == expected, + ), + flush=True, + ) os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) with open(args.out, "w", encoding="utf-8") as f: json.dump({"cases": results}, f, ensure_ascii=False, indent=2) diff --git a/scripts/gguf_routeb_probe_params.py b/scripts/gguf_routeb_probe_params.py index 6b32994d2..9a7890f72 100644 --- a/scripts/gguf_routeb_probe_params.py +++ b/scripts/gguf_routeb_probe_params.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """探针：用 mini qwen3_5 config 构造 InferEngine，导出 C++ 侧权威参数键与 shape。""" + import json import os import sys diff --git a/scripts/gguf_routeb_shape_contract.py b/scripts/gguf_routeb_shape_contract.py index 7140d60d0..84afefa73 100644 --- a/scripts/gguf_routeb_shape_contract.py +++ b/scripts/gguf_routeb_shape_contract.py @@ -22,17 +22,20 @@ import argparse import collections -from math import prod import json import os import sys +from math import prod sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py")) +sys.path.insert( + 0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py") +) -import gguf_mapping as M # noqa: E402 -from gguf import GGUFReader # noqa: E402 -from gguf.constants import GGML_QUANT_SIZES, GGMLQuantizationType as Q # noqa: E402 +import gguf_mapping as M # noqa: E402 +from gguf import GGUFReader # noqa: E402 +from gguf.constants import GGML_QUANT_SIZES # noqa: E402 +from gguf.constants import GGMLQuantizationType as Q DEFAULT_GGUF = "/home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf" TYPE_NAME = {int(v.value): str(v.name) for v in Q} @@ -55,12 +58,18 @@ def check(name, ok, detail=""): def dims_from_text_config(tc): """config.json 的 text_config 段 -> Dims。打包器写出 config.json 后也用它自检。""" return M.Dims( - hidden=tc["hidden_size"], n_q_heads=tc["num_attention_heads"], - n_kv_heads=tc["num_key_value_heads"], head_dim=tc["head_dim"], - ffn=tc["intermediate_size"], lin_k_heads=tc["linear_num_key_heads"], - lin_v_heads=tc["linear_num_value_heads"], lin_k_dim=tc["linear_key_head_dim"], - lin_v_dim=tc["linear_value_head_dim"], conv_kernel=tc["linear_conv_kernel_dim"], - vocab=tc["vocab_size"], n_layers=tc["num_hidden_layers"], + hidden=tc["hidden_size"], + n_q_heads=tc["num_attention_heads"], + n_kv_heads=tc["num_key_value_heads"], + head_dim=tc["head_dim"], + ffn=tc["intermediate_size"], + lin_k_heads=tc["linear_num_key_heads"], + lin_v_heads=tc["linear_num_value_heads"], + lin_k_dim=tc["linear_key_head_dim"], + lin_v_dim=tc["linear_value_head_dim"], + conv_kernel=tc["linear_conv_kernel_dim"], + vocab=tc["vocab_size"], + n_layers=tc["num_hidden_layers"], interval=tc["full_attention_interval"], ) @@ -68,23 +77,31 @@ def dims_from_text_config(tc): def framework_side(engine_device): print("\n== 1. 框架侧：mini InferEngine vs build_plan(MINI) ==") import infinicore + from gguf_routeb_probe_params import CFG from infinilm.cache import StaticKVCacheConfig from infinilm.distributed import DistConfig from infinilm.infer_engine import InferEngine - from gguf_routeb_probe_params import CFG - check("探针 CFG 与 MINI 维度一致", dims_from_text_config(CFG["text_config"]) == M.MINI, - "cfg=%s\n MINI=%s" % (dims_from_text_config(CFG["text_config"]), M.MINI)) + check( + "探针 CFG 与 MINI 维度一致", + dims_from_text_config(CFG["text_config"]) == M.MINI, + "cfg=%s\n MINI=%s" % (dims_from_text_config(CFG["text_config"]), M.MINI), + ) # 不能用 /tmp：开发机上只读，写不进去。缓存在 HOME 下，无需清理权限。 - tmp = os.path.join(os.environ.get("XDG_CACHE_HOME") - or os.path.expanduser("~/.cache"), "gguf_routeb_mini_cfg") + tmp = os.path.join( + os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache"), + "gguf_routeb_mini_cfg", + ) os.makedirs(tmp, exist_ok=True) with open(os.path.join(tmp, "config.json"), "w") as f: json.dump(CFG, f) - eng = InferEngine(model_path=tmp, device=infinicore.device(engine_device, 0), - distributed_config=DistConfig(1), - cache_config=StaticKVCacheConfig(max_batch_size=1, max_cache_len=16)) + eng = InferEngine( + model_path=tmp, + device=infinicore.device(engine_device, 0), + distributed_config=DistConfig(1), + cache_config=StaticKVCacheConfig(max_batch_size=1, max_cache_len=16), + ) sd = eng.state_dict()[0] actual = {k: tuple(int(x) for x in sd[k].shape) for k in eng.state_dict_keyname()} print(" -> 引擎导出 %d 个参数（device=%s）" % (len(actual), engine_device)) @@ -94,33 +111,54 @@ def framework_side(engine_device): for e in plan: assert e.infinilm not in want, "映射表内重复键：%s" % e.infinilm want[e.infinilm] = M.compress(e.shape) - check("映射表无重复键（%d 条）" % len(plan), len(want) == len(plan), "%d vs %d" % (len(want), len(plan))) + check( + "映射表无重复键（%d 条）" % len(plan), + len(want) == len(plan), + "%d vs %d" % (len(want), len(plan)), + ) missing = sorted(set(actual) - set(want)) extra = sorted(set(want) - set(actual)) - check("无缺键（框架要但映射表未提供 -> 会保持随机初始化）", not missing, str(missing[:12])) - check("无多键（映射表提供但框架无此参数 -> strict=False 下静默丢）", not extra, str(extra[:12])) + check( + "无缺键（框架要但映射表未提供 -> 会保持随机初始化）", + not missing, + str(missing[:12]), + ) + check( + "无多键（映射表提供但框架无此参数 -> strict=False 下静默丢）", + not extra, + str(extra[:12]), + ) - bad = [(k, want[k], M.compress(actual[k])) for k in sorted(set(actual) & set(want)) - if M.compress(actual[k]) != want[k]] + bad = [ + (k, want[k], M.compress(actual[k])) + for k in sorted(set(actual) & set(want)) + if M.compress(actual[k]) != want[k] + ] check("逐键 shape 全等（压缩长度为 1 的维后）", not bad, str(bad[:8])) def dense_iq_bf16(plan, tensors, gguf_types, prod): """v1 被稠密化的那 5 个 IQ4 张量若改回 blob，可省下的显存字节数。""" - return sum(prod(e.shape) * 2 - int(tensors[e.gguf].n_bytes) for e in plan - if not e.blob and gguf_types.get(e.gguf) in M.V1_IQUANT_DENSE - and e.gguf in tensors) + return sum( + prod(e.shape) * 2 - int(tensors[e.gguf].n_bytes) + for e in plan + if not e.blob + and gguf_types.get(e.gguf) in M.V1_IQUANT_DENSE + and e.gguf in tensors + ) def gguf_side(path): print("\n== 2. GGUF 侧：build_plan(REAL) vs 真文件 ==") reader = GGUFReader(path) tensors = {t.name: t for t in reader.tensors} - gguf_types = {n: TYPE_NAME.get(int(t.tensor_type), str(t.tensor_type)) - for n, t in tensors.items()} + gguf_types = { + n: TYPE_NAME.get(int(t.tensor_type), str(t.tensor_type)) + for n, t in tensors.items() + } plan = M.build_plan(M.REAL) - n_exc = M.apply_v1_exceptions(plan, gguf_types) # v1 稠密化 IQ4（阶段 6 取消） + n_exc = M.apply_v1_exceptions(plan, gguf_types) # v1 稠密化 IQ4（阶段 6 取消） check("映射条目数 = %d" % len(plan), len(plan) == 947, str(len(plan))) check("v1 稠密化例外命中 5 个 IQ4 张量", n_exc == 5, str(n_exc)) @@ -132,8 +170,8 @@ def gguf_side(path): if t is None: bad_name.append(e.gguf) continue - ne = tuple(int(x) for x in t.shape) # GGML ne 序 = [in, out] - hf = tuple(reversed(ne)) # HF/InfiniLM 序 = [out, in] + ne = tuple(int(x) for x in t.shape) # GGML ne 序 = [in, out] + hf = tuple(reversed(ne)) # HF/InfiniLM 序 = [out, in] tn = TYPE_NAME.get(int(t.tensor_type), str(t.tensor_type)) suffix = e.gguf.split(".", 2)[2] if e.gguf.startswith("blk.") else e.gguf type_hist[suffix][tn] += 1 @@ -148,8 +186,15 @@ def gguf_side(path): s, ep = e.slices[0] got = (ep - s,) + got[1:] if exp != got: - bad_shape.append("%s: 表 %s vs GGUF %s%s" % (e.gguf, exp, got, - "" if not e.slices else "（按行段 %s）" % (e.slices[0],))) + bad_shape.append( + "%s: 表 %s vs GGUF %s%s" + % ( + e.gguf, + exp, + got, + "" if not e.slices else "（按行段 %s）" % (e.slices[0],), + ) + ) continue if e.blob: blk, ts = (int(x) for x in GGML_QUANT_SIZES[Q[tn]]) @@ -159,15 +204,23 @@ def gguf_side(path): else: row_bytes = n_in // blk * ts if row_bytes * hf[0] != int(t.n_bytes): - bad_rows.append("%s: %d 行 x %d B != n_bytes %d" - % (e.gguf, hf[0], row_bytes, t.n_bytes)) + bad_rows.append( + "%s: %d 行 x %d B != n_bytes %d" + % (e.gguf, hf[0], row_bytes, t.n_bytes) + ) else: ok_blob += 1 check("每条目的 GGUF 源张量都存在", not bad_name, str(sorted(set(bad_name))[:10])) check("源类型均在可实现集合内", not bad_type, str(bad_type[:6])) - check("shape 与 ne 反序全等（含 conv1d squeeze）", not bad_shape, str(bad_shape[:8])) - check("blob 条目行字节可整除且与 n_bytes 相符（%d 条）" % ok_blob, not bad_rows, str(bad_rows[:6])) + check( + "shape 与 ne 反序全等（含 conv1d squeeze）", not bad_shape, str(bad_shape[:8]) + ) + check( + "blob 条目行字节可整除且与 n_bytes 相符（%d 条）" % ok_blob, + not bad_rows, + str(bad_rows[:6]), + ) print("\n== 3. 切片覆盖 + 反向无遗漏 ==") shared = collections.defaultdict(list) @@ -183,8 +236,11 @@ def gguf_side(path): continue if not segs: cov.append("%s: %d 个条目共用但无 slices 声明" % (name, len(es))) - elif segs[0][0] != 0 or segs[-1][1] != n_out or \ - any(segs[i][1] != segs[i + 1][0] for i in range(len(segs) - 1)): + elif ( + segs[0][0] != 0 + or segs[-1][1] != n_out + or any(segs[i][1] != segs[i + 1][0] for i in range(len(segs) - 1)) + ): cov.append("%s: 切片 %s 未无重叠覆盖 [0,%d)" % (name, segs, n_out)) check("共用源张量的切片精确覆盖全行", not cov, str(cov[:6])) @@ -192,56 +248,97 @@ def gguf_side(path): dropped = {n for n in tensors if n.startswith(M.DROP_PREFIXES)} orphan = sorted(set(tensors) - used - dropped) check("无既未消费又未丢弃的张量", not orphan, str(orphan[:10])) - print(" -> 消费 %d 个 / 丢弃 %d 个（MTP blk.%d.*）/ 文件共 %d 个" - % (len(set(tensors) & used), len(dropped), M.MTP_BLOCK, len(tensors))) + print( + " -> 消费 %d 个 / 丢弃 %d 个（MTP blk.%d.*）/ 文件共 %d 个" + % (len(set(tensors) & used), len(dropped), M.MTP_BLOCK, len(tensors)) + ) print("\n== 4. 阶段 3 kernel 作用域 ==") all_types = collections.Counter() for hist in type_hist.values(): all_types.update(hist) - print(" 按条目统计：" + ", ".join("%s x%d" % (tn, c) for tn, c in all_types.most_common())) - blob_types = {e.gguf: TYPE_NAME.get(int(tensors[e.gguf].tensor_type)) - for e in plan if e.blob and e.gguf in tensors} + print( + " 按条目统计：" + + ", ".join("%s x%d" % (tn, c) for tn, c in all_types.most_common()) + ) + blob_types = { + e.gguf: TYPE_NAME.get(int(tensors[e.gguf].tensor_type)) + for e in plan + if e.blob and e.gguf in tensors + } seen = collections.Counter(blob_types.values()) - print(" blob 条目源类型：" + ", ".join("%s x%d" % (tn, c) for tn, c in seen.most_common())) - check("阶段 3 v1 需实现的类型集合 = %s" % sorted(seen), - set(seen) == set(M.NATIVE_BLOB_TYPES), - "缺 %s / 多 %s" % (set(M.NATIVE_BLOB_TYPES) - set(seen), set(seen) - set(M.NATIVE_BLOB_TYPES))) - check("IQ4_* 已被 v1 稠密化例外排除", not ({"IQ4_NL", "IQ4_XS"} & set(seen)), str(sorted(seen))) - giB = 2 ** 30 + print( + " blob 条目源类型：" + + ", ".join("%s x%d" % (tn, c) for tn, c in seen.most_common()) + ) + check( + "阶段 3 v1 需实现的类型集合 = %s" % sorted(seen), + set(seen) == set(M.NATIVE_BLOB_TYPES), + "缺 %s / 多 %s" + % (set(M.NATIVE_BLOB_TYPES) - set(seen), set(seen) - set(M.NATIVE_BLOB_TYPES)), + ) + check( + "IQ4_* 已被 v1 稠密化例外排除", + not ({"IQ4_NL", "IQ4_XS"} & set(seen)), + str(sorted(seen)), + ) + giB = 2**30 total = sum(int(t.n_bytes) for t in reader.tensors) blob_src = {e.gguf for e in plan if e.blob and e.gguf in tensors} dense_src = {e.gguf for e in plan if not e.blob and e.gguf in tensors} - blob_src b = sum(int(tensors[n].n_bytes) for n in blob_src) d_src = sum(int(tensors[n].n_bytes) for n in dense_src) drop = sum(int(t.n_bytes) for n, t in tensors.items() if n in dropped) - print(" -> 文件 %.3f GiB = 逐字节 blob %.3f（%d 个） + 稠密化源 %.3f（%d 个）" - " + MTP 丢弃 %.3f" % (total / giB, b / giB, len(blob_src), - d_src / giB, len(dense_src), drop / giB)) + print( + " -> 文件 %.3f GiB = 逐字节 blob %.3f（%d 个） + 稠密化源 %.3f（%d 个）" + " + MTP 丢弃 %.3f" + % (total / giB, b / giB, len(blob_src), d_src / giB, len(dense_src), drop / giB) + ) # 稠密化条目的显存 = InfiniLM 元素数 x 2B（按行段拆分的条目只算自己那段） dense_bf16 = sum(prod(e.shape) * 2 for e in plan if not e.blob) budget = (b + dense_bf16) / giB - print(" -> v1 显存预算：blob %.3f + 稠密化 BF16 %.3f = %.3f GiB" - % (b / giB, dense_bf16 / giB, budget)) - check("v1 权重预算 <= 24.0 GiB（单卡 5090 32.6 GiB 留 KV 余量）", - budget <= 24.0, "%.3f GiB" % budget) + print( + " -> v1 显存预算：blob %.3f + 稠密化 BF16 %.3f = %.3f GiB" + % (b / giB, dense_bf16 / giB, budget) + ) + check( + "v1 权重预算 <= 24.0 GiB（单卡 5090 32.6 GiB 留 KV 余量）", + budget <= 24.0, + "%.3f GiB" % budget, + ) # 阶段 6 复利：IQ4 上原生 kernel 后再省；emb/lm_head 上 kernel 再省 2.51 GiB st6 = budget - dense_iq_bf16(plan, tensors, gguf_types, prod) / giB - emb_out_blob = int(tensors["token_embd.weight"].n_bytes) + int(tensors["output.weight"].n_bytes) - emb_out_bf16 = sum(prod(e.shape) * 2 for e in plan - if not e.blob and e.gguf in ("token_embd.weight", "output.weight")) + emb_out_blob = int(tensors["token_embd.weight"].n_bytes) + int( + tensors["output.weight"].n_bytes + ) + emb_out_bf16 = sum( + prod(e.shape) * 2 + for e in plan + if not e.blob and e.gguf in ("token_embd.weight", "output.weight") + ) st6b = st6 - (emb_out_bf16 - emb_out_blob) / giB - print(" -> 阶段 6：IQ4 原生 kernel %.3f GiB；再 emb/lm_head 原生 %.3f GiB" - % (st6, st6b)) - check("阶段 6 预算单调下降", st6b < st6 < budget, "%.3f / %.3f / %.3f" % (st6b, st6, budget)) - check("阶段 6 目标态 <= 20.5 GiB（相对路线 A 的 26.6 GiB 权重）", st6b <= 20.5, - "%.3f GiB" % st6b) + print( + " -> 阶段 6：IQ4 原生 kernel %.3f GiB；再 emb/lm_head 原生 %.3f GiB" + % (st6, st6b) + ) + check( + "阶段 6 预算单调下降", + st6b < st6 < budget, + "%.3f / %.3f / %.3f" % (st6b, st6, budget), + ) + check( + "阶段 6 目标态 <= 20.5 GiB（相对路线 A 的 26.6 GiB 权重）", + st6b <= 20.5, + "%.3f GiB" % st6b, + ) def main(): ap = argparse.ArgumentParser() ap.add_argument("--gguf", default=DEFAULT_GGUF) - ap.add_argument("--skip-min", action="store_true", help="跳过需要 infinilm 的框架侧检查") + ap.add_argument( + "--skip-min", action="store_true", help="跳过需要 infinilm 的框架侧检查" + ) ap.add_argument("--engine-device", default="cpu") a = ap.parse_args() diff --git a/scripts/gguf_routeb_stage2_check.py b/scripts/gguf_routeb_stage2_check.py index fb20ba130..eb99aaeb9 100644 --- a/scripts/gguf_routeb_stage2_check.py +++ b/scripts/gguf_routeb_stage2_check.py @@ -42,17 +42,19 @@ _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -sys.path.insert(0, os.path.join( - os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py")) +sys.path.insert( + 0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py") +) DEFAULT_MODEL = "/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8" BLOB_SUFFIX = "weight_bytes" # 与 csrc/layers/quantization/gguf.cpp 里那条诊断日志的格式对应 BLOB_RE = re.compile( r"linear_gguf: 首个 blob 前向 (\S+) — M=(\d+) N=(\d+) K=(\d+) " - r"ggml_type=(\d+) row_bytes=(\d+)") -MAX_DECODE_M = 8 # kMaxDecodeM：<=8 走 gemv，>8 走 prefill（阶段 3.3 起不再是上限） -PROMPT_TOKENS = 3 # 下面 forward_raw 喂的 token 数，用来核对契约行的 M + r"ggml_type=(\d+) row_bytes=(\d+)" +) +MAX_DECODE_M = 8 # kMaxDecodeM：<=8 走 gemv，>8 走 prefill（阶段 3.3 起不再是上限） +PROMPT_TOKENS = 3 # 下面 forward_raw 喂的 token 数，用来核对契约行的 M _PASS = 0 _FAIL = 0 @@ -77,44 +79,64 @@ def main(): a = ap.parse_args() import infinicore + from gguf.constants import GGML_QUANT_SIZES from infinilm.cache import StaticKVCacheConfig from infinilm.distributed import DistConfig from infinilm.infer_engine import InferEngine from infinilm.modeling_utils import load_model_state_dict_by_file from safetensors import safe_open - from gguf.constants import GGML_QUANT_SIZES # ---------------------------------------------------------------- 0. config print("\n== 0. 产物 config.json ==") with open(os.path.join(a.model_path, "config.json")) as f: cfg = json.load(f) qc = cfg.get("quantization_config") or {} - check("quantization_config 在顶层且 quant_method=gguf", qc.get("quant_method") == "gguf", - "qc keys=%s" % sorted(qc)) + check( + "quantization_config 在顶层且 quant_method=gguf", + qc.get("quant_method") == "gguf", + "qc keys=%s" % sorted(qc), + ) table = qc.get("ggml_types") or {} check("类型表非空（%d 条）" % len(table), bool(table)) bs_ts = {int(t): (int(v[0]), int(v[1])) for t, v in GGML_QUANT_SIZES.items()} ids = sorted({v for v in table.values() if isinstance(v, int)}) - check("表内 type id 都能从 gguf-py 查出 (block_size, type_size)：%s" % ids, - all(i in bs_ts for i in ids), str([i for i in ids if i not in bs_ts])) + check( + "表内 type id 都能从 gguf-py 查出 (block_size, type_size)：%s" % ids, + all(i in bs_ts for i in ids), + str([i for i in ids if i not in bs_ts]), + ) with open(os.path.join(a.model_path, "model.safetensors.index.json")) as f: weight_map = json.load(f)["weight_map"] n_blob = sum(1 for v in table.values() if isinstance(v, int)) - print(" -> 类型表 %d 条：blob %d / 稠密 %d；产物 index %d 个张量；key_prefix='%s'" - % (len(table), n_blob, len(table) - n_blob, len(weight_map), qc.get("key_prefix"))) + print( + " -> 类型表 %d 条：blob %d / 稠密 %d；产物 index %d 个张量；key_prefix='%s'" + % ( + len(table), + n_blob, + len(table) - n_blob, + len(weight_map), + qc.get("key_prefix"), + ) + ) # 溯源：表键有两种历史形态。新规则（§6.0 纠正 2）= 张量名原文（与产物 index 同名）； # 旧规则 = 去前缀的相对名且 blob 归一成 .weight（与 index 不同名）。两者 C++ 都能 # 命中（裁前缀时 key_prefix 缺失就取 ""，探键时 weight_bytes / weight 都探）， # 但必须知道眼下这份产物是哪一种，不然对不上时会查错方向。 n_ident = len(set(table) & set(weight_map)) - print(" -> 表键形态：%d/%d 条与产物张量名同名（新规则），其余 %d 条为相对名或前缀外键" - % (n_ident, len(table), len(table) - n_ident)) + print( + " -> 表键形态：%d/%d 条与产物张量名同名（新规则），其余 %d 条为相对名或前缀外键" + % (n_ident, len(table), len(table) - n_ident) + ) # ------------------------------------------------------------- 1. 构造引擎 print("\n== 1. 用 GGUFBlockQuantization 构造引擎（device=%s）==" % a.device) # infinicore.device("cuda:0", 0) 会报 “index should not be provided”，带冒号就不能再传 index - dev_spec = infinicore.device(a.device) if ":" in a.device else infinicore.device(a.device, 0) + dev_spec = ( + infinicore.device(a.device) + if ":" in a.device + else infinicore.device(a.device, 0) + ) try: eng = InferEngine( model_path=a.model_path, @@ -123,24 +145,34 @@ def main(): cache_config=StaticKVCacheConfig(max_batch_size=1, max_cache_len=16), ) ok, err = True, "" - except Exception as e: # noqa: BLE001 + except Exception as e: # noqa: BLE001 ok, err = False, "%s: %s" % (type(e).__name__, str(e)[:1200]) - check("构造通过（= 被查询的 stem 全部恰好命中 1 个候选，且无 TP/bias 违规）", ok, err) + check( + "构造通过（= 被查询的 stem 全部恰好命中 1 个候选，且无 TP/bias 违规）", ok, err + ) if not ok: print("\n构造都没过，后面全部跳过\n" + traceback.format_exc()) return 1 - check("引擎确实走 GGUF 方案", - (eng.hf_config.get("quantization_config") or {}).get("quant_method") == "gguf") + check( + "引擎确实走 GGUF 方案", + (eng.hf_config.get("quantization_config") or {}).get("quant_method") == "gguf", + ) # --------------------------------------------------------- 2. 键双向 diff print("\n== 2. 引擎参数键 vs 产物张量名 ==") keys = list(eng.state_dict_keyname()) extra = sorted(set(keys) - set(weight_map)) missing = sorted(set(weight_map) - set(keys)) - check("产物有、引擎不要（多键 -> strict=False 下静默丢权重）", not extra, str(extra[:12])) + check( + "产物有、引擎不要（多键 -> strict=False 下静默丢权重）", + not extra, + str(extra[:12]), + ) check("引擎要、产物没有（缺键 -> 保持随机初始化）", not missing, str(missing[:12])) - check("键数一致（引擎 %d / 产物 %d）" % (len(keys), len(weight_map)), - len(keys) == len(weight_map)) + check( + "键数一致（引擎 %d / 产物 %d）" % (len(keys), len(weight_map)), + len(keys) == len(weight_map), + ) # ---------------------------------------------- 3. 逐键 dtype / shape 对账 print("\n== 3. 逐键 shape 对账（blob 行字节独立重算）==") @@ -150,7 +182,10 @@ def main(): with safe_open(os.path.join(a.model_path, fn), framework="pt") as f: for k in f.keys(): if k in sd_keys: - meta[k] = (f.get_slice(k).get_dtype(), list(f.get_slice(k).get_shape())) + meta[k] = ( + f.get_slice(k).get_dtype(), + list(f.get_slice(k).get_shape()), + ) eng_sd = eng.state_dict()[0] # 照抄 C++ GGUFBlockQuantization::resolve() 的查表语义：表键 = 产物名裁掉 @@ -164,11 +199,11 @@ def table_hits(k): cands = {k, k[: -len(W_BLOB)] + ".weight" if k.endswith(W_BLOB) else k} for base in list(cands): if base.startswith(MOD_PREFIX): - cands.add(base[len(MOD_PREFIX):]) + cands.add(base[len(MOD_PREFIX) :]) declared = qc.get("key_prefix") or "" for base in list(cands): if declared and base.startswith(declared): - cands.add(base[len(declared):]) + cands.add(base[len(declared) :]) return sorted(c for c in cands if c in table) bad_shape, bad_dtype, n_blob_eng, n_table_form = [], [], 0, collections.Counter() @@ -179,31 +214,46 @@ def table_hits(k): if k.endswith("." + BLOB_SUFFIX): n_blob_eng += 1 if "U8" not in str(eng_sd[k].dtype).upper() or meta[k][0] != "U8": - bad_dtype.append("%s: 引擎 %s / 产物 %s" % (k, eng_sd[k].dtype, meta[k][0])) + bad_dtype.append( + "%s: 引擎 %s / 产物 %s" % (k, eng_sd[k].dtype, meta[k][0]) + ) hits = table_hits(k) if len(hits) != 1: - bad_shape.append("%s: 类型表命中 %d 个候选 %s（C++ 会抛或静默走稠密）" - % (k, len(hits), hits[:4])) + bad_shape.append( + "%s: 类型表命中 %d 个候选 %s（C++ 会抛或静默走稠密）" + % (k, len(hits), hits[:4]) + ) continue n_table_form["与张量名同名" if hits[0] == k else "相对名/归一后缀"] += 1 _bs, ts = bs_ts[int(table[hits[0]])] if e_shape and ts and e_shape[-1] % ts: - bad_shape.append("%s: row_bytes=%d 不是 type_size %d 的整数倍" - % (k, e_shape[-1], ts)) - check("%d 个 blob 键在引擎侧与产物侧都是 U8" % n_blob_eng, not bad_dtype, - str(bad_dtype[:6])) - check("全部 %d 键 shape 逐字相等（blob 为 [out, row_bytes]）" % len(sd_keys), - not bad_shape, str(bad_shape[:8])) - print(" -> %d 个 blob 命中的表键形态：%s" - % (n_blob_eng, ", ".join("%s x%d" % kv for kv in n_table_form.most_common()) - or "无")) + bad_shape.append( + "%s: row_bytes=%d 不是 type_size %d 的整数倍" % (k, e_shape[-1], ts) + ) + check( + "%d 个 blob 键在引擎侧与产物侧都是 U8" % n_blob_eng, + not bad_dtype, + str(bad_dtype[:6]), + ) + check( + "全部 %d 键 shape 逐字相等（blob 为 [out, row_bytes]）" % len(sd_keys), + not bad_shape, + str(bad_shape[:8]), + ) + print( + " -> %d 个 blob 命中的表键形态：%s" + % ( + n_blob_eng, + ", ".join("%s x%d" % kv for kv in n_table_form.most_common()) or "无", + ) + ) # ----------------------------------------------------------------- 4. 加载 print("\n== 4. 加载（末尾 check_parameters 会对缺/多键抛错 = 判据 1）==") try: load_model_state_dict_by_file(eng, a.model_path, dtype=eng.dtype) ok, err = True, "" - except Exception as e: # noqa: BLE001 + except Exception as e: # noqa: BLE001 ok, err = False, "%s: %s" % (type(e).__name__, str(e)[:1200]) check("%d 个条目全部装载完毕" % len(weight_map), ok, err) @@ -211,12 +261,15 @@ def table_hits(k): if a.no_forward: print("\n== 5. 跳过（--no-forward）==") else: - print("\n== 5. 首个 blob Linear 必须进 linear_gguf 并返回（判据 3：不静默回落稠密）==") + print( + "\n== 5. 首个 blob Linear 必须进 linear_gguf 并返回（判据 3：不静默回落稠密）==" + ) import torch def to_dev(t): return infinicore.from_torch( - t.cuda(0) if a.device.startswith("cuda") else t) + t.cuda(0) if a.device.startswith("cuda") else t + ) ids = to_dev(torch.tensor([[114, 5, 7]], dtype=torch.int32)) # qwen3_5 是 mrope（position_id_axes=3），position_ids 的轴序在 C++ 侧 @@ -226,6 +279,7 @@ def to_dev(t): to_dev(torch.tensor([[0, 1, 2], [0, 1, 2], [0, 1, 2]], dtype=torch.int32)), to_dev(torch.tensor([[0, 1, 2]], dtype=torch.int32)), ] + # RankWorker 会把工作线程里的异常换个文案再抛一次（python 侧只看到 # “RankWorker is closing”），真实抛出只落在 spdlog 里。实测 spdlog 走的是 # **stdout**（把 2 单独分流到文件后 “linear_gguf” 那条 [error] 仍留在 @@ -234,8 +288,11 @@ def open_cap(): try: return os.memfd_create("stage2_log") except AttributeError: - return os.open(os.path.join(_HERE, ".stage2_log.tmp"), - os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600) + return os.open( + os.path.join(_HERE, ".stage2_log.tmp"), + os.O_RDWR | os.O_CREAT | os.O_TRUNC, + 0o600, + ) libc = ctypes.CDLL(None) caps = {fd: open_cap() for fd in (1, 2)} @@ -253,11 +310,13 @@ def open_cap(): eng.forward_raw(input_ids=ids, position_ids=pos) msgs.append("<没抛异常：blob 被当成稠密权重跑了！>") break - except Exception as e: # noqa: BLE001 - msgs.append("%s: %s" % (type(e).__name__, - str(e).strip().splitlines()[0][:200])) + except Exception as e: # noqa: BLE001 + msgs.append( + "%s: %s" + % (type(e).__name__, str(e).strip().splitlines()[0][:200]) + ) finally: - libc.fflush(None) # C++ 侧重定向到文件时是块缓冲，不冲读不到 + libc.fflush(None) # C++ 侧重定向到文件时是块缓冲，不冲读不到 sys.stdout.flush() sys.stderr.flush() for fd, mem in caps.items(): @@ -269,25 +328,29 @@ def open_cap(): os.lseek(mem, 0, os.SEEK_SET) captured += os.read(mem, 1 << 20).decode("utf-8", "replace") os.close(mem) - haystack = "\n".join(msgs) + "\n" + captured line = next((ln for ln in captured.splitlines() if "linear_gguf" in ln), "") m = BLOB_RE.search(line) if not m: - check("首个 blob Linear 进入 linear_gguf 并返回（未回落稠密）", False, - "python: %s\n 日志尾部: %s" % (" | ".join(msgs), - captured[-600:])) + check( + "首个 blob Linear 进入 linear_gguf 并返回（未回落稠密）", + False, + "python: %s\n 日志尾部: %s" + % (" | ".join(msgs), captured[-600:]), + ) else: M, N, K, tid, row_bytes = [int(m.group(i)) for i in range(2, 7)] check("首个 blob Linear 进入 linear_gguf 并返回（未回落稠密）", True) # 只留 linear_gguf 之后的部分：spdlog 前缀占掉大半行，按整行截断会把张量名切掉 - print(" %s" % line[line.find("linear_gguf"):].strip()) + print(" %s" % line[line.find("linear_gguf") :].strip()) bs, ts = bs_ts[tid] # 阶段 3.3 前这里评的是“M <= 8”（当时的 decode 护栏）；现在 M 的唯一 # 契约是“等于本次喂进去的 token 数”，大了小了都算错。 - check("契约行自洽：M=%d 等于 prompt token 数 %d 且 row_bytes=%d == (K/%d)*%d" - % (M, PROMPT_TOKENS, row_bytes, bs, ts), - M == PROMPT_TOKENS and row_bytes == (K // bs) * ts, - "type=%d (block_size, type_size)=(%d,%d)" % (tid, bs, ts)) + check( + "契约行自洽：M=%d 等于 prompt token 数 %d 且 row_bytes=%d == (K/%d)*%d" + % (M, PROMPT_TOKENS, row_bytes, bs, ts), + M == PROMPT_TOKENS and row_bytes == (K // bs) * ts, + "type=%d (block_size, type_size)=(%d,%d)" % (tid, bs, ts), + ) print("\n== 结果：%d PASS / %d FAIL ==" % (_PASS, _FAIL)) return 0 if _FAIL == 0 else 1 diff --git a/scripts/gguf_routeb_stage3_check.py b/scripts/gguf_routeb_stage3_check.py index d443c4bda..c5b10d7c1 100644 --- a/scripts/gguf_routeb_stage3_check.py +++ b/scripts/gguf_routeb_stage3_check.py @@ -40,16 +40,18 @@ import tempfile _HERE = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.join( - os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py")) +sys.path.insert( + 0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py") +) DEFAULT_MODEL = "/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8" BLOB_RE = re.compile( r"linear_gguf: 首个 blob 前向 (\S+) — M=(\d+) N=(\d+) K=(\d+) " - r"ggml_type=(\d+) row_bytes=(\d+)") -MAX_DECODE_M = 8 # kMaxDecodeM：<=8 走 gemv，>8 走 prefill（两条路径同一个谓词） -PREFILL_M = 12 # > MAX_DECODE_M：阶段 3.3 的 prefill 正例（旧行为是必抛） -DECODE_M = 4 # <= MAX_DECODE_M：decode 回归用例 + r"ggml_type=(\d+) row_bytes=(\d+)" +) +MAX_DECODE_M = 8 # kMaxDecodeM：<=8 走 gemv，>8 走 prefill（两条路径同一个谓词） +PREFILL_M = 12 # > MAX_DECODE_M：阶段 3.3 的 prefill 正例（旧行为是必抛） +DECODE_M = 4 # <= MAX_DECODE_M：decode 回归用例 _PASS = 0 _FAIL = 0 @@ -75,8 +77,11 @@ def _open_cap(): try: return os.open(path, os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600) except OSError: - return os.open(os.path.join(_HERE, ".stage3_log.tmp"), - os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600) + return os.open( + os.path.join(_HERE, ".stage3_log.tmp"), + os.O_RDWR | os.O_CREAT | os.O_TRUNC, + 0o600, + ) class capture: @@ -100,7 +105,7 @@ def __enter__(self): return self def __exit__(self, *exc): - self._libc.fflush(None) # C++ 侧块缓冲，不冲就读不到 + self._libc.fflush(None) # C++ 侧块缓冲，不冲就读不到 sys.stdout.flush() sys.stderr.flush() for fd, mem in self._caps.items(): @@ -122,17 +127,21 @@ def count_blob_calls(inner_argv): script = os.path.join(tempfile.gettempdir(), "stage3_count.gdb") try: with open(script, "w") as f: - f.write("set pagination off\nset confirm off\n" - "set breakpoint pending on\n" - "break infiniopLinearGguf\ncommands\nsilent\ncontinue\nend\n" - "run\nprintf \"\\n===BPSTAT===\\n\"\ninfo breakpoints\n") + f.write( + "set pagination off\nset confirm off\n" + "set breakpoint pending on\n" + "break infiniopLinearGguf\ncommands\nsilent\ncontinue\nend\n" + 'run\nprintf "\\n===BPSTAT===\\n"\ninfo breakpoints\n' + ) except OSError: script = os.path.join(_HERE, ".stage3_count.gdb") with open(script, "w") as f: - f.write("set pagination off\nset confirm off\n" - "set breakpoint pending on\n" - "break infiniopLinearGguf\ncommands\nsilent\ncontinue\nend\n" - "run\nprintf \"\\n===BPSTAT===\\n\"\ninfo breakpoints\n") + f.write( + "set pagination off\nset confirm off\n" + "set breakpoint pending on\n" + "break infiniopLinearGguf\ncommands\nsilent\ncontinue\nend\n" + 'run\nprintf "\\n===BPSTAT===\\n"\ninfo breakpoints\n' + ) cmd = ["gdb", "-q", "-batch", "-x", script, "--args", sys.executable] + inner_argv print(" -> %s" % " ".join(cmd[:8])) p = subprocess.run(cmd, capture_output=True, text=True) @@ -148,23 +157,29 @@ def main(): ap.add_argument("--num-blocks", type=int, default=16) ap.add_argument("--block-size", type=int, default=256) ap.add_argument("--count-blob-calls", action="store_true") - ap.add_argument("--route-b-inner", action="store_true", - help="内部用：gdb 子进程模式，只做前 6 条判据") + ap.add_argument( + "--route-b-inner", + action="store_true", + help="内部用：gdb 子进程模式，只做前 6 条判据", + ) a, _unknown = ap.parse_known_args() + import json + import infinicore + from gguf.constants import GGML_QUANT_SIZES from infinilm.cache import PagedKVCacheConfig from infinilm.distributed import DistConfig from infinilm.infer_engine import GenerationConfig, InferEngine from infinilm.modeling_utils import load_model_state_dict_by_file - from gguf.constants import GGML_QUANT_SIZES - import json with open(os.path.join(a.model_path, "config.json")) as f: cfg = json.load(f) table = (cfg.get("quantization_config") or {}).get("ggml_types") or {} n_blob = sum(1 for v in table.values() if isinstance(v, int)) - text_cfg = cfg.get("text_config") if isinstance(cfg.get("text_config"), dict) else cfg + text_cfg = ( + cfg.get("text_config") if isinstance(cfg.get("text_config"), dict) else cfg + ) vocab = int(text_cfg.get("vocab_size") or 0) def build(): @@ -172,28 +187,33 @@ def build(): model_path=a.model_path, device=infinicore.device("cuda:0"), distributed_config=DistConfig(1), - cache_config=PagedKVCacheConfig(a.num_blocks, a.block_size, - max_batch_size=1), + cache_config=PagedKVCacheConfig( + a.num_blocks, a.block_size, max_batch_size=1 + ), attention_backend="paged-attn", ) # ------------------------------------------------------------- 1. 构造加载 - print("\n== 1. paged 引擎构造 + 加载（blob %d / 稠密 %d）==" % (n_blob, - len(table) - n_blob)) + print( + "\n== 1. paged 引擎构造 + 加载（blob %d / 稠密 %d）==" + % (n_blob, len(table) - n_blob) + ) try: eng = build() ok, err = True, "" - except Exception as e: # noqa: BLE001 + except Exception as e: # noqa: BLE001 ok, err = False, "%s: %s" % (type(e).__name__, str(e)[:1200]) check("构造通过（PagedKVCacheConfig + paged-attn）", ok, err) if not ok: return 1 - check("has_mamba_cache 且 enable_paged_attn（GDN 模型只能走这条路）", - eng.has_mamba_cache and eng.enable_paged_attn) + check( + "has_mamba_cache 且 enable_paged_attn（GDN 模型只能走这条路）", + eng.has_mamba_cache and eng.enable_paged_attn, + ) try: load_model_state_dict_by_file(eng, a.model_path, dtype=eng.dtype) ok, err = True, "" - except Exception as e: # noqa: BLE001 + except Exception as e: # noqa: BLE001 ok, err = False, "%s: %s" % (type(e).__name__, str(e)[:1200]) check("权重装载完毕", ok, err) if not ok: @@ -201,9 +221,17 @@ def build(): def do_generate(tokens): ids = infinicore.from_list([tokens], dtype=infinicore.int64) - out = eng.generate(ids, GenerationConfig( - max_new_tokens=a.new_tokens, temperature=0.0, top_k=1, top_p=1.0, - eos_token_id=None, stop_on_eos=False)) + out = eng.generate( + ids, + GenerationConfig( + max_new_tokens=a.new_tokens, + temperature=0.0, + top_k=1, + top_p=1.0, + eos_token_id=None, + stop_on_eos=False, + ), + ) return [int(x.to_numpy().reshape(-1)[0]) for x in out] # 成功的 generate 次数；每完成一次 = 1 次 prefill + (new_tokens-1) 次 decode @@ -217,75 +245,109 @@ def one_generate(tokens): return toks # ------------------------------------- 2/3/4/5. prefill 正例（prompt > decode 上限） - print("\n== 2-5. prefill：prompt=%d token（> kMaxDecodeM=%d）==" % (PREFILL_M, MAX_DECODE_M)) + print( + "\n== 2-5. prefill：prompt=%d token（> kMaxDecodeM=%d）==" + % (PREFILL_M, MAX_DECODE_M) + ) pre_prompt = list(range(100, 100 + PREFILL_M)) with capture() as cap: try: toks1 = one_generate(pre_prompt) perr = "" - except BaseException as e: # noqa: BLE001 - toks1, perr = None, "%s: %s" % ( - type(e).__name__, str(e).strip().splitlines()[:1]) + except BaseException as e: # noqa: BLE001 + toks1, perr = ( + None, + "%s: %s" % (type(e).__name__, str(e).strip().splitlines()[:1]), + ) log1 = cap.captured - check("prefill generate 走完 %d 步（M=%d 不再抛）" % (a.new_tokens, PREFILL_M), - toks1 is not None, perr + "\n 日志尾部: " + log1[-500:]) + check( + "prefill generate 走完 %d 步（M=%d 不再抛）" % (a.new_tokens, PREFILL_M), + toks1 is not None, + perr + "\n 日志尾部: " + log1[-500:], + ) if toks1 is None: print("\n== 结果：%d PASS / %d FAIL ==" % (_PASS, _FAIL)) return 1 print(" tokens=%s" % toks1) - check("token id 落在词表 [0,%d) 内" % vocab, - not vocab or all(0 <= t < vocab for t in toks1)) + check( + "token id 落在词表 [0,%d) 内" % vocab, + not vocab or all(0 <= t < vocab for t in toks1), + ) toks2 = one_generate(pre_prompt) check("贪心两次结果逐字相同", toks1 == toks2, "%s vs %s" % (toks1, toks2)) m = BLOB_RE.search(log1) - check("日志出现 blob 前向契约行（= blob 没被当稠密权重跑）", bool(m), - "捕获 %d 字节，未见 linear_gguf 行" % len(log1)) + check( + "日志出现 blob 前向契约行（= blob 没被当稠密权重跑）", + bool(m), + "捕获 %d 字节，未见 linear_gguf 行" % len(log1), + ) if m: - key, M, N, K, tid, row_bytes = m.group(1), *[int(m.group(i)) for i in - range(2, 7)] - print(" %s — M=%d N=%d K=%d ggml_type=%d row_bytes=%d" - % (key, M, N, K, tid, row_bytes)) + key, M, N, K, tid, row_bytes = ( + m.group(1), + *[int(m.group(i)) for i in range(2, 7)], + ) + print( + " %s — M=%d N=%d K=%d ggml_type=%d row_bytes=%d" + % (key, M, N, K, tid, row_bytes) + ) bs, ts = GGML_QUANT_SIZES[tid] # 契约行是进 kernel 的第一个 blob，而第一个 blob 就在 prompt 的 prefill 里。 # M 必须等于 prompt 长度：小了就是上层把 prompt 拆碎了/没走 prefill。 - check("契约行 M=%d 等于 prompt 长度 %d（整批进 kernel）" % (M, PREFILL_M), - M == PREFILL_M, "M=%d" % M) - check("该批只能由 prefill 路径处理（M=%d > kMaxDecodeM=%d）" % (M, MAX_DECODE_M), - M > MAX_DECODE_M, "M=%d" % M) - check("契约行 row_bytes == (K/%d)*%d 自洽" % (bs, ts), - row_bytes == (K // int(bs)) * int(ts), - "row_bytes=%d 期望=%d" % (row_bytes, (K // int(bs)) * int(ts))) + check( + "契约行 M=%d 等于 prompt 长度 %d（整批进 kernel）" % (M, PREFILL_M), + M == PREFILL_M, + "M=%d" % M, + ) + check( + "该批只能由 prefill 路径处理（M=%d > kMaxDecodeM=%d）" % (M, MAX_DECODE_M), + M > MAX_DECODE_M, + "M=%d" % M, + ) + check( + "契约行 row_bytes == (K/%d)*%d 自洽" % (bs, ts), + row_bytes == (K // int(bs)) * int(ts), + "row_bytes=%d 期望=%d" % (row_bytes, (K // int(bs)) * int(ts)), + ) # ------------------------------------------- 6. decode 回归（短 prompt 仍可用） - print("\n== 6. decode 回归：prompt=%d token（<= %d，仍走 gemv）==" - % (DECODE_M, MAX_DECODE_M)) + print( + "\n== 6. decode 回归：prompt=%d token（<= %d，仍走 gemv）==" + % (DECODE_M, MAX_DECODE_M) + ) dec_prompt = list(range(300, 300 + DECODE_M)) try: toks3 = one_generate(dec_prompt) err3 = "" - except BaseException as e: # noqa: BLE001 + except BaseException as e: # noqa: BLE001 toks3, err3 = None, "%s: %s" % (type(e).__name__, str(e).strip()[:200]) - check("短 prompt 用例走完 %d 步（撤护栏未弄坏 gemv 路径）" % a.new_tokens, - toks3 is not None, err3) + check( + "短 prompt 用例走完 %d 步（撤护栏未弄坏 gemv 路径）" % a.new_tokens, + toks3 is not None, + err3, + ) if toks3 is not None: print(" tokens=%s" % toks3) # --------------------------------------------------- 7. 断点命中数（可选） if a.count_blob_calls and not a.route_b_inner: print("\n== 7. gdb 断点计数：每步 × 每个 blob ==") - inner = [os.path.abspath(sys.argv[0])] + \ - [x for x in sys.argv[1:] if x != "--count-blob-calls"] + \ - ["--route-b-inner"] + inner = ( + [os.path.abspath(sys.argv[0])] + + [x for x in sys.argv[1:] if x != "--count-blob-calls"] + + ["--route-b-inner"] + ) n, tail = count_blob_calls(inner) steps = re.search(r"INNER_STEPS=(\d+)", tail) steps = int(steps.group(1)) if steps else None expect = steps * n_blob if steps else None - check("infiniopLinearGguf 命中 %s 次 == 步数 %s × blob %d = %s" - % (n, steps, n_blob, expect), n is not None and n == expect, - "实际 %s / 期望 %s\n 子进程输出尾部: %s" - % (n, expect, tail[-600:])) + check( + "infiniopLinearGguf 命中 %s 次 == 步数 %s × blob %d = %s" + % (n, steps, n_blob, expect), + n is not None and n == expect, + "实际 %s / 期望 %s\n 子进程输出尾部: %s" % (n, expect, tail[-600:]), + ) elif a.route_b_inner: # 子进程里：把实际完成的前向步数报给外层。每次 generate = 1 次 prefill + # (max_new_tokens-1) 次 decode = max_new_tokens 步；本脚本一共跑 3 次。 diff --git a/scripts/gguf_routeb_tokenizer_check.py b/scripts/gguf_routeb_tokenizer_check.py index 1b3940edb..29eca2cdf 100755 --- a/scripts/gguf_routeb_tokenizer_check.py +++ b/scripts/gguf_routeb_tokenizer_check.py @@ -58,7 +58,8 @@ def main() -> int: tok_default = AutoTokenizer.from_pretrained(args.model_path, **common) try: tok_fixed = AutoTokenizer.from_pretrained( - args.model_path, fix_mistral_regex=True, **common) + args.model_path, fix_mistral_regex=True, **common + ) fixed_error = None except Exception as exc: # compatibility with older transformers tok_fixed = None @@ -67,32 +68,50 @@ def main() -> int: results = [] default_ok = fixed_ok = True for case in cases: - llama = post_json(args.server.rstrip("/") + "/tokenize", { - "content": case["prompt"], - "add_special": False, - "parse_special": True, - "with_pieces": False, - })["tokens"] + llama = post_json( + args.server.rstrip("/") + "/tokenize", + { + "content": case["prompt"], + "add_special": False, + "parse_special": True, + "with_pieces": False, + }, + )["tokens"] llama = [int(x) for x in llama] - local_default = [int(x) for x in tok_default.encode( - case["prompt"], add_special_tokens=False)] - local_fixed = None if tok_fixed is None else [int(x) for x in tok_fixed.encode( - case["prompt"], add_special_tokens=False)] + local_default = [ + int(x) for x in tok_default.encode(case["prompt"], add_special_tokens=False) + ] + local_fixed = ( + None + if tok_fixed is None + else [ + int(x) + for x in tok_fixed.encode(case["prompt"], add_special_tokens=False) + ] + ) match_default = llama == local_default match_fixed = local_fixed is not None and llama == local_fixed default_ok &= match_default fixed_ok &= match_fixed - results.append({ - **case, - "input_ids": llama, - "local_default_ids": local_default, - "local_fixed_ids": local_fixed, - "default_match": match_default, - "fixed_match": match_fixed, - }) - print("%-10s llama=%3d default=%s fixed=%s" % ( - case["id"], len(llama), match_default, - "NA" if local_fixed is None else str(match_fixed))) + results.append( + { + **case, + "input_ids": llama, + "local_default_ids": local_default, + "local_fixed_ids": local_fixed, + "default_match": match_default, + "fixed_match": match_fixed, + } + ) + print( + "%-10s llama=%3d default=%s fixed=%s" + % ( + case["id"], + len(llama), + match_default, + "NA" if local_fixed is None else str(match_fixed), + ) + ) if default_ok: selected_variant = "default" @@ -115,8 +134,10 @@ def main() -> int: os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) with open(args.out, "w", encoding="utf-8") as f: json.dump(output, f, ensure_ascii=False, indent=2) - print("RESULT default_all=%s fixed_all=%s selected=%s cases=%d" % ( - default_ok, fixed_ok, selected_variant, len(results))) + print( + "RESULT default_all=%s fixed_all=%s selected=%s cases=%d" + % (default_ok, fixed_ok, selected_variant, len(results)) + ) return 0 if selected_variant else 1 diff --git a/scripts/gguf_routeb_typecensus.py b/scripts/gguf_routeb_typecensus.py index 5e496acd9..f676fe82d 100644 --- a/scripts/gguf_routeb_typecensus.py +++ b/scripts/gguf_routeb_typecensus.py @@ -1,5 +1,5 @@ -import sys import collections +import sys sys.path.insert(0, "/home/liuxd/llama.cpp/gguf-py") from gguf import GGUFReader # noqa: E402 @@ -17,11 +17,25 @@ def tn(name): per = collections.defaultdict(collections.Counter) for i in range(64): full = (i + 1) % 4 == 0 - names = ([f"blk.{i}.attn_q.weight", f"blk.{i}.attn_k.weight", - f"blk.{i}.attn_v.weight", f"blk.{i}.attn_output.weight"] if full - else [f"blk.{i}.attn_qkv.weight", f"blk.{i}.attn_gate.weight", - f"blk.{i}.ssm_out.weight"]) - names += [f"blk.{i}.ffn_gate.weight", f"blk.{i}.ffn_up.weight", f"blk.{i}.ffn_down.weight"] + names = ( + [ + f"blk.{i}.attn_q.weight", + f"blk.{i}.attn_k.weight", + f"blk.{i}.attn_v.weight", + f"blk.{i}.attn_output.weight", + ] + if full + else [ + f"blk.{i}.attn_qkv.weight", + f"blk.{i}.attn_gate.weight", + f"blk.{i}.ssm_out.weight", + ] + ) + names += [ + f"blk.{i}.ffn_gate.weight", + f"blk.{i}.ffn_up.weight", + f"blk.{i}.ffn_down.weight", + ] for n in names: per[n.split(".")[2]][tn(n)] += 1 @@ -30,8 +44,10 @@ def tn(name): print(f" {k:14s}", dict(v)) print("=== full-attn 层内 q/k/v 类型是否一致（决定融合 blob 能否共用一块 buffer）===") -bad = [(i, [tn(f"blk.{i}.attn_{x}.weight") for x in ("q", "k", "v")]) - for i in range(3, 64, 4)] +bad = [ + (i, [tn(f"blk.{i}.attn_{x}.weight") for x in ("q", "k", "v")]) + for i in range(3, 64, 4) +] bad = [b for b in bad if len(set(b[1])) != 1] print(f" 不一致层数 = {len(bad)} 样例 = {bad[:6]}") @@ -42,11 +58,25 @@ def tn(name): print("=== GDN 层 attn_qkv / attn_gate / ssm_out 抽样类型 ===") for i in (0, 1, 2, 4, 62): - print(" ", i, {s: tn(f"blk.{i}.{s}.weight") for s in - ("attn_qkv", "attn_gate", "ssm_out", "ffn_gate", "ffn_down")}) + print( + " ", + i, + { + s: tn(f"blk.{i}.{s}.weight") + for s in ("attn_qkv", "attn_gate", "ssm_out", "ffn_gate", "ffn_down") + }, + ) print("=== 每个 Linear 的 (角色 -> 类型) 逐层矩阵，看同一角色跨层是否稳定 ===") -for role in ("attn_q", "attn_k", "attn_v", "attn_output", "ffn_gate", "ffn_up", "ffn_down"): +for role in ( + "attn_q", + "attn_k", + "attn_v", + "attn_output", + "ffn_gate", + "ffn_up", + "ffn_down", +): c = collections.Counter() for i in range(64): n = f"blk.{i}.{role}.weight" diff --git a/scripts/gguf_to_infinilm.py b/scripts/gguf_to_infinilm.py index 6857b8ff9..6ffa7e3fc 100644 --- a/scripts/gguf_to_infinilm.py +++ b/scripts/gguf_to_infinilm.py @@ -30,15 +30,17 @@ _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -sys.path.insert(0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py")) - -import numpy as np # noqa: E402 - -import gguf_mapping as M # noqa: E402 -import gguf_transforms as X # noqa: E402 -from gguf import GGUFReader # noqa: E402 -from gguf.constants import GGML_QUANT_SIZES, GGMLQuantizationType as Q # noqa: E402 -from gguf.quants import dequantize # noqa: E402 +sys.path.insert( + 0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py") +) + +import gguf_mapping as M # noqa: E402 +import gguf_transforms as X # noqa: E402 +import numpy as np # noqa: E402 +from gguf import GGUFReader # noqa: E402 +from gguf.constants import GGML_QUANT_SIZES # noqa: E402 +from gguf.constants import GGMLQuantizationType as Q # noqa: E402 +from gguf.quants import dequantize # noqa: E402 DEFAULT_GGUF = "/home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf" DEFAULT_OUT = "/home/liuxd/models/Qwen3.8-27B-GGUF-native" @@ -49,11 +51,19 @@ UNQUANTIZED = ("F32", "F16", "BF16") # 分词器配置文件：词表本身从 GGUF 导出，这些附属文件优先从 --tokenizer-dir 复制。 -TOKENIZER_FILES = ("tokenizer_config.json", "chat_template.jinja", "generation_config.json", - "preprocessor_config.json", "video_preprocessor_config.json", - "special_tokens_map.json", "merges.txt", "vocab.json", "tokenizer.json") +TOKENIZER_FILES = ( + "tokenizer_config.json", + "chat_template.jinja", + "generation_config.json", + "preprocessor_config.json", + "video_preprocessor_config.json", + "special_tokens_map.json", + "merges.txt", + "vocab.json", + "tokenizer.json", +) -_GiB = 2 ** 30 +_GiB = 2**30 def log(msg: str) -> None: @@ -79,6 +89,7 @@ def norm_dtype(s) -> str: # 源 -> 目标：单一实现 # --------------------------------------------------------------------------- + def dense_float32(src: np.ndarray, type_name: str, chunk_rows: int) -> np.ndarray: """源张量的若干行 -> float32 [rows, in]。未量化类型只是换 dtype。""" if type_name in UNQUANTIZED: @@ -96,8 +107,9 @@ def dense_float32(src: np.ndarray, type_name: str, chunk_rows: int) -> np.ndarra out = np.empty((rows,) + first.shape[1:], dtype=np.float32) out[:chunk_rows] = first for i in range(chunk_rows, rows, chunk_rows): - out[i:i + chunk_rows] = np.asarray(dequantize(src[i:i + chunk_rows], q), - dtype=np.float32) + out[i : i + chunk_rows] = np.asarray( + dequantize(src[i : i + chunk_rows], q), dtype=np.float32 + ) return out @@ -107,14 +119,17 @@ def make_blob(e, t, dims, opt): n_out, n_in = int(e.shape[0]), int(e.shape[1]) rb = M.row_bytes(n_in, bs, ts) if int(t.data.shape[-1]) != rb: - raise ValueError("%s: 源行字节 %d != 映射表期望 %d" - % (e.gguf, int(t.data.shape[-1]), rb)) + raise ValueError( + "%s: 源行字节 %d != 映射表期望 %d" % (e.gguf, int(t.data.shape[-1]), rb) + ) arr = t.data if e.slices: s, ep = e.slices[0] arr = arr[s:ep] if int(arr.shape[0]) != n_out: - raise ValueError("%s: 取段后 %d 行 != 映射表 %d" % (e.gguf, arr.shape[0], n_out)) + raise ValueError( + "%s: 取段后 %d 行 != 映射表 %d" % (e.gguf, arr.shape[0], n_out) + ) if M.needs_vperm(e): arr = X.apply_vperm(arr, e, dims, opt.vperm) return torch_from(arr, np.uint8) @@ -143,9 +158,10 @@ def entry_float32(e, t, dims, opt) -> np.ndarray: want = tuple(int(x) for x in e.shape) if tuple(arr.shape) != want: if arr.size != prod(want): - raise ValueError("%s: 变换后 shape %s != 映射表 %s" - % (e.infinilm, arr.shape, want)) - arr = arr.reshape(want) # §2.11 第 5 条：conv1d 补中间维 + raise ValueError( + "%s: 变换后 shape %s != 映射表 %s" % (e.infinilm, arr.shape, want) + ) + arr = arr.reshape(want) # §2.11 第 5 条：conv1d 补中间维 return arr @@ -156,6 +172,7 @@ def make_dense(e, t, dims, opt): def torch_from(arr: np.ndarray, dtype): import torch + t = torch.from_numpy(np.ascontiguousarray(arr)) return t.to(torch.bfloat16) if dtype == "bf16" else t @@ -180,10 +197,17 @@ def build(e, t, dims, opt, dense_all: bool): n_k, r, hd = dims.lin_k_heads, dims.v_per_k, dims.lin_v_dim out_dim, in_dim = int(e.shape[0]), int(e.shape[1]) if in_dim != n_k * r * hd: - raise ValueError("%s: in_dim %d != num_k_heads*num_v_per_k*head_dim = %d，无法按头分块置换列" - % (e.infinilm, in_dim, n_k * r * hd)) + raise ValueError( + "%s: in_dim %d != num_k_heads*num_v_per_k*head_dim = %d，无法按头分块置换列" + % (e.infinilm, in_dim, n_k * r * hd) + ) # [out, in] 解释为 [out, r, n_k, hd]（tiled 序）-> 对调 1,2 轴 -> [out, n_k, r, hd]（grouped 序）-> flatten - tens = tens.view(out_dim, r, n_k, hd).transpose(1, 2).contiguous().view(out_dim, in_dim) + tens = ( + tens.view(out_dim, r, n_k, hd) + .transpose(1, 2) + .contiguous() + .view(out_dim, in_dim) + ) # dense-ref 版删掉了 quantization_config（见主写盘处），框架按普通 HF 模型加载； # python 侧 `_remap_qwen3_5`（modeling_utils L808）对**非 gguf** 模型会把 norm 权重 +1 # （HF 存 delta、C++ 用完整权重的约定）。而 dense-ref 的 norm 值是从 GGUF 原样搬来的 @@ -205,8 +229,17 @@ def _as_dense(e): v = _DENSE_CACHE.get(key) if v is None: tr = tuple(x for x in e.transforms if x != M.T_NONE) + (M.T_DENSE,) - v = M.Entry(e.infinilm, e.gguf, e.shape, False, tr, e.types, e.slices, e.vperm, - "dense-ref " + (e.note or "")) + v = M.Entry( + e.infinilm, + e.gguf, + e.shape, + False, + tr, + e.types, + e.slices, + e.vperm, + "dense-ref " + (e.note or ""), + ) _DENSE_CACHE[key] = v return v @@ -215,6 +248,7 @@ def _as_dense(e): # 维度：从 GGUF 元数据推导，并与映射表的 REAL 对账 # --------------------------------------------------------------------------- + def _dec(x) -> float: """float32 元数据归回十进制字面量（1e-6 而非 9.999999974752427e-07）， 让 config.json 与 HF 原始 config 逐字符一致。7 位有效数字对 float32 无损。""" @@ -223,7 +257,7 @@ def _dec(x) -> float: def dims_from_gguf(reader) -> M.Dims: """元数据键名沿用 llama.cpp 标准写法，与审计脚本 E 节实测同一批键。""" - g = lambda suffix, idx=0: X.gguf_meta(reader, suffix)[idx] # noqa: E731 + g = lambda suffix, idx=0: X.gguf_meta(reader, suffix)[idx] # noqa: E731 n_layers = int(g("block_count")) - int(g("nextn_predict_layers")) inner = int(g("ssm.inner_size")) state = int(g("ssm.state_size")) @@ -245,7 +279,7 @@ def dims_from_gguf(reader) -> M.Dims: vocab=vocab, n_layers=n_layers, interval=int(g("full_attention_interval")), - mrope_section=tuple(sec[:3]), # 丢掉尾 0：§2.11 第 4 条 + mrope_section=tuple(sec[:3]), # 丢掉尾 0：§2.11 第 4 条 rope_theta=_dec(g("rope.freq_base")), partial_rotary_factor=_dec(dim_cnt / head_dim), rms_norm_eps=_dec(g("attention.layer_norm_rms_epsilon")), @@ -268,13 +302,18 @@ def check_dims(d: M.Dims) -> None: elif got != want: diff.append("%s: %r != %r" % (f.name, got, want)) if diff: - raise SystemExit("GGUF 元数据推导出的维度与 gguf_mapping.REAL 不符：%s\n" - "=> 先按新模型实测重做阶段 0，不要改打包器来迁就。" % diff) - log(" rms_norm_eps：GGUF float32 %r -> config 写 HF 十进制 %r" - % (float(d.rms_norm_eps), M.REAL.rms_norm_eps)) + raise SystemExit( + "GGUF 元数据推导出的维度与 gguf_mapping.REAL 不符：%s\n" + "=> 先按新模型实测重做阶段 0，不要改打包器来迁就。" % diff + ) + log( + " rms_norm_eps：GGUF float32 %r -> config 写 HF 十进制 %r" + % (float(d.rms_norm_eps), M.REAL.rms_norm_eps) + ) -from dataclasses import fields as _dc_fields # noqa: E402 +from dataclasses import fields as _dc_fields # noqa: E402 + _DIM_FIELDS = [f for f in _dc_fields(M.Dims) if f.name != "architectures"] @@ -282,6 +321,7 @@ def check_dims(d: M.Dims) -> None: # 分片写出 # --------------------------------------------------------------------------- + class ShardWriter: def __init__(self, out_dir: str, max_bytes: int): self.dir, self.max = out_dir, max_bytes @@ -306,10 +346,14 @@ def flush(self) -> None: idx = len(self.shards) fname = "model-%05d.safetensors" % idx from safetensors.torch import save_file + save_file(self.buf, os.path.join(self.dir, fname), metadata={"format": "pt"}) for k in self.buf: self.weight_map[k] = fname - log(" 写出 %s（%.2f GiB，%d 个张量）" % (fname, self.buf_bytes / _GiB, len(self.buf))) + log( + " 写出 %s（%.2f GiB，%d 个张量）" + % (fname, self.buf_bytes / _GiB, len(self.buf)) + ) self.shards[-1] = fname self.buf, self.buf_bytes = {}, 0 @@ -324,8 +368,12 @@ def finish(self) -> None: renamed[f] = new self.weight_map = {k: renamed[v] for k, v in self.weight_map.items()} with open(os.path.join(self.dir, "model.safetensors.index.json"), "w") as fp: - json.dump({"metadata": {"total_size": self.total}, - "weight_map": self.weight_map}, fp, indent=1, sort_keys=True) + json.dump( + {"metadata": {"total_size": self.total}, "weight_map": self.weight_map}, + fp, + indent=1, + sort_keys=True, + ) log(" 分片 %d 个，合计 %.3f GiB" % (n, self.total / _GiB)) @@ -333,6 +381,7 @@ def finish(self) -> None: # 自检 # --------------------------------------------------------------------------- + def rows_hash(a) -> str: """把 [rows, cols] 字节阵的**行多重集**压成一个摘要（排序后逐行喂 hash）。 @@ -340,6 +389,7 @@ def rows_hash(a) -> str: 但可以无条件断言“产物行集 == 源行集”（置换只是整行搬，不允许改字节）。 """ import hashlib + a = np.ascontiguousarray(a) v = a.view(np.void(a.shape[1] * a.dtype.itemsize)).ravel() h = hashlib.sha256() @@ -355,9 +405,14 @@ def dense_bits_check(e, t, dims, opt, prod_t) -> bool: V 头置换 / A_log 是跨行或逐元素语义，不能切块，但这类条目都很小，走全量路径。 """ import torch + if M.needs_vperm(e): - return bool(np.array_equal(prod_t.view(torch.uint16).numpy(), - X.bf16_bits(entry_float32(e, t, dims, opt)))) + return bool( + np.array_equal( + prod_t.view(torch.uint16).numpy(), + X.bf16_bits(entry_float32(e, t, dims, opt)), + ) + ) src = np.asarray(t.data) if e.slices: s, ep = e.slices[0] @@ -370,26 +425,33 @@ def dense_bits_check(e, t, dims, opt, prod_t) -> bool: if src.shape[0] != n: return False for i in range(0, n, rows): - blk = src[i:i + rows] - arr = (np.asarray(blk, dtype=np.float32) if tn in UNQUANTIZED - else dense_float32(blk, tn, opt.chunk_rows)) + blk = src[i : i + rows] + arr = ( + np.asarray(blk, dtype=np.float32) + if tn in UNQUANTIZED + else dense_float32(blk, tn, opt.chunk_rows) + ) exp = X.bf16_bits(arr.reshape((blk.shape[0],) + tail)) - got = prod_t[i:i + rows].view(torch.uint16).numpy() + got = prod_t[i : i + rows].view(torch.uint16).numpy() if not np.array_equal(got, exp): return False return True -_BIG_ELEMS = 64 * 1024 * 1024 # 切块阈值：一次最多算 64M 元素（float32 峰值 256 MB） +_BIG_ELEMS = 64 * 1024 * 1024 # 切块阈值：一次最多算 64M 元素（float32 峰值 256 MB） def verify(out_dir: str, plan, tensors, dims, opt, sample) -> int: """重读产物：全量比键/shape/dtype，分类抽样比字节。返回 FAIL 数。""" import torch from safetensors import safe_open + log("\n== 自检：重读产物 ==") - bs_files = sorted(f for f in os.listdir(out_dir) - if f.endswith(".safetensors") and not f.startswith(".")) + bs_files = sorted( + f + for f in os.listdir(out_dir) + if f.endswith(".safetensors") and not f.startswith(".") + ) with open(os.path.join(out_dir, "model.safetensors.index.json")) as fp: index = json.load(fp) got: dict[str, tuple] = {} @@ -421,8 +483,10 @@ def verify(out_dir: str, plan, tensors, dims, opt, sample) -> int: bad = [k for k in set(want) & set(got) if want[k][:2] != got[k]] if bad: fails += 1 - log(" FAIL shape/dtype 不符 %d 个：%s" % (len(bad), - [(k, want[k][:2], got[k]) for k in sorted(bad)[:4]])) + log( + " FAIL shape/dtype 不符 %d 个：%s" + % (len(bad), [(k, want[k][:2], got[k]) for k in sorted(bad)[:4]]) + ) else: log(" PASS 全部 %d 个键的 shape+dtype 与映射表一致" % len(want)) @@ -435,14 +499,18 @@ def verify(out_dir: str, plan, tensors, dims, opt, sample) -> int: table = qcfg.get("ggml_types") or {} if qcfg.get("quant_method") != "gguf": fails += 1 - log(" FAIL config.json 顶层 quantization_config.quant_method != 'gguf'（或在 text_config 里）") + log( + " FAIL config.json 顶层 quantization_config.quant_method != 'gguf'（或在 text_config 里）" + ) elif qcfg.get("key_prefix") != M.PREFIX: fails += 1 log(" FAIL config.json 缺 key_prefix=%r（阶段 2 C++ 用它裁表 key）" % M.PREFIX) else: log(" PASS quantization_config 在顶层，key_prefix=%r" % M.PREFIX) - for label, keys in (("类型表缺键", sorted(set(got) - set(table))), - ("类型表多键", sorted(set(table) - set(got)))): + for label, keys in ( + ("类型表缺键", sorted(set(got) - set(table))), + ("类型表多键", sorted(set(table) - set(got))), + ): if keys: fails += 1 log(" FAIL %s %d 个：%s" % (label, len(keys), keys[:6])) @@ -453,17 +521,33 @@ def verify(out_dir: str, plan, tensors, dims, opt, sample) -> int: # 纯随机抽 3 个会全部落在“未置换 memcpy”上，那样根本测不到置换与切片。 def sel(pred): return sorted(k for k, (_, _, e) in want.items() if pred(e)) - cats = [("blob 未置换", sel(lambda e: e.blob and not e.slices and not M.needs_vperm(e))), - ("blob V 置换", sel(lambda e: e.blob and not e.slices and M.needs_vperm(e))), - ("blob 融合切片", sel(lambda e: e.blob and e.slices)), - ("bf16 反量化", sel(lambda e: not e.blob and not e.slices and not M.needs_vperm(e))), - ("bf16 置换+alog", sel(lambda e: not e.blob and M.needs_vperm(e))), - ("bf16 融合切片", sel(lambda e: not e.blob and e.slices))] + + cats = [ + ( + "blob 未置换", + sel(lambda e: e.blob and not e.slices and not M.needs_vperm(e)), + ), + ("blob V 置换", sel(lambda e: e.blob and not e.slices and M.needs_vperm(e))), + ("blob 融合切片", sel(lambda e: e.blob and e.slices)), + ( + "bf16 反量化", + sel(lambda e: not e.blob and not e.slices and not M.needs_vperm(e)), + ), + ("bf16 置换+alog", sel(lambda e: not e.blob and M.needs_vperm(e))), + ("bf16 融合切片", sel(lambda e: not e.blob and e.slices)), + ] picks = [c[1][0] for c in cats if c[1]] if sample == "all": picks = sorted(k for k, (_, _, e) in want.items() if e.blob) - log(" 抽样 %d 个：%s" % (len(picks), "全部 blob" if sample == "all" else - " ".join("%s=%s" % (c, len(v)) for c, v in cats))) + log( + " 抽样 %d 个：%s" + % ( + len(picks), + "全部 blob" + if sample == "all" + else " ".join("%s=%s" % (c, len(v)) for c, v in cats), + ) + ) for k in picks: shape, dt, e = want[k] prod_t = handles[index["weight_map"][k]].get_tensor(k) @@ -484,14 +568,23 @@ def sel(pred): checks.append(("与 GGUF 源逐字节", np.array_equal(p, src))) else: # BF16：拿 numpy 的 RNE 位模式比，相当于独立验一次 torch 的 cast + 读写往返 - checks.append(("BF16 位与 numpy RNE 一致", - dense_bits_check(e, tensors[e.gguf], dims, opt, prod_t))) + checks.append( + ( + "BF16 位与 numpy RNE 一致", + dense_bits_check(e, tensors[e.gguf], dims, opt, prod_t), + ) + ) ok = all(v for _, v in checks) fails += 0 if ok else 1 - log(" %s %-52s %-16s %s" % ("PASS" if ok else "FAIL", k, - str(tuple(int(x) for x in prod_t.shape)), - "，".join("%s=%s" % (n, "Y" if v else "N") - for n, v in checks))) + log( + " %s %-52s %-16s %s" + % ( + "PASS" if ok else "FAIL", + k, + str(tuple(int(x) for x in prod_t.shape)), + "，".join("%s=%s" % (n, "Y" if v else "N") for n, v in checks), + ) + ) return fails @@ -499,36 +592,67 @@ def sel(pred): # main # --------------------------------------------------------------------------- + def main() -> int: - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) ap.add_argument("--gguf", default=DEFAULT_GGUF) ap.add_argument("--out", default=DEFAULT_OUT) ap.add_argument("--tokenizer-dir", default=DEFAULT_TOKENIZER) - ap.add_argument("--dense-iq", action=argparse.BooleanOptionalAction, default=True, - help="v1 把 5 个 IQ4_NL/IQ4_XS 稠密化（阶段 6 上了码本 kernel 后 --no-dense-iq）") - ap.add_argument("--dense-embed", action=argparse.BooleanOptionalAction, default=True, - help="v1 恒为 True；--no-dense-embed 需要阶段 6 的 embedding kernel") - ap.add_argument("--vperm", choices=("inv", "fwd", "none"), default="inv", - help="V 头 tiled->grouped 方向；阶段 4 A/B 用（§2.7）") - ap.add_argument("--emit-dense-ref", metavar="PATH", default=None, - help="额外产出一份全反量化 BF16 版（阶段 4 自洽基准，不部署）") + ap.add_argument( + "--dense-iq", + action=argparse.BooleanOptionalAction, + default=True, + help="v1 把 5 个 IQ4_NL/IQ4_XS 稠密化（阶段 6 上了码本 kernel 后 --no-dense-iq）", + ) + ap.add_argument( + "--dense-embed", + action=argparse.BooleanOptionalAction, + default=True, + help="v1 恒为 True；--no-dense-embed 需要阶段 6 的 embedding kernel", + ) + ap.add_argument( + "--vperm", + choices=("inv", "fwd", "none"), + default="inv", + help="V 头 tiled->grouped 方向；阶段 4 A/B 用（§2.7）", + ) + ap.add_argument( + "--emit-dense-ref", + metavar="PATH", + default=None, + help="额外产出一份全反量化 BF16 版（阶段 4 自洽基准，不部署）", + ) ap.add_argument("--max-shard-gib", type=float, default=4.0) - ap.add_argument("--chunk-rows", type=int, default=8192, - help="反量化分块行数，限制峰值内存") - ap.add_argument("--layers", type=int, default=None, - help="只打前 N 层，并同步把 config 的 num_hidden_layers 改成 N" - "（产物可直接被框架构造 + 加载，阶段 2/3 用小模型验收用）") + ap.add_argument( + "--chunk-rows", type=int, default=8192, help="反量化分块行数，限制峰值内存" + ) + ap.add_argument( + "--layers", + type=int, + default=None, + help="只打前 N 层，并同步把 config 的 num_hidden_layers 改成 N" + "（产物可直接被框架构造 + 加载，阶段 2/3 用小模型验收用）", + ) ap.add_argument("--verify", choices=("off", "sample", "all"), default="sample") - ap.add_argument("--skip-pack", action="store_true", - help="不重写 23 GiB 权重，只做分词器导出 + 自检（迭代自检逻辑用）") - ap.add_argument("--dry-run", action="store_true", - help="全量校验取向/shape/字节数，不写盘（稠密化条目也只算 shape）") + ap.add_argument( + "--skip-pack", + action="store_true", + help="不重写 23 GiB 权重，只做分词器导出 + 自检（迭代自检逻辑用）", + ) + ap.add_argument( + "--dry-run", + action="store_true", + help="全量校验取向/shape/字节数，不写盘（稠密化条目也只算 shape）", + ) a = ap.parse_args() if not a.dense_embed: - raise SystemExit("--no-dense-embed 需要阶段 6 的 embedding / lm_head 原生 kernel，" - "v1 没有它们就只能稠密化（§2.4）") + raise SystemExit( + "--no-dense-embed 需要阶段 6 的 embedding / lm_head 原生 kernel，" + "v1 没有它们就只能稠密化（§2.4）" + ) t0 = time.time() log("读取 GGUF 元数据：%s" % a.gguf) @@ -536,18 +660,23 @@ def main() -> int: tensors = {t.name: t for t in reader.tensors} dims = dims_from_gguf(reader) check_dims(dims) - log(" 维度与映射表 REAL 一致：%d 层，hidden=%d，vocab=%d" - % (dims.n_layers, dims.hidden, dims.vocab)) + log( + " 维度与映射表 REAL 一致：%d 层，hidden=%d，vocab=%d" + % (dims.n_layers, dims.hidden, dims.vocab) + ) # --layers 必须在 check_dims **之后**覆盖：维度照旧逐项校 REAL（防止换模型后硬套本表）， # 但 config 的 num_hidden_layers / layer_types 要跟着改，否则截断产物与 config # 不自洽，框架构造 64 层却只拿到 N 层权重（旧版本里这条表现为“不可加载”）。 if a.layers is not None: if not 0 < a.layers < dims.n_layers: - raise SystemExit("--layers 必须在 (0, %d) 之间，实际 %d" - % (dims.n_layers, a.layers)) - log(" --layers %d：num_hidden_layers %d -> %d，产物可加载" - % (a.layers, dims.n_layers, a.layers)) + raise SystemExit( + "--layers 必须在 (0, %d) 之间，实际 %d" % (dims.n_layers, a.layers) + ) + log( + " --layers %d：num_hidden_layers %d -> %d，产物可加载" + % (a.layers, dims.n_layers, a.layers) + ) dims.n_layers = a.layers opt = type("Opt", (), {})() @@ -557,8 +686,10 @@ def main() -> int: n_exc = M.apply_v1_exceptions(plan, opt.types, enabled=a.dense_iq) log(" 映射条目 %d，v1 稠密化例外命中 %d 个 IQ4" % (len(plan), n_exc)) blob = [e for e in plan if e.blob] - log(" blob %d 个 / 稠密化 %d 个 / 丢弃 MTP 前缀 %s" - % (len(blob), len(plan) - len(blob), M.DROP_PREFIXES)) + log( + " blob %d 个 / 稠密化 %d 个 / 丢弃 MTP 前缀 %s" + % (len(blob), len(plan) - len(blob), M.DROP_PREFIXES) + ) if a.dry_run: log("\n== dry-run：逐条目校验取向与字节数（不写盘、不反量化）==") @@ -570,30 +701,42 @@ def main() -> int: bs, ts = blk_sizes(opt.types[e.gguf]) rb = M.row_bytes(int(e.shape[1]), bs, ts) if int(t.data.shape[-1]) != rb: - raise ValueError("%s: 源行字节 %d != 期望 %d" - % (e.gguf, int(t.data.shape[-1]), rb)) + raise ValueError( + "%s: 源行字节 %d != 期望 %d" + % (e.gguf, int(t.data.shape[-1]), rb) + ) if int(t.data.shape[0]) < n_out: - raise ValueError("%s: 源 %d 行 < 条目需 %d 行" - % (e.gguf, t.data.shape[0], n_out)) + raise ValueError( + "%s: 源 %d 行 < 条目需 %d 行" % (e.gguf, t.data.shape[0], n_out) + ) blob_bytes += n_out * rb else: n = prod(tuple(int(x) for x in e.shape)) if not e.slices and prod(int(x) for x in t.shape) != n: - raise ValueError("%s: 源元素数 %s != 条目 shape %s" - % (e.gguf, t.shape, tuple(e.shape))) + raise ValueError( + "%s: 源元素数 %s != 条目 shape %s" + % (e.gguf, t.shape, tuple(e.shape)) + ) dense_bytes += n * 2 - log(" PASS %d 个条目取向/字节数自洽：blob %.3f GiB + 稠密化 BF16 %.3f GiB" + log( + " PASS %d 个条目取向/字节数自洽：blob %.3f GiB + 稠密化 BF16 %.3f GiB" " = 产物应占 %.3f GiB" - % (len(plan), blob_bytes / _GiB, dense_bytes / _GiB, - (blob_bytes + dense_bytes) / _GiB)) + % ( + len(plan), + blob_bytes / _GiB, + dense_bytes / _GiB, + (blob_bytes + dense_bytes) / _GiB, + ) + ) return 0 os.makedirs(a.out, exist_ok=True) w = ShardWriter(a.out, int(a.max_shard_gib * _GiB)) ggml_types = {} for e in plan: - ggml_types[M.type_table_key(M.ckpt_name(e))] = \ + ggml_types[M.type_table_key(M.ckpt_name(e))] = ( TYPE_ID[opt.types[e.gguf]] if e.blob else "dense_bf16" + ) if a.skip_pack: with open(os.path.join(a.out, "model.safetensors.index.json")) as fp: w.total = json.load(fp)["metadata"]["total_size"] @@ -619,28 +762,48 @@ def main() -> int: cfg = M.make_root_config(dims, ggml_types, rules) with open(os.path.join(a.out, "config.json"), "w") as fp: json.dump(cfg, fp, indent=1, sort_keys=True) - log(" config.json：%d 个 ggml_types 键（quantization_config 在顶层）+ 激活 V 头置换规则 %d 条：%s" - % (len(ggml_types), len(rules), - " ".join("%s=%dx%dx%d" % (r["suffix"], r["num_k_heads"], r["num_v_per_k"], - r["head_dim"]) for r in rules) or "无")) + log( + " config.json：%d 个 ggml_types 键（quantization_config 在顶层）+ 激活 V 头置换规则 %d 条：%s" + % ( + len(ggml_types), + len(rules), + " ".join( + "%s=%dx%dx%d" + % (r["suffix"], r["num_k_heads"], r["num_v_per_k"], r["head_dim"]) + for r in rules + ) + or "无", + ) + ) fails = export_tokenizer(reader, a.out, a.tokenizer_dir, dims) if not a.skip_pack: with open(os.path.join(a.out, "pack_report.json"), "w") as fp: - json.dump({"gguf": os.path.abspath(a.gguf), - # 张量 data 区之和 != 文件大小（后者含元数据与对齐填充）， - # 两者都记下来，免得日后拿这个数去对 stat 产生误会 - "gguf_file_bytes": os.path.getsize(os.path.abspath(a.gguf)), - "gguf_tensor_data_bytes": sum(int(t.n_bytes) for t in reader.tensors), - "n_gguf_tensors": len(tensors), - "v1_dense_iq": bool(a.dense_iq), "vperm": a.vperm, - "n_entries": len(plan), "n_blob": len(blob), - "n_v1_exceptions": n_exc, - "blob_type_ids": sorted({TYPE_ID[opt.types[e.gguf]] for e in blob}), - "out_bytes": w.total, "shards": w.shards, - "seconds": round(time.time() - t0, 1)}, - fp, indent=1, sort_keys=True) + json.dump( + { + "gguf": os.path.abspath(a.gguf), + # 张量 data 区之和 != 文件大小（后者含元数据与对齐填充）， + # 两者都记下来，免得日后拿这个数去对 stat 产生误会 + "gguf_file_bytes": os.path.getsize(os.path.abspath(a.gguf)), + "gguf_tensor_data_bytes": sum( + int(t.n_bytes) for t in reader.tensors + ), + "n_gguf_tensors": len(tensors), + "v1_dense_iq": bool(a.dense_iq), + "vperm": a.vperm, + "n_entries": len(plan), + "n_blob": len(blob), + "n_v1_exceptions": n_exc, + "blob_type_ids": sorted({TYPE_ID[opt.types[e.gguf]] for e in blob}), + "out_bytes": w.total, + "shards": w.shards, + "seconds": round(time.time() - t0, 1), + }, + fp, + indent=1, + sort_keys=True, + ) if a.verify != "off": fails += verify(a.out, plan, tensors, dims, opt, a.verify) @@ -653,8 +816,9 @@ def main() -> int: name, tens = build(e, tensors[e.gguf], dims, opt, True) wr.add(name, tens) wr.finish() - ref_cfg = M.make_root_config(dims, {M.type_table_key(k): "dense_bf16" - for k in ggml_types}, rules) + ref_cfg = M.make_root_config( + dims, {M.type_table_key(k): "dense_bf16" for k in ggml_types}, rules + ) # 稠密基准版不写 quantization_config：框架默认 NoneQuantization，C++ 里没有人 # 执行置换。它的 ssm_out 列序在打包期已置换为 grouped（见 build() 里的 act_vperm 分支）， # 与 blob 路径（运行时 gather）语义相同，可做逐层 cos_sim 对拍（§8.3）。 @@ -663,8 +827,10 @@ def main() -> int: json.dump(ref_cfg, fp, indent=1, sort_keys=True) export_tokenizer(reader, a.emit_dense_ref, a.tokenizer_dir, dims) - log("\n===== 完成：%.1f s，产物 %.3f GiB，自检 FAIL %d 处 =====" - % (time.time() - t0, w.total / _GiB, fails)) + log( + "\n===== 完成：%.1f s，产物 %.3f GiB，自检 FAIL %d 处 =====" + % (time.time() - t0, w.total / _GiB, fails) + ) return 1 if fails else 0 @@ -682,17 +848,29 @@ def export_tokenizer(reader, out_dir: str, tokenizer_dir: str, dims) -> int: merges = [str(m) for m in X.gguf_meta(reader, "tokenizer.ggml.merges")] model = str(X.gguf_meta(reader, "tokenizer.ggml.model")[0]) if len(tokens) != dims.vocab: - raise SystemExit("GGUF 词表 %d != config vocab_size %d，词表与 embedding 不同源" - % (len(tokens), dims.vocab)) + raise SystemExit( + "GGUF 词表 %d != config vocab_size %d，词表与 embedding 不同源" + % (len(tokens), dims.vocab) + ) if model != "gpt2": - log(" 警告：tokenizer.ggml.model=%r 非 gpt2，vocab.json/merges.txt 写法需复核" % model) + log( + " 警告：tokenizer.ggml.model=%r 非 gpt2，vocab.json/merges.txt 写法需复核" + % model + ) with open(os.path.join(out_dir, "vocab.json"), "w", encoding="utf-8") as fp: json.dump({t: i for i, t in enumerate(tokens)}, fp, ensure_ascii=False) with open(os.path.join(out_dir, "merges.txt"), "w", encoding="utf-8") as fp: fp.write("#version: 0.2\n" + "\n".join(merges) + "\n") - log(" 词表来自 GGUF：vocab %d / merges %d（model=%s）" % (len(tokens), len(merges), model)) + log( + " 词表来自 GGUF：vocab %d / merges %d（model=%s）" + % (len(tokens), len(merges), model) + ) - have = os.listdir(tokenizer_dir) if tokenizer_dir and os.path.isdir(tokenizer_dir) else [] + have = ( + os.listdir(tokenizer_dir) + if tokenizer_dir and os.path.isdir(tokenizer_dir) + else [] + ) if not have: log(" 警告：分词器配置目录不存在：%s（只写了词表）" % tokenizer_dir) copied = [] @@ -703,8 +881,10 @@ def export_tokenizer(reader, out_dir: str, tokenizer_dir: str, dims) -> int: copied.append(f) log(" 附属配置复制 %d 个：%s" % (len(copied), " ".join(sorted(copied)))) if "tokenizer_config.json" not in copied + have: - raise SystemExit("产物缺 tokenizer_config.json：既没从 %s 复制到，也没导出兜底" - % tokenizer_dir) + raise SystemExit( + "产物缺 tokenizer_config.json：既没从 %s 复制到，也没导出兜底" + % tokenizer_dir + ) return check_tokenizer(out_dir, dims) @@ -721,10 +901,12 @@ def check_tokenizer(out_dir: str, dims) -> int: s = "你好，世界 hello world 27B" ids = tk.encode(s) ok = n == dims.vocab and tk.decode(ids) == s - log(" %s 分词器 %s vocab=%d 往返=%s" % ("PASS" if ok else "FAIL", cls, n, - tk.decode(ids) == s)) + log( + " %s 分词器 %s vocab=%d 往返=%s" + % ("PASS" if ok else "FAIL", cls, n, tk.decode(ids) == s) + ) return 0 if ok else 1 - except Exception as exc: # noqa: BLE001 + except Exception as exc: # noqa: BLE001 log(" FAIL 分词器加载：%s: %s" % (type(exc).__name__, str(exc)[:200])) return 1 diff --git a/scripts/gguf_transforms.py b/scripts/gguf_transforms.py index b9f1c2bd8..7cba7132e 100644 --- a/scripts/gguf_transforms.py +++ b/scripts/gguf_transforms.py @@ -19,28 +19,32 @@ import numpy as np - # --------------------------------------------------------------------------- # V 头置换 # --------------------------------------------------------------------------- + def reorder_v(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: """grouped -> tiled，与 llama.cpp `_reorder_v_heads` 同语义（沿 dim0 的整头/整元素置换）。 支持任意尾部维度：1-D（A_log/dt_bias，hd=1）、2-D（权重行）、3-D（conv1d [C,1,K]）。 """ rest = t.shape[1:] - return (t.reshape((n_k, n_v_per_k, hd) + rest) - .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) - .reshape((n_k * n_v_per_k * hd,) + rest)) + return ( + t.reshape((n_k, n_v_per_k, hd) + rest) + .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) + .reshape((n_k * n_v_per_k * hd,) + rest) + ) def reorder_v_inverse(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: """逆变换 = 两个轴参数对调后再调用一次。""" rest = t.shape[1:] - return (t.reshape((n_v_per_k, n_k, hd) + rest) - .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) - .reshape((n_k * n_v_per_k * hd,) + rest)) + return ( + t.reshape((n_v_per_k, n_k, hd) + rest) + .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) + .reshape((n_k * n_v_per_k * hd,) + rest) + ) _VPERM = {"inv": reorder_v_inverse, "fwd": reorder_v, "none": None} @@ -56,8 +60,9 @@ def vperm_head_dim(e, dims) -> int: n_heads = dims.lin_v_heads rows = int(e.shape[0]) if e.vperm == "all" else dims.value_dim if rows % n_heads: - raise ValueError("%s：作用域行数 %d 不能被 value 头数 %d 整除" - % (e.infinilm, rows, n_heads)) + raise ValueError( + "%s：作用域行数 %d 不能被 value 头数 %d 整除" % (e.infinilm, rows, n_heads) + ) return rows // n_heads @@ -69,13 +74,15 @@ def apply_vperm(arr: np.ndarray, e, dims, direction: str = "inv") -> np.ndarray: n_k, hd = dims.lin_k_heads, vperm_head_dim(e, dims) v_per_k = dims.lin_v_heads // n_k if int(dims.lin_v_heads) % n_k: - raise ValueError("lin_v_heads %d 不能被 lin_k_heads %d 整除" - % (dims.lin_v_heads, n_k)) + raise ValueError( + "lin_v_heads %d 不能被 lin_k_heads %d 整除" % (dims.lin_v_heads, n_k) + ) if e.vperm == "v_tail": n_v = n_k * v_per_k * hd if arr.shape[0] < n_v: - raise ValueError("%s：dim0=%d 小于 value 段长度 %d" - % (e.infinilm, arr.shape[0], n_v)) + raise ValueError( + "%s：dim0=%d 小于 value 段长度 %d" % (e.infinilm, arr.shape[0], n_v) + ) out = np.asarray(arr, dtype=arr.dtype) return np.concatenate([out[:-n_v], fn(out[-n_v:], n_k, v_per_k, hd)], axis=0) return fn(np.asarray(arr, dtype=arr.dtype), n_k, v_per_k, hd) @@ -85,6 +92,7 @@ def apply_vperm(arr: np.ndarray, e, dims, direction: str = "inv") -> np.ndarray: # 其它变换 # --------------------------------------------------------------------------- + def alog_from_ssm_a(a: np.ndarray) -> np.ndarray: """A_log = log(-ssm_a)。 @@ -94,8 +102,10 @@ def alog_from_ssm_a(a: np.ndarray) -> np.ndarray: """ a = np.asarray(a, dtype=np.float32) if not np.all(a < 0): - raise ValueError("ssm_a 存在非负值（min=%g），无法取 log(-x)；" - "请核对 conversion/qwen.py 的 A_log 约定" % float(a.min())) + raise ValueError( + "ssm_a 存在非负值（min=%g），无法取 log(-x)；" + "请核对 conversion/qwen.py 的 A_log 约定" % float(a.min()) + ) return np.log(-a) diff --git a/test/bench/backends/infinilm.py b/test/bench/backends/infinilm.py index fbd99379d..7c10291ca 100644 --- a/test/bench/backends/infinilm.py +++ b/test/bench/backends/infinilm.py @@ -50,6 +50,9 @@ def __init__( print(f"Graph compilation: {'enabled' if enable_graph else 'disabled'}") print(f"Attention backend: {attn_backend}") + model_type = self.config_dict.get("model_type") + enable_prefix_caching = model_type not in {"qwen3_5", "qwen3_5_moe"} + self.model = LLM( model_path=model_dir_path, device=device_name, @@ -60,6 +63,7 @@ def __init__( block_size=256, enable_graph=enable_graph, attn_backend=attn_backend, + enable_prefix_caching=enable_prefix_caching, ) self.processor = self.model.engine.processor self.tokenizer = self.processor.get_tokenizer() From 2e604db034dc743bbb0dab32a7989a1741426e6e Mon Sep 17 00:00:00 2001 From: xindongliu594 Date: Fri, 4 Sep 2026 14:03:56 +0800 Subject: [PATCH 3/5] chore: prepare GGUF Route B for upstream review --- GGUF_ROUTE_B_QWEN38.md | 753 ++++-------------- csrc/config/quant_config.cpp | 3 +- csrc/layers/linear/base_linear.cpp | 16 +- csrc/layers/linear/base_linear.hpp | 28 +- csrc/layers/linear/fused_linear.cpp | 24 +- csrc/layers/linear/fused_linear.hpp | 6 +- csrc/layers/mlp/mlp.hpp | 5 +- csrc/layers/quantization/gguf.cpp | 211 ++--- csrc/layers/quantization/gguf.hpp | 78 +- csrc/models/qwen3_5/qwen3_5_attention.cpp | 5 +- csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp | 3 +- .../qwen3_5/qwen3_5_fused_qkv_linear.cpp | 17 +- .../qwen3_5/qwen3_5_fused_qkv_linear.hpp | 4 +- .../qwen3_next/qwen3_next_gated_deltanet.cpp | 3 +- python/infinilm/modeling_utils.py | 8 +- scripts/gguf_mapping.py | 238 +++--- scripts/gguf_routeb_audit.py | 728 ----------------- scripts/gguf_routeb_blocks_probe.cpp | 81 -- scripts/gguf_routeb_blocks_probe.cu | 128 --- scripts/gguf_routeb_blocks_ref.py | 744 ----------------- scripts/gguf_routeb_compare.py | 102 --- scripts/gguf_routeb_env.sh | 21 - scripts/gguf_routeb_first_diff.py | 213 ----- scripts/gguf_routeb_first_diff_batch.py | 217 ----- scripts/gguf_routeb_gemv_check.py | 368 --------- scripts/gguf_routeb_gemv_probe.cu | 158 ---- scripts/gguf_routeb_head_precision.py | 116 --- scripts/gguf_routeb_infinilm_ref.py | 123 --- scripts/gguf_routeb_infinilm_trace.py | 300 ------- scripts/gguf_routeb_llama_probe.py | 64 -- scripts/gguf_routeb_llama_ref.py | 124 --- scripts/gguf_routeb_llama_trace.py | 86 -- scripts/gguf_routeb_probe_params.py | 71 -- scripts/gguf_routeb_prompts.jsonl | 32 - scripts/gguf_routeb_shape_contract.py | 354 -------- scripts/gguf_routeb_stage2_check.py | 360 --------- scripts/gguf_routeb_stage3_check.py | 361 --------- scripts/gguf_routeb_tokenizer_check.py | 145 ---- scripts/gguf_routeb_typecensus.py | 85 -- scripts/gguf_to_infinilm.py | 335 ++++---- scripts/gguf_transforms.py | 76 +- test/scripts/test_gguf_routeb.py | 69 ++ 42 files changed, 704 insertions(+), 6159 deletions(-) delete mode 100644 scripts/gguf_routeb_audit.py delete mode 100644 scripts/gguf_routeb_blocks_probe.cpp delete mode 100644 scripts/gguf_routeb_blocks_probe.cu delete mode 100644 scripts/gguf_routeb_blocks_ref.py delete mode 100755 scripts/gguf_routeb_compare.py delete mode 100644 scripts/gguf_routeb_env.sh delete mode 100755 scripts/gguf_routeb_first_diff.py delete mode 100644 scripts/gguf_routeb_first_diff_batch.py delete mode 100644 scripts/gguf_routeb_gemv_check.py delete mode 100644 scripts/gguf_routeb_gemv_probe.cu delete mode 100644 scripts/gguf_routeb_head_precision.py delete mode 100755 scripts/gguf_routeb_infinilm_ref.py delete mode 100644 scripts/gguf_routeb_infinilm_trace.py delete mode 100644 scripts/gguf_routeb_llama_probe.py delete mode 100755 scripts/gguf_routeb_llama_ref.py delete mode 100644 scripts/gguf_routeb_llama_trace.py delete mode 100644 scripts/gguf_routeb_probe_params.py delete mode 100644 scripts/gguf_routeb_prompts.jsonl delete mode 100644 scripts/gguf_routeb_shape_contract.py delete mode 100644 scripts/gguf_routeb_stage2_check.py delete mode 100644 scripts/gguf_routeb_stage3_check.py delete mode 100755 scripts/gguf_routeb_tokenizer_check.py delete mode 100644 scripts/gguf_routeb_typecensus.py create mode 100644 test/scripts/test_gguf_routeb.py diff --git a/GGUF_ROUTE_B_QWEN38.md b/GGUF_ROUTE_B_QWEN38.md index f48929d0c..fae4ac2bc 100644 --- a/GGUF_ROUTE_B_QWEN38.md +++ b/GGUF_ROUTE_B_QWEN38.md @@ -1,664 +1,191 @@ -# InfiniLM 适配 Qwen3.8-27B GGUF 技术报告 +# GGUF Route B for Qwen3.5 -> 路线：GGUF 原生块量化（Route B） -> 目标模型：Qwen3.8-27B-UD-Q6_K -> 目标框架：InfiniLM + InfiniCore -> 文档日期：2026-09-03 -> 状态：核心适配已完成并可运行；严格逐 token 一致性优化仍有可选提升空间 +## Overview -## 1. 摘要 +This integration runs selected GGUF block-quantized weights directly from an +InfiniLM checkpoint. The converter copies supported GGUF block bytes into +safetensors as `uint8` tensors and records their GGML type in +`quantization_config.ggml_types`. InfiniLM resolves each weight by checkpoint +name and dispatches it to InfiniCore's `linear_gguf` operator. -本工作完成了 Qwen3.8-27B GGUF 模型到 InfiniLM 的原生块量化适配。这里的“原生”是指： +The design keeps model-specific mapping in Python while making the C++ packed +Linear path reusable by other model integrations. -- GGUF 中的 Q8_0、Q4_K、Q5_K、Q6_K 权重块不先完整反量化为 BF16； -- 打包时直接保留 GGUF block bytes，并以 `torch.uint8` 张量写入 safetensors； -- 推理时由新增的 `linear_gguf` 算子在 GPU kernel 内按块解码并参与矩阵乘； -- 小 batch/decode 使用寄存器 GEMV，大 batch/prefill 使用分块解码加 GEMM； -- 暂不支持原生执行的少量权重在打包阶段显式转为 BF16，不允许静默回退。 +## Supported scope -最终产物能够完成全量 27B 模型加载、prefill、逐 token decode 和确定性生成。打包模型包含 -947 个张量、6 个 safetensors 分片，总权重体积 23.264 GiB；其中 491 个张量保持 GGUF -block bytes，456 个张量为 BF16。 +- Model profile: Qwen3.5 / Qwen3.8 27B Route B mapping. +- Native packed Linear types: `Q8_0`, `Q4_K`, `Q5_K`, and `Q6_K`. +- Dense BF16 fallback for parameters without a native packed execution path, + including embeddings, output head, normalization/scalar tensors, and IQ4 + tensors. +- NVIDIA execution through InfiniCore `linear_gguf`. +- Decode/small-batch and prefill execution paths selected inside InfiniCore. -功能层面，GGUF 适配已经跑通。以 llama.cpp 为参照进行 32 个样例、每例 32 个 token 的严格 -比较，当前接受的严格基线达到 **27/32 个样例完全一致、920/1024 个 token 一致**。这个指标 -衡量的是两个不同推理后端的逐 token 数值复现程度，不等同于模型能否正确运行。原计划中的 -`>=29/32` 属于额外的严格一致性优化目标，目前尚未达到，也不是 GGUF 适配可用性的必要条件。 +The current implementation intentionally rejects tensor parallelism for packed +GGUF weights. It also does not provide native IQ4, packed embedding, or packed +output-head kernels. -## 2. 背景、目标与非目标 +## Dependency -### 2.1 输入和输出 +The InfiniLM changes require the corresponding InfiniCore `linear_gguf` +operator and its supported GGML block decoders: -源模型： +- InfiniCore pull request: https://github.com/InfiniTensor/InfiniCore/pull/1545 -```text -/home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf -``` - -当前正式打包产物： - -```text -/home/liuxd/models/Qwen3.8-27B-GGUF-native-v2 -``` - -源 GGUF 文件约 21.97 GB。打包后的 v1/native-v2 产物采用 InfiniLM 可装载的 safetensors -目录结构，同时在 `config.json` 中保存每个权重的 GGML 类型和 GGUF 量化配置。 - -### 2.2 主要目标 - -1. 在 InfiniLM 中加载并运行 Qwen3.8-27B GGUF 模型。 -2. 尽可能保留 GGUF 原生量化块，避免将全部权重展开为 BF16。 -3. 支持 Q8_0、Q4_K、Q5_K、Q6_K 四种主要 GGUF block 类型。 -4. 同时覆盖 prompt prefill 和 autoregressive decode。 -5. 建立可复用的 GGUF 打包、类型路由、算子和验证框架，以便后续适配其他模型。 -6. 用独立门禁证明没有错映射、错 shape、错字节布局或静默稠密回退。 - -### 2.3 当前非目标 - -- 不要求 InfiniLM 与 llama.cpp 在所有输入上逐 bit 或逐 token 完全相同； -- 不在本阶段实现所有 IQ 系列 GGUF 量化格式； -- 不在本阶段实现 GGUF blob 的 tensor parallel 切分； -- 不把实验性 Q8 激活量化或局部 F32 路径默认启用； -- 不以 llama.cpp 的速度数据替代 InfiniLM 自身性能测试。 - -## 3. 模型特点与适配难点 - -Qwen3.8-27B 不是只包含标准全注意力层的简单 Transformer。模型共有 64 个 decoder layer， -其中包含全注意力层和 Gated DeltaNet/线性注意力层，并维护额外的 GDN/SSM state。适配难点主要 -来自以下方面： - -1. GGUF 张量命名与 InfiniLM 参数命名不一致；部分 GGUF 融合权重需要拆成多个运行时权重。 -2. 不同 GGUF 类型具有不同 block size 和字节布局，U8 张量的第二维不是逻辑输入维度。 -3. GGUF 某些二维权重的物理取向与运行时线性层的逻辑取向不同。 -4. 线性注意力 `out_proj` 的 V head 布局需要额外的 grouped-to-tiled 置换。 -5. GGUF 与原始模型在 RMSNorm gain 表达约定上存在差异，错误处理会造成系统性数值偏差。 -6. prefill 的 M 较大，不能只实现单 token GEMV；同时又不能把完整权重永久展开为 BF16。 -7. BF16 舍入、归约顺序、采样语义会使接近的 logits 在两个后端产生不同 top-1 token。 - -## 4. 总体技术路线 - -完整数据流如下： - -```text -Qwen3.8-27B GGUF - | - v -gguf_mapping.py 生成唯一映射计划 - | - v -gguf_to_infinilm.py - |-- 支持类型：原始 block bytes -> U8 weight_bytes - |-- 例外类型：显式解码 -> BF16 weight - |-- 写入 ggml_types / quantization_config - | - v -InfiniLM safetensors 目录（6 shards） - | - v -GGUFBlockQuantization 按完整权重键解析类型和 shard - | - v -BaseLinear / Qwen3.5 模型层调用 linear_gguf - | - +-- decode：寄存器内 block decode + GEMV - | - +-- prefill：tile decode 到 workspace + GEMM - v -BF16/F32 hidden -> 后续 attention、GDN、MLP、norm、lm_head -``` - -这条路线的关键原则是：量化格式信息从打包到运行时始终显式存在；若类型、shape 或映射不满足 -约束，程序直接报错，而不是悄悄改走稠密权重。 - -## 5. GGUF 打包与权重映射 - -### 5.1 单一映射事实源 - -`/home/liuxd/InfiniLM/scripts/gguf_mapping.py` 是张量映射的单一事实源。每个映射条目描述： - -- InfiniLM 参数名； -- GGUF tensor 名； -- 逻辑 shape； -- 是否保存为 blob； -- 支持的 GGML 类型； -- 融合张量的 slice 范围； -- 是否执行转置或 V head 置换； -- checkpoint 键和类型表键。 - -打包器、shape contract、内存预算和 C++ 运行时检查都基于同一映射计划，避免 Python 打包 -规则和 C++ 加载规则分别维护后逐渐漂移。 - -### 5.2 支持的原生量化类型 - -| GGML 类型 | 类型 ID | 典型 block | 当前处理方式 | -|---|---:|---|---| -| Q8_0 | 8 | 32 个权重 / 34 B | 原生 U8 blob + GPU 解码 | -| Q4_K | 12 | K-quant block | 原生 U8 blob + GPU 解码 | -| Q5_K | 13 | K-quant block | 原生 U8 blob + GPU 解码 | -| Q6_K | 14 | 256 个权重 / 210 B | 原生 U8 blob + GPU 解码 | - -例如： - -- Q6_K，`K=5120` 时每行 `5120 / 256 * 210 = 4200 B`； -- Q6_K，`K=6144` 时每行 5040 B； -- Q6_K，`K=10240` 时每行 8400 B； -- Q6_K，`K=17408` 时每行 14280 B； -- Q8_0，`K=5120` 时每行 `5120 / 32 * 34 = 5440 B`。 - -因此 blob 的物理 shape 是 `[N, row_bytes]`，而不是普通线性权重的 `[N, K]`。运行时从 -descriptor 中同时获得逻辑 K、GGML type 和 row bytes，并检查三者是否一致。 - -### 5.3 BF16 例外路径 - -当前模型中不属于四种原生类型的少量 IQ4_XS/IQ4_NL 权重在打包阶段显式解码为 BF16。 -embedding 和 lm_head 在当前 v1 也采用 BF16 例外路径，以降低首次集成的复杂度。例外是映射 -计划的一部分，不是运行时静默 fallback。 - -后续若补充 embedding gather-dequant 和量化 lm_head，可预计再节省约 2.51 GiB 权重显存。 - -### 5.4 融合权重、取向与置换 - -模型包含 GGUF 融合张量到多个 InfiniLM 参数的拆分。947 个产物条目多于 851 个实际消费 -GGUF 权重，主要来自 48 个 GDN 层的融合 `attn_qkv` 一分为三。打包器按映射表定义的 slice -拆分，不能仅依靠名称替换。 - -对于线性注意力输出投影，还需要对 V head 执行 grouped-to-tiled 置换。当前实现按 -`16 x 3 x 128` 的语义布局转换，使 GGUF 权重布局与 InfiniLM GDN 计算布局一致。 - -### 5.5 RMSNorm 约定修正 - -排查中发现 GGUF/RMSNorm gain 与原模型权重的表达约定不同。若把已经 baked `+1` 的 norm -权重再次加 1，会产生明显的逐层漂移。打包器新增 `_is_baked_plus1_norm()`，对所有 norm -权重统一判断，并排除不应套用该规则的张量。 - -### 5.6 打包器 - -主要脚本： - -```text -/home/liuxd/InfiniLM/scripts/gguf_to_infinilm.py -/home/liuxd/InfiniLM/scripts/gguf_mapping.py -/home/liuxd/InfiniLM/scripts/gguf_transforms.py -``` - -打包器完成以下工作： - -1. 读取 GGUF metadata 和 tensor directory； -2. 根据模型维度生成完整映射计划； -3. 对原生支持类型逐行复制 block bytes； -4. 对明确列出的例外解码为 BF16； -5. 执行 slice、转置、V permutation 和 norm convention 修正； -6. 写入带 index 的 safetensors shards； -7. 在 `config.json` 写入 `quantization_config` 和 947 项 `ggml_types`； -8. 复制 tokenizer/config 所需文件； -9. 对 shape、dtype、字节数和抽样原始 bytes 做自检。 - -脚本支持 `--dry-run` 和 `--skip-pack`，可以在不重复生成 23 GiB 产物的情况下审计映射或复用 -已有分片。 - -### 5.7 最终产物组成 - -| 项目 | 数量/大小 | -|---|---:| -| safetensors 分片 | 6 | -| 总张量数 | 947 | -| 原生 U8 blob | 491 | -| BF16 张量 | 456 | -| U8 blob 体积 | 17.648 GiB | -| BF16 体积 | 5.615 GiB | -| 合计 | 23.264 GiB | - -`ggml_types` 的类型直方图为：`dense_bf16=456`、`Q6_K=304`、`Q5_K=124`、 -`Q8_0=59`、`Q4_K=4`。 - -## 6. InfiniLM 模型和量化框架接线 - -### 6.1 GGUFBlockQuantization - -新增： - -```text -/home/liuxd/InfiniLM/csrc/layers/quantization/gguf.hpp -/home/liuxd/InfiniLM/csrc/layers/quantization/gguf.cpp -``` - -`GGUFBlockQuantization` 的职责包括： - -- 从模型配置读取 `ggml_types`； -- 按完整 checkpoint tensor key 查找具体 GGML 类型； -- 区分 `.weight_bytes` blob 和 BF16 `.weight`； -- 处理 fused shard 对应关系； -- 对需要的 shard 应用 V permutation 语义； -- 创建并调用 InfiniCore `linear_gguf_`； -- 对未知类型、缺失类型、shape 不符和不支持的 tensor parallel 显式报错。 - -### 6.2 Linear 层传递完整权重身份 - -普通量化框架只知道当前 Linear 的逻辑维度，但 GGUF 路由还必须知道它对应哪个 checkpoint -tensor。为此扩展了 `BaseLinear` 及相关线性层，使其保存 checkpoint stem 或 `shard_stems_`。 -量化对象据此解析每个 fused shard 的类型，而不是按当前 C++ 对象名进行模糊匹配。 - -### 6.3 Qwen3.5/Qwen3.8 模型结构 - -完成了 Qwen3.5 风格模型在配置、注册、权重映射和运行时模块上的接入，包括： - -- decoder layer； -- full attention； -- Gated DeltaNet/linear attention； -- MLP； -- GDN/SSM cache state； -- final norm 和 causal LM 输出； -- tokenizer/chat template 相关配置。 - -模型结构适配与 GGUF block 算子相互独立：前者决定“哪些张量放到哪里”，后者决定“某个 -量化 Linear 怎样执行”。这种拆分是后续复用到其他架构的基础。 - -### 6.4 `ignore_eos` 语义修正 - -严格比较时发现，llama.cpp 的 `ignore_eos=true` 是在采样前屏蔽 EOS，而 InfiniLM 原有的 -`stop_on_eos=false` 仅表示采到 EOS 后不停止，并不会阻止 EOS 被选中。这是采样语义差异， -不是算子误差。 - -为此扩展： - -- C++ RankWorker Input 的 `suppressed_token_ids`； -- pybind 和 InferEngine 的字段传递； -- 低层 `GenerationConfig.ignore_eos`； -- 高层 SamplingParams 到每请求屏蔽列表的转换。 - -修正后 `ctx_03` 从第 27 token 分叉变为 32/32 完全一致，同时保留其他 stopping criteria。 - -## 7. 新增和扩展的算子 - -### 7.1 算子总表 - -| 算子/模块 | 类型 | 作用 | 默认状态 | -|---|---|---|---| -| `linear_gguf` | 新增 | 直接消费 GGUF U8 block 权重 | 启用 | -| GGML block decoders | 新增 | 解码 Q8_0/Q4_K/Q5_K/Q6_K | 启用 | -| register GGUF GEMV | 新增 | 小 M 的 decode/small-prefill | 启用 | -| tile dequant + GEMM | 新增 | 大 M prefill | 启用 | -| mixed add-RMSNorm | 扩展 | 承接实验性 F32 hidden 边界 | 普通路径不触发 | -| mixed GEMM fallback | 扩展 | BF16 权重乘 F32 hidden | 仅实验路径触发 | -| Q8A activation path | 实验新增 | Q8 激活量化后与 GGUF 权重计算 | 默认关闭 | -| F32 GGUF output | 实验扩展 | 指定 Linear 保留 F32 输出 | 默认关闭 | - -### 7.2 `linear_gguf` 完整注册链 - -新增文件： - -```text -/home/liuxd/InfiniCore/include/infiniop/ops/linear_gguf.h -/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/linear_gguf.h -/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/info.h -/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/operator.cc -/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/nvidia/linear_gguf_nvidia.cuh -/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/nvidia/linear_gguf_nvidia.cu -/home/liuxd/InfiniCore/include/infinicore/ops/linear_gguf.hpp -/home/liuxd/InfiniCore/src/infinicore/ops/linear_gguf/linear_gguf.cc -/home/liuxd/InfiniCore/src/infinicore/ops/linear_gguf/linear_gguf_infiniop.cc -``` - -这条链覆盖 C API descriptor、InfiniCore C++ dispatcher、workspace 计算、设备 dispatch、 -plan/run/cleanup。`info.h` 是 shape、dtype、GGML type 和 row bytes 契约的集中校验点。 - -### 7.3 GGML block 解码器 - -文件： - -```text -/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/ggml_blocks.h -``` - -这里实现 Q8_0、Q4_K、Q5_K、Q6_K 的共享 host/device block decoder。decoder 按 GGUF 的 -原始位布局读取 scale、高位掩码和量化值。因为某些 row stride 只保证 2 字节对齐，不能假设 -每个 block 都满足 4/16 字节对齐；实现使用安全的 byte load/拷贝方式，避免未对齐访问错误。 +Build and install that InfiniCore revision before building InfiniLM. -### 7.4 小 M 寄存器 GEMV +## Checkpoint format -文件： +Packed Linear weights use a `weight_bytes` suffix and shape +`[out_features, row_bytes]`, where: ```text -/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/nvidia/linear_gguf_gemv.cuh +row_bytes = in_features / block_size * type_size ``` -执行方式： - -1. 一个 warp 负责一个输出行； -2. warp lanes 沿 K 方向处理多个 GGUF block； -3. block 权重在寄存器中即时解码； -4. 与输入激活做 FP32 累加； -5. warp reduction 得到输出； -6. 正式路径把结果写为 BF16。 - -kernel 编译容量支持 `M<=16`。当前严格基线通过 -`INFINI_GGUF_STRICT_SMALL_PREFILL_MAX_M=10` 选择 `M<=10` 使用该路径，因为它在当前 -32x32 对拍中比更早切换到 prefill 路径更接近 llama.cpp 的归约结果。 - -### 7.5 大 M prefill - -文件： - -```text -/home/liuxd/InfiniCore/src/infiniop/ops/linear_gguf/nvidia/linear_gguf_dequant.cuh -``` - -当 M 超过小 M 路由阈值时，算子按 tile 把量化权重解码到临时 workspace，再调用 GEMM。 -这种方式没有把整套模型权重永久还原为 BF16，只为当前 Linear 分配必要的临时 scratch，因而 -仍符合 Route B。数值门覆盖到 `M=1024`，端到端验证覆盖 `M=12/64/512`。 - -### 7.6 mixed add-RMSNorm 扩展 +The converter writes a top-level configuration: -修改文件： - -```text -/home/liuxd/InfiniCore/src/infiniop/ops/add_rms_norm/info.h -/home/liuxd/InfiniCore/src/infiniop/ops/add_rms_norm/nvidia/add_rms_norm_nvidia.cu -``` - -为研究 BF16 物化边界，新增两类受限组合： - -- F32 `a` + BF16 residual `b` + BF16 weight，FP32 求和和 RMS，输出 BF16； -- BF16 `a` + BF16 `b` + BF16 weight，FP32 求和和 RMS，输出 F32。 - -第一类用于让单个 F32 GGUF Linear 安全跨过 residual+norm 后回到 BF16；第二类用于 final -residual+RMSNorm 全 F32 实验。正常 BF16 模型路径保持不变。 - -### 7.7 GEMM mixed-dtype 修复与 fallback - -修改文件： - -```text -/home/liuxd/InfiniCore/src/infiniop/ops/gemm/nvidia/gemm_nvidia.cu +```json +{ + "quantization_config": { + "quant_method": "gguf", + "key_prefix": "model.language_model.", + "ggml_types": { + "model.language_model.layers.0.mlp.gate_proj.weight_bytes": 14, + "model.language_model.layers.0.input_layernorm.weight": "dense_bf16" + }, + "activation_vperm": [] + } +} ``` -完成两项改动： - -1. 修复 row-major 转置执行中交换 A/B 指针却没有同步交换 `a_type/b_type` 的问题； -2. 为 `BF16 large matrix x F32 hidden -> F32` 增加 batch=1 的 tiled register-GEMV fallback。 - -第二项是因为当前 cuBLAS 对该实际 mixed 组合返回 `CUBLAS_STATUS_NOT_SUPPORTED`。fallback -按 16 个 hidden column 分 tile，可覆盖任意 prompt M，但只在实验性 F32 final path 中使用。 - -### 7.8 Q8A 激活量化实验 - -`linear_gguf` 中还加入了受环境变量控制的 Q8A/Q8_1-like 激活量化路径，用于研究 llama.cpp -的激活量化和归约方式。它支持全 GGUF 类型或只命中某个 GGML type。实测该路径能修复个别 -样例，但会使其他样例退化，因此保留代码用于研究，默认关闭。 - -## 8. 数值一致性优化 - -### 8.1 为什么两个后端不会天然完全一致 - -即使权重 block 解码公式正确，以下差异仍可能改变非常接近的 top-1： - -- GEMV/GEMM 的分块和归约顺序； -- 中间结果何时从 FP32 舍入到 BF16； -- llama.cpp 的 Q8 激活量化与 InfiniLM 的 BF16 激活； -- RMSNorm residual sum 的物化 dtype； -- lm_head 累加精度； -- EOS 屏蔽等采样语义。 - -因此严格逐 token 一致性是独立的高标准验证项，不能简单等同于“算子正确性”。 - -### 8.2 当前接受的严格配置 - -当前接受配置包含： - -- GGUF 四类型原生 block kernel； -- FP32 lm_head logits； -- 通用 `ignore_eos` 采样语义； -- small-prefill register path； -- `INFINI_GGUF_STRICT_SMALL_PREFILL=1`； -- `INFINI_GGUF_STRICT_SMALL_PREFILL_MAX_M=10`； -- Q8A、局部 F32 GGUF 输出和 final-FP32 全部关闭。 - -该配置得到： - -```text -27 / 32 cases exact -920 / 1024 tokens match -``` - -剩余首分叉为： - -```text -zh_04 @ token 28 -zh_05 @ token 19 -zh_06 @ token 4 -code_04 @ token 4 -math_04 @ token 1 -``` +`ggml_types` keys are exact safetensors parameter names. Values are GGML type +ids or `"dense_bf16"`. Runtime lookup requires exactly one packed or dense +candidate and fails on missing or ambiguous metadata. -### 8.3 已验证但未采用的实验 +## Conversion -| 实验 | 结果 | 决策 | -|---|---|---| -| 全类型 Q8A | 能修复 `zh_05`，但五个重点例合计仅 75/160 token | 默认关闭 | -| Q6_K-only Q8A | 27/32、910/1024 | 退化，关闭 | -| Q5_K-only Q8A | 27/32、907/1024 | 退化，关闭 | -| layer0 attention out_proj F32 | `math_04` margin 明显恶化 | 关闭 | -| layer0 MLP down_proj F32 | `math_04` margin 明显恶化 | 关闭 | -| 强制 cuBLAS/寄存器 GEMV切换 | 未稳定修复剩余分叉 | 不采用 | -| final residual+RMSNorm F32 | 27/32、921/1024；修复 `zh_05` 但回归 `math_02` | 默认关闭 | +The converter depends on Python packages used by the project plus +`gguf-py`. Install `gguf-py` or point `LLAMA_CPP_DIR` at a llama.cpp +checkout: -final-FP32 的全量结果比基线多匹配 1 个 token，但 exact case 仍为 27/32。它将 `zh_05` -修复为 32/32，同时使原本 exact 的 `math_02` 在 token 18 分叉。`math_02` 的参考 token 只落后 -约 `5.95e-4`，说明这是非常临界的归约/舍入翻转，但在没有消除回归前不能作为默认优化。 - -## 9. 验证方法与结果 - -### 9.1 字节布局和解码公式 - -- 映射/字节布局审计：48 PASS / 0 FAIL； -- 四类型 block decode 交叉验证：47 PASS / 0 FAIL； -- 使用真实 GGUF blocks 做大规模抽样； -- 对 half 的 65536 种 bit pattern 做穷举扫描，并覆盖四种格式相关路径； -- blob 原始字节与源 GGUF 分类抽样逐字节一致。 - -### 9.2 Linear 数值门 - -- decode GEMV：两套独立产物各 56 PASS / 0 FAIL，cosine similarity 均大于 0.999； -- prefill：316 PASS / 0 FAIL； -- 覆盖四种 GGUF 类型、多种 N/K/M 和真实模型行字节； -- 端到端 prefill 覆盖 M=12、64、512。 - -### 9.3 映射、shape 与加载 - -- 映射计划共 947 条； -- 491 个 blob 的行字节均可整除且与 GGUF `n_bytes` 一致； -- 947 个产物 tensor 与引擎消费 tensor 双向集合一致； -- 947/947 shape 一致； -- 491 个 blob 在配置和 safetensors 中均为 U8； -- 6 个分片全量加载成功； -- 首个 blob forward 日志证明进入 `linear_gguf`，不存在稠密静默回退。 - -### 9.4 mini8 端到端 - -构造覆盖全部四种量化类型的 mini8 模型，61 个 blob 的分布为： - -```text -Q6_K: 35 -Q5_K: 12 -Q8_0: 10 -Q4_K: 4 -``` - -模型成功执行 `generate()`，阶段检查 11 PASS / 0 FAIL，确认加载、路由、decode 和状态推进 -形成闭环。 - -### 9.5 全量模型 - -全量 Qwen3.8-27B native-v2 已完成： - -- 配置构造； -- 947 项权重装载； -- prompt prefill； -- autoregressive decode； -- GDN/SSM state 更新； -- 多 prompt 重复确定性； -- 32 x 32 严格 token 对拍。 - -llama.cpp 参考运行记录约为 prompt 29.0 token/s、generation 24.6 token/s。该数字只用于描述 -参考后端，不是 InfiniLM 性能结论。InfiniLM 的正式吞吐、首 token 延迟、显存峰值和不同 -prompt 长度曲线仍需要独立 benchmark 后才能下结论。 - -## 10. 最终结果与完成度判断 - -### 10.1 已完成 - -1. Qwen3.8-27B 模型架构可以在 InfiniLM 中构造和执行。 -2. GGUF 到 InfiniLM 的映射、打包和配置生成已经完成。 -3. Q8_0/Q4_K/Q5_K/Q6_K 四种原生权重算子已经完成。 -4. decode 和 prefill 两条执行路径都已跑通。 -5. 27B 全量模型可加载、生成，且明确走 U8 blob 原生 kernel。 -6. shape、字节、数值、端到端和严格对拍均有报告留档。 -7. 采样侧 `ignore_eos` 语义已与参考设置对齐。 -8. 通用的 F32 边界、mixed add-RMSNorm 和 mixed GEMM 实验能力已经实现。 - -### 10.2 当前最终采用结果 - -```text -功能适配：成功 -全量模型加载：成功 -Prefill：成功 -Decode：成功 -原生量化类型：Q8_0 / Q4_K / Q5_K / Q6_K -严格一致性：27/32 cases，920/1024 tokens -严格目标 >=29/32：未完成，属于可选优化项 +```bash +export LLAMA_CPP_DIR=/path/to/llama.cpp +python3 scripts/gguf_to_infinilm.py \ + --gguf /path/to/model.gguf \ + --out /path/to/infinilm-checkpoint \ + --tokenizer-dir /path/to/tokenizer-config \ + --verify sample ``` -### 10.3 如何理解“成功” - -如果验收标准是“让 InfiniLM 正确加载并推理 Qwen3.8-27B GGUF，并保留主要 GGUF 量化权重 -不展开”，本工作已经完成。 - -如果验收标准额外要求“InfiniLM 与 llama.cpp 在固定 32 个样例中至少 29 个逐 token 完全 -一致”，当前还差 2 个 exact case。后者是跨后端数值复现目标，不影响模型基本可用性；是否 -继续投入，应由比赛规则、评测规则或业务需求决定。 - -## 11. 对其他 GGUF 模型的复用方式 - -### 11.1 可直接复用的通用部分 - -- safetensors U8 `weight_bytes` 存储约定； -- `config.json` 中的 `ggml_types` 和 `quantization_config`； -- Q8_0/Q4_K/Q5_K/Q6_K block decoder； -- `linear_gguf` C API、C++ API 和 NVIDIA backend； -- small-M register GEMV 与 large-M prefill dispatch； -- row bytes、dtype、shape 和无 silent fallback 契约； -- GGUF 原字节抽样、解码交叉验证和 Linear 数值门； -- mixed-dtype 边界诊断能力； -- 32x32 token 对拍、首分叉和 logits margin 工具。 - -### 11.2 每个新模型仍需适配的部分 +`--tokenizer-dir` is optional. Vocabulary and merges are exported from GGUF; +the directory only supplies auxiliary files such as +`tokenizer_config.json` and a chat template. -- GGUF tensor name 到模型参数名的映射； -- fused tensor 的拆分/拼接规则； -- transpose、head permutation 或专家布局； -- norm gain 等模型特有权重约定； -- attention、MoE、SSM/GDN 等模型结构； -- cache/state 形状和生命周期； -- tokenizer、chat template、EOS 和 stop semantics； -- 模型实际包含但当前 kernel 未支持的 GGML 类型。 +Useful options: -### 11.3 推荐的新模型适配流程 +- `--dry-run`: validate metadata, shapes, orientation, and packed row sizes + without writing tensors. +- `--layers N`: create a loadable checkpoint containing the first N layers. +- `--verify {off,sample,all}`: control post-write verification. +- `--skip-pack`: keep existing tensor shards while refreshing configuration, + tokenizer files, and verification. +- `--emit-dense-ref PATH`: create a fully dequantized BF16 reference for + numerical comparison. -1. 读取 GGUF metadata，列出架构、tensor names、types、shapes 和 block bytes。 -2. 新增模型维度类和 `build_plan()` 映射，不先写运行时特例。 -3. 运行 dry-run，做源 GGUF 与目标模型参数的双向集合/shape 审计。 -4. 将已支持的四种类型标记为 blob；其他类型明确列为 BF16 例外或新增 decoder。 -5. 实现模型特有的 fused slice、transpose、permutation 和 norm convention。 -6. 打包并执行全量字节/shape/dtype 自检。 -7. 给各类型建立独立 block decode 和 Linear 数值门。 -8. 用 mini 模型覆盖所有类型，完成 prefill+decode 闭环。 -9. 加载全量模型，确认日志中首个和代表性权重进入 `linear_gguf`。 -10. 最后做生成质量、确定性、性能和参考后端一致性测试。 +The converter is intentionally fail-closed. A model whose dimensions differ +from the Qwen3.8 27B profile requires a new mapping profile instead of silently +reusing incompatible shapes. -这种流程下，新增一个结构相近且量化类型相同的模型，主要工作会集中在映射和模型结构层； -底层 GGUF 算子无需重复实现。 +## Mapping and transforms -## 12. 当前限制与后续建议 +`scripts/gguf_mapping.py` is the single source of truth for: -### 12.1 当前限制 +- GGUF-to-InfiniLM parameter names and shapes; +- packed versus dense storage; +- fused tensor slices; +- conversion-time value-head permutations; +- runtime activation-permutation metadata; +- generated InfiniLM model configuration. -1. embedding 和 lm_head 尚未采用原生 GGUF kernel。 -2. IQ4_XS/IQ4_NL 等 IQ 类型尚未原生支持。 -3. GGUF blob 当前未实现 tensor parallel 切分。 -4. prefill 已正确运行，但 tile 解码 workspace 和 GEMM 路由仍有性能优化空间。 -5. 当前 32x32 与 llama.cpp 不是完全一致，剩余 5 个样例存在首分叉。 -6. InfiniLM 正式性能数据尚未形成完整 benchmark 报告。 -7. Q8A、F32 GGUF output、final-FP32 等研究路径均默认关闭。 +`scripts/gguf_transforms.py` contains the shared NumPy transformations. GGUF +Qwen conversion stores selected value heads in tiled `[value][key]` order, +while InfiniLM uses grouped `[key][value]` order. Complete packed rows can be +permuted during conversion without modifying block bytes. -### 12.2 优先级建议 +Some output-projection transformations affect columns instead of rows. Moving +packed columns across quantization blocks would require requantization, so the +converter emits `activation_vperm` rules and the runtime applies the +equivalent grouped-to-tiled permutation to the input activation. -如果目标是工程交付，建议按以下顺序继续： +## Runtime integration -1. 固化默认环境、构建说明和一键回归； -2. 测 InfiniLM 吞吐、TTFT、decode latency 和显存峰值； -3. 增加 native lm_head 和 embedding，降低约 2.51 GiB 权重占用； -4. 根据目标模型分布决定是否实现 IQ4； -5. 若有多卡需求，再设计 blob tensor parallel； -6. 只有评测明确要求时，再继续追求 `>=29/32` 的严格一致性。 +`GGUFBlockQuantization` provides: -若继续严格一致性，下一步应拆分 final-FP32 变量，分别测试：仅 F32 norm output、F32 -residual sum + BF16 norm output、以及不同 lm_head reduction order。候选必须同时保留 -`math_02` exact 并修复 `zh_05`，再允许跑完整 32x32，避免无方向地枚举局部精度开关。 +- name-aware parameter layout selection; +- exact GGML type resolution; +- independent buffers for fused Linear shards; +- per-shard dispatch when fused projections use different GGML types; +- dense BF16 execution through the regular Linear operator; +- packed execution through `linear_gguf`; +- validation for dtype, contiguity, block divisibility, bias, and unsupported + tensor-parallel configurations. -## 13. 运行与复现要点 +Linear constructors pass a checkpoint stem to the quantization layer. Fused +projections retain one stem per shard so Q/K/V or gate/up components can resolve +different source types and concatenate their outputs in the original order. -环境脚本： +Qwen3.5 model changes supply these stems for attention, MLP, and gated-delta-net +projections. The Python remap also avoids applying a second normalization +`+1` adjustment because llama.cpp already bakes that offset into GGUF. -```bash -source /home/liuxd/InfiniLM/scripts/gguf_routeb_env.sh -``` - -严格基线的关键环境变量： - -```bash -export INFINI_GGUF_STRICT_SMALL_PREFILL=1 -export INFINI_GGUF_STRICT_SMALL_PREFILL_MAX_M=10 -``` +## Validation performed -实验变量默认不应设置： +The submitted branch has been validated with: -```text -INFINI_GGUF_DECODE_Q8A -INFINI_GGUF_DECODE_Q8A_TYPE -INFINI_GGUF_F32_DECODE_OUT -INFINI_GGUF_F32_DECODE_OUT_MATCH -INFINILM_FINAL_NORM_FP32_FUSED -``` +- repository formatting checks; +- a successful InfiniLM extension build; +- the official single-request test; +- the official offline benchmark; +- a local fixed MMLU-format smoke test; +- the official service test with 64/64 successful requests; +- end-to-end Qwen3.8 27B packed-checkpoint loading and generation. -关键报告： +Observed offline performance on the validation machine was approximately: ```text -/home/liuxd/tmp_routeb/reports/R3_strict_small_prefill_maxm10_32x32.json -/home/liuxd/tmp_routeb/reports/R3_compare_strict_small_prefill_maxm10_32x32.json -/home/liuxd/tmp_routeb/reports/R3_final_fp32_32x32.json -/home/liuxd/tmp_routeb/reports/R3_compare_final_fp32_32x32.json +decode throughput: 5.33 tokens/s +prefill throughput: 6.1 tokens/s +time to first token: 10.49 s ``` -构建时需要特别注意：只执行 `xmake build/install infiniop` 不足以保证 Python runtime 使用 -最新库。运行时实际优先加载： +These numbers establish functionality, not a portable performance claim. They +were not collected as a controlled comparison against llama.cpp with identical +prompts, context lengths, sampling, and device settings. -```text -/home/liuxd/InfiniCore/python/infinicore/lib/libinfiniop.so -``` +## Known limitations -因此 InfiniCore 改动后还必须执行 `xmake install _infinicore`，并核对安装目录与构建目录的 -动态库哈希一致。此前多次“代码改了但结果不变”的根因就是只更新了 `/home/liuxd/.infini/lib` -而没有更新 Python 实际加载的副本。 +- Packed GGUF tensor parallelism is not implemented. +- IQ4 tensors, embeddings, and the output head use dense BF16 fallbacks. +- Strict token-for-token agreement with llama.cpp is not guaranteed; numerical + comparisons are the appropriate correctness criterion for quantized kernels. +- A full external MMLU dataset run was unavailable on the validation machine; + only the local MMLU-format execution path was exercised. +- Upstream CI and maintainer review remain authoritative for merge readiness. -## 14. 结论 +## Extending Route B to another model -本次工作已经建立了一条完整、可验证、可复用的 GGUF Route B：从 GGUF tensor 映射、原始 -block bytes 打包，到 InfiniLM 类型路由、InfiniCore 原生 GPU 解码、prefill/decode，再到 -全量 27B 模型生成和跨后端对拍，整个链路已经打通。 +Most C++ work is reusable. A new model integration should: -新增的核心能力不是只针对某一个 Qwen 权重文件的临时代码，而是一个可承载多模型的 GGUF -块量化线性算子框架。适配其他 GGUF 大模型时，可以复用存储协议、四类 block decoder、 -`linear_gguf` 执行后端和验证体系，只需重点补充模型映射、结构接线和新的量化类型。 +1. Define a model-specific mapping profile with exact checkpoint names, logical + shapes, fused slices, and required transforms. +2. Generate exact `ggml_types` entries and any activation-permutation rules. +3. Pass checkpoint stems from each model Linear constructor. +4. Add native block types to InfiniCore only when the model uses unsupported + GGML formats; otherwise reuse `linear_gguf`. +5. Validate conversion with dry-run, exact key/shape/dtype checks, packed-row + byte preservation, a loadable small-layer checkpoint, and end-to-end output. +6. Benchmark correctness and performance separately with controlled settings. -当前应将项目状态定义为：**GGUF 功能适配完成，主要量化权重原生执行成功；严格一致性达到 -27/32，但 29/32 目标尚未完成且不是基本可用性的必要条件。** +This separation keeps GGUF storage and dispatch generic while isolating +architecture-specific tensor naming and permutation rules in the converter. diff --git a/csrc/config/quant_config.cpp b/csrc/config/quant_config.cpp index 195c70d44..7a25457b8 100644 --- a/csrc/config/quant_config.cpp +++ b/csrc/config/quant_config.cpp @@ -23,7 +23,8 @@ QuantConfig::get_quantization_method() const { } else if (quant_method == "fp8") { return std::make_shared(quantization_config); } else if (quant_method == "gguf") { - // 路线 B：GGUF block 字节原样进显存，kernel 在 InfiniCore（阶段 3） + // Route B keeps the original GGUF block bytes on the device and + // delegates decoding and multiplication to InfiniCore. return std::make_shared(quantization_config); } else if (quant_method == "quark") { return std::make_shared(quantization_config); diff --git a/csrc/layers/linear/base_linear.cpp b/csrc/layers/linear/base_linear.cpp index 92cebfbdb..10c51363b 100644 --- a/csrc/layers/linear/base_linear.cpp +++ b/csrc/layers/linear/base_linear.cpp @@ -24,8 +24,8 @@ BaseLinear::BaseLinear(size_t in_features, size_t out_features, in_features, out_features, split_dim, tp_rank, tp_size, tp_num_heads, dtype, bias, stem); - // 空布局 = 量化方案声明“这个 Linear 的参数不是一整块”（GGUF 的融合组）， - // 具体 shard 由派生类在构造体内用 init_fused_shards() 申请。 + // An empty layout marks a fused group whose shards are allocated by the + // derived class through init_fused_shards(). sharded_ = layout.empty(); for (const auto &desc : layout) { @@ -40,7 +40,7 @@ BaseLinear::BaseLinear(size_t in_features, size_t out_features, infinicore::Tensor BaseLinear::compute_linear(infinicore::Tensor &input) const { if (sharded_ && parameters_.empty()) { throw std::runtime_error( - "BaseLinear::compute_linear: 融合量化布局的 shard 还没注册（内部错误）"); + "BaseLinear::compute_linear: fused quantization shards were not registered"); } // Build params map from direct parameters only (not state_dict which uses a // static local and is not thread-safe across RankWorker threads). @@ -177,20 +177,20 @@ std::vector BaseLinear::init_fused_shards( shard_stems_.reserve(shards.size()); for (size_t i = 0; i < shards.size(); ++i) { const auto &sh = shards[i]; - // 下标 i 同时是参数 key 里的 shard 和 shard_stems_ 的位置：两者在同一行里产生 + // The index is shared by the shard parameter key and shard_stems_. shard_stems_.push_back(sh.stem); - // 各 shard 自己是一块完整的列并行参数，不做 TP 切分（GGUF 路径 tp_size 恒为 1， - // 量化类里会对 tp_size > 1 直接抛错，见方案 §6.2） + // Each shard is a complete column-parallel parameter. GGUF currently + // supports tp_size == 1 and rejects tensor-parallel execution. auto layout = quantization_->get_param_layout( in_features_, sh.out_features, split_dim_, 0, 1, -1, dtype_, false, sh.stem); if (layout.empty()) { throw std::runtime_error( - "BaseLinear::init_fused_shards: shard '" + sh.stem + "' 又返回了空布局"); + "BaseLinear::init_fused_shards: shard '" + sh.stem + "' returned an empty layout"); } for (const auto &desc : layout) { infinicore::nn::Parameter param( desc.shape, desc.dtype, device_, desc.split_dim, 0, 1, 0); - // key 里的 "shard." 前缀是量化类在 forward() 里还原拼接顺序的依据 + // The "shard." prefix preserves concatenation order in forward(). this->register_parameter( std::string(infinilm::quantization::GGUFBlockQuantization::SHARD_PREFIX) + std::to_string(i) + "." + desc.name, param); diff --git a/csrc/layers/linear/base_linear.hpp b/csrc/layers/linear/base_linear.hpp index ba1948200..b8d742a88 100644 --- a/csrc/layers/linear/base_linear.hpp +++ b/csrc/layers/linear/base_linear.hpp @@ -57,17 +57,17 @@ class BaseLinear : public infinicore::nn::Module { // One shard of a fused linear, for schemes that cannot share a single fused // buffer (GGUF block quantization: row_bytes differs per shard type). struct FusedShard { - std::string name; // "q_proj" / "gate_proj" ... 注册到父模块时用 - size_t out_features; // 本 shard 的逻辑输出行数 - std::string stem; // "layers.0.self_attn.q_proj." 类型表查询用 + std::string name; // Name used when registering with the parent module. + size_t out_features; // Logical output rows for this shard. + std::string stem; // Checkpoint stem used for quantization lookup. }; - // 为融合 Linear 逐 shard 各分配一块独立 buffer：本对象 parameters_ 里的 key 是 - // "shard."（i 即输出 dim(-1) 上的顺序），返回值里的 full_name 是 - // "."，交给调用方的 register_fn 注册到父模块（与 split_params 同路）。 - // 只有 get_param_layout(带 stem) 返回空布局（融合组）的方案才走这里。 - // 顺带把每个 shard 的 checkpoint stem 记进 shard_stems_（下标 = 上面的 i）： - // 组 stem 查不出各 shard 的格式，forward 必须把它们交还给量化方案。 + // Allocate one buffer per fused Linear shard. Local parameter keys use + // "shard.", while returned names use "." for + // registration with the parent module. This path is selected when the + // quantization scheme returns an empty layout for the fused group. + // shard_stems_ preserves each checkpoint stem for per-shard lookup during + // forward execution. std::vector init_fused_shards( const std::vector &shards); @@ -85,12 +85,12 @@ class BaseLinear : public infinicore::nn::Module { infinicore::DataType dtype_; int split_dim_ = -1; float alpha_ = 1.0f; - std::string stem_; // checkpoint 张量名路径（只给按名字查表的量化方案用，见 §6.0 纠正 2） - // init_fused_shards 记下的逐 shard checkpoint stem，下标 == parameters_ key 里的 i。 - // 与 key 在同一个循环里产生、forward 里消费，因此只是个局部不变量（不是跨阶段约定）； - // 非融合路径为空。语义见 BaseQuantization::forward 的 shard_stems 重载。 + std::string stem_; // Checkpoint tensor path used by name-based quantization lookup. + // Per-shard checkpoint stems recorded by init_fused_shards. The index + // matches the i in the corresponding "shard.*" parameter key. + // This vector is empty for non-fused Linear layers. std::vector shard_stems_; - bool sharded_ = false; // 融合量化布局：本对象不持有融合 buffer，参数在 shard.* 里 + bool sharded_ = false; // Fused layout whose parameters live in shard.* buffers. std::shared_ptr quantization_; }; diff --git a/csrc/layers/linear/fused_linear.cpp b/csrc/layers/linear/fused_linear.cpp index 11aada775..ee89783d6 100644 --- a/csrc/layers/linear/fused_linear.cpp +++ b/csrc/layers/linear/fused_linear.cpp @@ -105,11 +105,13 @@ QKVParallelLinear::QKVParallelLinear(size_t hidden_size, : QKVParallelLinear(hidden_size, q_dim, k_dim, v_dim, num_q_head, num_k_head, num_v_head, q_bias, k_bias, v_bias, quantization, dtype, device, rank_info, prefix) { register_fn_ = register_fn; if (this->sharded_) { - // GGUF：q/k/v 在本文件里 ggml 类型全不相同（§6.0 纠正 1），没有可 narrow 的 - // 融合 buffer —— 每 shard 各自一块，stem 指向各自的 checkpoint 张量。 + // GGUF Q, K, and V shards may use different GGML types, so there is + // no fused buffer to narrow. Each shard owns a separate buffer and + // its stem identifies the corresponding checkpoint tensor. if (prefix.empty()) { throw std::runtime_error( - "QKVParallelLinear: 按 checkpoint 张量名查表的量化方案（GGUF）必须传 prefix"); + "QKVParallelLinear requires `prefix` when the quantization " + "scheme resolves layouts by checkpoint tensor name."); } shard_specs_ = { {q_name, q_out_size_, prefix + "." + q_name + "."}, @@ -140,8 +142,9 @@ void QKVParallelLinear::register_fused_params() { void QKVParallelLinear::process_weights_after_loading() { BaseLinear::process_weights_after_loading(); - // 融合量化布局（sharded_）下 split_infos_ 为空，不会重跑：那些 shard 参数就是 - // 加载目标，重新分配会把已读进来的字节丢掉 + // `split_infos_` is empty for sharded quantization layouts because the + // shard parameters are the load targets. Reallocation would discard the + // bytes that were already loaded. if (register_fn_ && !split_infos_.empty()) { register_fused_params(); } @@ -202,11 +205,13 @@ GateUpParallelLinear::GateUpParallelLinear(size_t hidden_size, size_t intermedia : GateUpParallelLinear(hidden_size, intermediate_size, quantization, bias, dtype, device, rank_info, prefix) { register_fn_ = register_fn; if (this->sharded_) { - // GGUF：gate/up 在本文件 28/64 层类型不同（§6.0 纠正 1），两者 row_bytes 不同， - // 装不进同一块融合 buffer，所以各自一块、各自查自己是 blob 还是稠密。 + // GGUF gate and up shards may use different types and row sizes, so + // they cannot share a fused buffer. Each shard owns its buffer and + // independently resolves whether it is quantized or dense. if (prefix.empty()) { throw std::runtime_error( - "GateUpParallelLinear: 按 checkpoint 张量名查表的量化方案（GGUF）必须传 prefix"); + "GateUpParallelLinear requires `prefix` when the " + "quantization scheme resolves layouts by checkpoint tensor name."); } const size_t half = intermediate_size / tp_size_; shard_specs_ = { @@ -241,7 +246,8 @@ void GateUpParallelLinear::register_fused_params() { void GateUpParallelLinear::process_weights_after_loading() { BaseLinear::process_weights_after_loading(); - // 同 QKVParallelLinear：sharded_ 时 split_infos_ 为空，不重跑切分 + // As in `QKVParallelLinear`, sharded layouts have no `split_infos_` and + // must not repeat the split allocation. if (register_fn_ && !split_infos_.empty()) { register_fused_params(); } diff --git a/csrc/layers/linear/fused_linear.hpp b/csrc/layers/linear/fused_linear.hpp index 0a9d6c987..edb81c899 100644 --- a/csrc/layers/linear/fused_linear.hpp +++ b/csrc/layers/linear/fused_linear.hpp @@ -94,10 +94,12 @@ class QKVParallelLinear : public infinilm::nn::ColumnParallelLinear { size_t num_kv_head_replicas_ = 1; RegisterParamFn register_fn_; std::vector split_infos_; - // GGUF 等「每 shard 一块 buffer」的方案用（与 split_infos_ 二选一，见 sharded_） + // Used by layouts such as GGUF that allocate one buffer per shard. This + // is mutually exclusive with `split_infos_`; see `sharded_`. std::vector shard_specs_; - // 把各 shard 参数交给 register_fn（narrow 视图或独立 buffer，两条路同一入口） + // Pass each shard parameter to `register_fn`, whether it is a narrowed + // view or an independent buffer. void register_fused_params(); }; diff --git a/csrc/layers/mlp/mlp.hpp b/csrc/layers/mlp/mlp.hpp index 7e19436e7..d929b1cd5 100644 --- a/csrc/layers/mlp/mlp.hpp +++ b/csrc/layers/mlp/mlp.hpp @@ -24,8 +24,9 @@ class MLP : public infinicore::nn::Module { * * @param model_config: Model configuration. * @param device Device to create tensors on - * @param prefix 本层在 checkpoint 里的路径（形如 "layers.0.mlp"）。只给需要 - * 按张量名查表的量化方案用（GGUF）；其他方案留空即可。 + * @param prefix Checkpoint path for this layer, such as + * `layers.0.mlp`. It is used only by quantization schemes, such as + * GGUF, that resolve layouts by tensor name. */ MLP(std::shared_ptr model_config, const infinicore::Device &device, diff --git a/csrc/layers/quantization/gguf.cpp b/csrc/layers/quantization/gguf.cpp index 583a06856..c15bc2aaa 100644 --- a/csrc/layers/quantization/gguf.cpp +++ b/csrc/layers/quantization/gguf.cpp @@ -15,10 +15,9 @@ namespace infinilm::quantization { namespace { -// ggml 块的 (block_size, type_size)：一行 row_bytes = in / block_size * type_size。 -// 数值取自 §2.3 的实测（与 gguf-py 的 GGML_QUANT_SIZES、llama.cpp 的 ggml.h 一致）。 -// 只列本路线 kernel 计划支持的类型；表外的 id 一律抛错，逼着打包期把它稠密化， -// 而不是运行期猜一个 stride（猜错 = 读越界 = 结果错）。 +// GGML block metadata. A packed row contains +// in_features / block_size * type_size bytes. Unsupported types must be +// converted to dense BF16 instead of relying on a guessed runtime stride. struct GgmlBlock { int64_t id; const char *name; @@ -75,13 +74,13 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) : BaseQuantization(quant_config) { if (!quant_config_.is_object() || !quant_config_.contains("ggml_types")) { throw std::runtime_error( - "GGUFBlockQuantization: quantization_config 缺 ggml_types（阶段 1 打包器写入）"); + "GGUFBlockQuantization: quantization_config is missing ggml_types"); } key_prefix_ = get_or("key_prefix", ""); const auto &table = quant_config_.at("ggml_types"); if (!table.is_object() || table.empty()) { - throw std::runtime_error("GGUFBlockQuantization: ggml_types 表为空"); + throw std::runtime_error("GGUFBlockQuantization: ggml_types is empty"); } size_t n_blob = 0; @@ -89,10 +88,8 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) size_t n_outside = 0; for (const auto &kv : table.items()) { const std::string &name = kv.key(); - // 实测：产物 121 张量里只有 lm_head.weight 不在 model.language_model. 子树下 - // （lm_head 在 C++ 模块树里是根节点的兄弟），这类键原样保留、不裁前缀。 - // 它们永远不会被 stem 查到（lm_head 走非量化 ctor），留着是为了让类型表 - // 与产物张量名保持双向逐字相等（阶段 1 自检的判据）。 + // Keep keys outside key_prefix unchanged. This includes root-level + // tensors such as lm_head.weight and preserves exact checkpoint names. std::string key = name; if (!key_prefix_.empty() && name.compare(0, key_prefix_.size(), key_prefix_) == 0) { key = name.substr(key_prefix_.size()); @@ -105,53 +102,55 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) const std::string v = kv.value().get(); if (v != DENSE_MARK) { throw std::runtime_error( - "GGUFBlockQuantization: '" + name + "' 的取值 '" + v + "' 既不是整数 type id 也不是 \"" + DENSE_MARK + "\""); + "GGUFBlockQuantization: value '" + v + "' for '" + name + + "' is neither an integer type id nor \"" + DENSE_MARK + "\""); } ++n_dense; } else { if (!kv.value().is_number_integer()) { throw std::runtime_error( - "GGUFBlockQuantization: '" + name + "' 的取值不是整数 ggml type id"); + "GGUFBlockQuantization: value for '" + name + "' is not an integer ggml type id"); } id = kv.value().get(); if (id == DENSE_BF16) { throw std::runtime_error( - "GGUFBlockQuantization: '" + name + "' 的 type id 与稠密标记 -1 冲突"); + "GGUFBlockQuantization: type id for '" + name + "' conflicts with dense sentinel -1"); } if (!ggml_block(id)) { throw std::runtime_error( - "GGUFBlockQuantization: '" + name + "' 是不支持的 ggml type id=" + std::to_string(id) + "（当前支持 " + supported_types() + "；其余类型必须在打包期稠密化，不能留到运行期猜）"); + "GGUFBlockQuantization: '" + name + "' uses unsupported ggml type id=" + + std::to_string(id) + " (supported: " + supported_types() + + "); unsupported types must be converted to dense BF16"); } ++n_blob; } if (!types_.emplace(std::move(key), TypeEntry{id, name}).second) { throw std::runtime_error( - "GGUFBlockQuantization: 裁掉 key_prefix 后键重复：'" + name + "'"); + "GGUFBlockQuantization: duplicate key after removing key_prefix: '" + name + "'"); } } - // 激活 V 头置换规则（out_proj 一类「要置换的是权重列」的条目）。缺这个键 = 拒启， - // 不静默不置换：漏一次置换 = 48 个 value head 与权重列整体错位 - // =「能加载、能跑、输出错」，正是阶段 4 §8.5 要排除的那一类错。 + // Activation value-head permutation is required even when the rule list is + // empty. Missing metadata could silently misalign activations and columns. if (!quant_config_.contains("activation_vperm")) { throw std::runtime_error( - "GGUFBlockQuantization: quantization_config 缺 activation_vperm（out_proj 的 V 头" - "列序置换规则）——旧产物用打包器 --skip-pack 刷新 config.json 即可，不必重打包权重"); + "GGUFBlockQuantization: quantization_config is missing activation_vperm; " + "refresh config.json with the converter --skip-pack option"); } { const auto &rules = quant_config_.at("activation_vperm"); if (!rules.is_array()) { - throw std::runtime_error("GGUFBlockQuantization: activation_vperm 必须是数组，实际是 " + std::string(rules.type_name())); + throw std::runtime_error("GGUFBlockQuantization: activation_vperm must be an array, got " + std::string(rules.type_name())); } for (const auto &j : rules) { if (!j.is_object()) { - throw std::runtime_error("GGUFBlockQuantization: activation_vperm 条目不是对象"); + throw std::runtime_error("GGUFBlockQuantization: activation_vperm entry must be an object"); } ActVPerm r; for (const char *key : {"suffix", "num_k_heads", "num_v_per_k", "head_dim"}) { if (!j.contains(key)) { - throw std::runtime_error("GGUFBlockQuantization: activation_vperm 条目缺 '" + std::string(key) + "'"); + throw std::runtime_error("GGUFBlockQuantization: activation_vperm entry is missing '" + std::string(key) + "'"); } } r.suffix = j.at("suffix").get(); @@ -160,26 +159,26 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) r.hd = j.at("head_dim").get(); if (r.suffix.empty() || r.suffix.back() != '.' || !r.n_k || !r.r || !r.hd) { throw std::runtime_error( - "GGUFBlockQuantization: activation_vperm 条目不合法：suffix='" + r.suffix + "' 需以 '.' 结尾，三个维度需为正（实际 " + std::to_string(r.n_k) + "/" + std::to_string(r.r) + "/" + std::to_string(r.hd) + "）"); + "GGUFBlockQuantization: invalid activation_vperm entry: suffix='" + + r.suffix + "' must end with '.', and dimensions must be positive (got " + + std::to_string(r.n_k) + "/" + std::to_string(r.r) + "/" + + std::to_string(r.hd) + ")"); } if (std::any_of(vperm_.begin(), vperm_.end(), [&r](const ActVPerm &e) { return e.suffix == r.suffix; })) { - throw std::runtime_error("GGUFBlockQuantization: activation_vperm 里 '" + r.suffix + "' 出现多次（同一条规则只能有一份）"); + throw std::runtime_error("GGUFBlockQuantization: duplicate activation_vperm suffix '" + r.suffix + "'"); } vperm_.push_back(std::move(r)); } } - // n_outside 有两种成因，得分开说：早先全量产物未声明 key_prefix，整张表都被计入 - // 「前缀外」（实测日志里印成「前缀外 947」），很容易被读成「947 条都查不到」。 spdlog::info( - "GGUF block quantization: 类型表 {} 条（blob {} / 稠密 {} / 未裁前缀 {}），key_prefix='{}'{}", + "GGUF block quantization: {} entries (blob {} / dense {} / outside prefix {}), key_prefix='{}'{}", types_.size(), n_blob, n_dense, n_outside, key_prefix_, key_prefix_.empty() - ? "（未声明：表键即 safetensors 张量名的相对形态，整表不裁前缀）" - : "（在 prefix 之外，如 lm_head）"); + ? " (not set; table keys are relative safetensors names)" + : " (for example, root-level lm_head entries)"); - // 与下一行一起构成「本次加载到底有没有在做置换」的唯一可 grep 证据（A/B 靠它） std::string vs; for (const auto &r : vperm_) { if (!vs.empty()) { @@ -187,13 +186,13 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) } vs += r.suffix + "=" + std::to_string(r.n_k) + "x" + std::to_string(r.r) + "x" + std::to_string(r.hd); } - spdlog::info("GGUF block quantization: 激活 V 头置换规则 {} 条（grouped->tiled）：{}", - vperm_.size(), vs.empty() ? "无" : vs); + spdlog::info("GGUF block quantization: {} activation V-head permutation rules (grouped->tiled): {}", + vperm_.size(), vs.empty() ? "none" : vs); } GGUFBlockQuantization::~GGUFBlockQuantization() { if (n_blob_ + n_dense_ + n_group_ > 0) { - spdlog::info("GGUF block quantization: 布局查表命中 blob {} / 稠密 {} / 融合组 {}", + spdlog::info("GGUF block quantization: layout matches blob {} / dense {} / fused group {}", n_blob_, n_dense_, n_group_); } } @@ -203,8 +202,8 @@ bool GGUFBlockQuantization::is_known_type(int64_t type_id) { } std::string GGUFBlockQuantization::describe(const std::string &stem) const { - // 报错信息里拼回绝对名，方便直接在产物 / pack_report.json 里 grep - return (stem.empty() ? std::string("<空 stem>") : key_prefix_ + stem); + // Restore the absolute checkpoint name for searchable diagnostics. + return (stem.empty() ? std::string("") : key_prefix_ + stem); } int64_t GGUFBlockQuantization::resolve(const std::string &stem, std::string *matched_key) const { @@ -214,11 +213,14 @@ int64_t GGUFBlockQuantization::resolve(const std::string &stem, std::string *mat const auto dense_it = types_.find(dense_key); const int hits = (blob_it != types_.end()) + (dense_it != types_.end()); - // 命中 0 个 = 拼错或产物缺张量；命中 2 个 = 打包器同时写了 blob 与稠密版本。 - // 两种都必须是异常：任何「查不到就走稠密」的回落都会变成能加载、显存暴涨、结果错。 + // Require exactly one packed or dense candidate. Falling back on missing + // metadata could load successfully while producing incorrect output. if (hits != 1) { throw std::runtime_error( - "GGUFBlockQuantization: stem '" + describe(stem) + "' 在类型表里命中 " + std::to_string(hits) + " 个候选（期望恰好 1 个：'" + blob_key + "' 或 '" + dense_key + "'）；表共 " + std::to_string(types_.size()) + " 条"); + "GGUFBlockQuantization: stem '" + describe(stem) + "' matched " + + std::to_string(hits) + " type-table candidates; expected exactly one of '" + + blob_key + "' or '" + dense_key + "' among " + + std::to_string(types_.size()) + " entries"); } const auto &hit = blob_it != types_.end() ? *blob_it : *dense_it; if (matched_key) { @@ -238,11 +240,14 @@ size_t GGUFBlockQuantization::row_bytes(size_t in_features, int64_t type_id) con const GgmlBlock *b = ggml_block(type_id); if (!b) { throw std::runtime_error( - "GGUFBlockQuantization: 不支持的 ggml type id=" + std::to_string(type_id) + "（当前支持 " + supported_types() + "）"); + "GGUFBlockQuantization: unsupported ggml type id=" + std::to_string(type_id) + + " (supported: " + supported_types() + ")"); } if (in_features % b->block_size != 0) { throw std::runtime_error( - "GGUFBlockQuantization: in_features=" + std::to_string(in_features) + " 不能被 " + b->name + " 的块大小 " + std::to_string(b->block_size) + " 整除"); + "GGUFBlockQuantization: in_features=" + std::to_string(in_features) + + " is not divisible by " + b->name + " block size " + + std::to_string(b->block_size)); } return in_features / b->block_size * b->type_size; } @@ -263,17 +268,18 @@ infinicore::Tensor GGUFBlockQuantization::gather_grouped_to_tiled( const size_t ndim = shape.size(); if (ndim < 2) { throw std::runtime_error( - "GGUFBlockQuantization: " + name + " 的激活 rank=" + std::to_string(ndim) + "，至少要是 [..., in_features]"); + "GGUFBlockQuantization: activation for " + name + " has rank=" + + std::to_string(ndim) + "; expected at least [..., in_features]"); } const size_t K = shape[ndim - 1]; const size_t want = rule.n_k * rule.r * rule.hd; if (K != want) { - // TP 会把 in 维切成没关头数不等的分片，套上整头置换就是静默错位； - // 与 get_param_layout 里「暂不支持 tensor parallel」的护栏保持同一口径。 throw std::runtime_error( - "GGUFBlockQuantization: " + name + " 的激活末维 " + std::to_string(K) + " != activation_vperm 的 num_k_heads*num_v_per_k*head_dim = " + std::to_string(want) + "（切分后的分片不能套整头置换）"); + "GGUFBlockQuantization: activation last dimension for " + name + " is " + + std::to_string(K) + ", expected num_k_heads*num_v_per_k*head_dim=" + + std::to_string(want) + "; head permutation cannot be applied to a shard"); } - // [..., n_k, r, hd] -> [..., r, n_k, hd]：把 grouped（k-major）的激活置换为 tiled（v-major）。 + // [..., n_k, r, hd] -> [..., r, n_k, hd], grouped to tiled order. const size_t k_axis = ndim - 1; infinicore::Shape grouped(shape.begin(), shape.end() - 1); grouped.insert(grouped.end(), {rule.n_k, rule.r, rule.hd}); @@ -292,8 +298,7 @@ std::vector GGUFBlockQuantization::get_param_layout( size_t, size_t, int, int, int, int, const infinicore::DataType &, bool) const { throw std::runtime_error( - "GGUFBlockQuantization: 不接受无名字的 get_param_layout 调用（每个权重的 ggml " - "类型只能由 checkpoint 张量名决定）"); + "GGUFBlockQuantization: get_param_layout requires a checkpoint stem to resolve the ggml type"); } std::vector GGUFBlockQuantization::get_param_layout( @@ -307,23 +312,26 @@ std::vector GGUFBlockQuantization::get_param_layout( if (stem.empty()) { throw std::runtime_error( - "GGUFBlockQuantization: 构造 Linear 时没有传 checkpoint stem（in=" + std::to_string(in_features) + ", out=" + std::to_string(out_features) + "）——方案 §6.1 列出的构造点必须全部补上 prefix/stem"); + "GGUFBlockQuantization: missing checkpoint stem while constructing Linear (in=" + + std::to_string(in_features) + ", out=" + std::to_string(out_features) + ")"); } if (tp_size != 1 || tp_rank != 0) { throw std::runtime_error( - "GGUFBlockQuantization: 暂不支持 tensor parallel（blob 的 TP 切分留待多卡阶段）：" + describe(stem)); + "GGUFBlockQuantization: tensor parallelism is not supported for packed GGUF weights: " + + describe(stem)); } if (bias) { throw std::runtime_error( - "GGUFBlockQuantization: GGUF 产物里没有 bias 张量：" + describe(stem)); + "GGUFBlockQuantization: GGUF checkpoint has no bias tensor for " + describe(stem)); } - // 不带结尾 '.' 的 stem 表示「融合 Linear」：本类不为它分配任何 buffer， - // 各 shard 由 BaseLinear::init_fused_shards 用各自的 stem 单独申请。 + // A stem without a trailing '.' identifies a fused Linear group. Its + // individual shard buffers are allocated by BaseLinear::init_fused_shards. if (stem.back() != '.') { if (!has_group(stem)) { throw std::runtime_error( - "GGUFBlockQuantization: 融合组 stem '" + stem + "' 在类型表里没有任何 '" + stem + "..*' 条目"); + "GGUFBlockQuantization: fused-group stem '" + stem + + "' has no '" + stem + "..*' entry in the type table"); } ++n_group_; return {}; @@ -332,7 +340,7 @@ std::vector GGUFBlockQuantization::get_param_layout( const int64_t id = resolve(stem); if (id == DENSE_BF16) { ++n_dense_; - // 与 NoneQuantization 同形：打包期已反量化成 BF16，正常 GEMM + // The converter stored this tensor as dense BF16; use regular GEMM. return {{"weight", {out_features, in_features}, dtype, split_dim, tp_rank, tp_size}}; } @@ -344,8 +352,7 @@ std::vector GGUFBlockQuantization::get_param_layout( infinicore::Tensor GGUFBlockQuantization::forward( const ParamsMap &, const infinicore::Tensor &, bool, float) const { throw std::runtime_error( - "GGUFBlockQuantization: 不接受无名字的 forward 调用（每个权重的 ggml 类型只能由 " - "checkpoint 名字决定，融合 Linear 还需要 shard_stems）"); + "GGUFBlockQuantization: forward requires a checkpoint stem; fused Linear also requires shard_stems"); } infinicore::Tensor GGUFBlockQuantization::forward_shard( @@ -356,22 +363,24 @@ infinicore::Tensor GGUFBlockQuantization::forward_shard( int64_t type_id, const std::string &table_key) const { if (suffix == DENSE_SUFFIX) { - // 参数后缀是 get_param_layout 按 resolve() 结果选的，两者不一致 = 有地方改坏了 - // （blob 被当成 BF16 读就是「能加载、结果错」），宁可抛。 + // The suffix selected by get_param_layout must agree with the type table. if (type_id != DENSE_BF16) { throw std::runtime_error( - "GGUFBlockQuantization: " + table_key + " 的参数后缀是 " + DENSE_SUFFIX + "，但类型表给出的 ggml type id=" + std::to_string(type_id) + "（不一致）"); + "GGUFBlockQuantization: " + table_key + " has parameter suffix " + + DENSE_SUFFIX + " but type table reports ggml type id=" + + std::to_string(type_id)); } auto x = input->is_contiguous() ? input : input->contiguous(); auto w = weight->is_contiguous() ? weight : weight->contiguous(); return infinicore::op::linear(x, w, std::nullopt, alpha); } if (suffix == BLOB_SUFFIX) { - // 权重保持量化形态：块字节直接喂 kernel。这里绝不静默回落稠密 GEMM—— - // 那等于把块字节当成 BF16 读，能跑完但结果是错的，宁可抛。 + // Pass packed block bytes directly to the kernel. Never reinterpret + // them as BF16 through a dense fallback. if (alpha != 1.0F) { throw std::runtime_error( - "linear_gguf: 不支持 alpha=" + std::to_string(alpha) + "（GGUF blob 路径没有缩放权重，alpha!=1 说明上层期望与实现不符）：" + table_key); + "linear_gguf: alpha=" + std::to_string(alpha) + + " is unsupported for packed GGUF weights: " + table_key); } auto x = input->is_contiguous() ? input : input->contiguous(); auto w = weight->is_contiguous() ? weight : weight->contiguous(); @@ -384,9 +393,8 @@ infinicore::Tensor GGUFBlockQuantization::forward_shard( M *= x_shape[i]; } const size_t N = static_cast(w->size(0)); - // 这里不再设 M 上限：gemv（小 M）与 prefill（大 M）两条路径在 - // linear_gguf 算子内部按同一个 kMaxDecodeM 谓词选。上层再留一份数字， - // 两边一旦不同步就只剩一条过时的门（阶段 3.3 之前正是这种情形）。 + // linear_gguf selects the decode or prefill kernel using its shared + // kMaxDecodeM threshold, so this layer does not duplicate that limit. auto flat = x->view({M, K}); flat = flat->is_contiguous() ? flat : flat->contiguous(); @@ -395,19 +403,18 @@ infinicore::Tensor GGUFBlockQuantization::forward_shard( ? infinicore::DataType::F32 : input->dtype(); auto out = infinicore::Tensor::empty({M, N}, out_dtype, input->device()); - // 只报第一个 blob 调用：端到端排障时区分「死在 blob 路径之前」与 - // 「已在 kernel 里」，两者处置完全不同（前者是接线问题，后者是下游算子）。 + // Log the first packed invocation as a lightweight wiring diagnostic. static std::atomic blob_calls{0}; if (blob_calls.fetch_add(1) == 0) { spdlog::info( - "linear_gguf: 首个 blob 前向 {} — M={} N={} K={} ggml_type={} row_bytes={}", + "linear_gguf: first packed forward {} -- M={} N={} K={} ggml_type={} row_bytes={}", table_key, M, N, K, type_id, w->size(1)); } if (f32_decode_out) { static std::atomic f32_calls{0}; if (f32_calls.fetch_add(1) == 0) { spdlog::warn( - "linear_gguf: 实验性 F32 decode 输出已启用，首个命中 {} — M={} N={} K={}", + "linear_gguf: experimental F32 decode output enabled; first match {} -- M={} N={} K={}", table_key, M, N, K); } } @@ -418,7 +425,8 @@ infinicore::Tensor GGUFBlockQuantization::forward_shard( return out->view(out_shape); } throw std::runtime_error( - "GGUFBlockQuantization: " + table_key + " 的参数后缀 '" + suffix + "' 既不是 " + DENSE_SUFFIX + " 也不是 " + BLOB_SUFFIX); + "GGUFBlockQuantization: parameter suffix '" + suffix + "' for " + table_key + + " is neither " + DENSE_SUFFIX + " nor " + BLOB_SUFFIX); } infinicore::Tensor GGUFBlockQuantization::forward( @@ -427,7 +435,7 @@ infinicore::Tensor GGUFBlockQuantization::forward( bool has_bias, float alpha, const std::string &stem) const { - // 没有 shard stems 就只能服务非融合布局；融合 Linear 走下面那个重载。 + // The overload below handles fused Linear layers with per-shard stems. return forward(params, input, has_bias, alpha, stem, {}); } @@ -440,39 +448,40 @@ infinicore::Tensor GGUFBlockQuantization::forward( const std::vector &shard_stems) const { if (has_bias) { throw std::runtime_error( - "GGUFBlockQuantization: 不支持 bias（" + describe(stem) + "）"); + "GGUFBlockQuantization: bias is not supported (" + describe(stem) + ")"); } - // 先按规则置换激活，再进 blob / 稠密两条路：两条路的权重列序都直接来自同一个 GGUF - // 张量（稠密化只换 dtype 不动列序），需要置换的语义完全一致。 + // Apply activation permutation before either packed or dense execution; + // both layouts preserve the source GGUF column order. infinicore::Tensor x = input; const ActVPerm *rule = vperm_rule(stem); if (!shard_stems.empty()) { for (const auto &s : shard_stems) { if (vperm_rule(s)) { throw std::runtime_error( - "GGUFBlockQuantization: 融合组 '" + describe(stem) + "' 的 shard '" - + describe(s) + "' 命中激活置换规则，但一根 input 同时服务于所有 shard，" - "无法按 shard 分别置换（实际产物里 out_proj 不是融合 Linear，走到这里=接线错）"); + "GGUFBlockQuantization: shard '" + describe(s) + "' in fused group '" + + describe(stem) + "' matches an activation-permutation rule, but one input " + "cannot be permuted independently for each shard"); } } } else if (rule) { x = gather_grouped_to_tiled(*rule, input, describe(stem)); - // 只报第一次：端到端排障时它是「gather 真的在跑」的唯一证据，不靠日志量堆 + // Log the first permutation as a lightweight wiring diagnostic. static std::atomic vperm_applied{0}; if (vperm_applied.fetch_add(1) == 0) { spdlog::info( - "linear_gguf: 首个激活 V 头置换 {} — grouped->tiled {}x{}x{}", + "linear_gguf: first activation V-head permutation {} -- grouped->tiled {}x{}x{}", describe(stem), rule->n_k, rule->r, rule->hd); } } - // 非融合：一个参数（weight 或 weight_bytes），stem 就是它自己的完整 checkpoint 路径 + // A non-fused layer owns exactly one weight or weight_bytes parameter. if (shard_stems.empty()) { if (params.size() != 1) { throw std::runtime_error( - "GGUFBlockQuantization: " + describe(stem) + " 有 " + std::to_string(params.size()) + " 个参数却没收到 shard_stems" - "（内部错误：BaseLinear::compute_linear 没有把 shard_stems_ 传下来）"); + "GGUFBlockQuantization: " + describe(stem) + " has " + + std::to_string(params.size()) + + " parameters but no shard_stems; BaseLinear::compute_linear did not pass them"); } const auto &kv = *params.begin(); std::string table_key; @@ -480,31 +489,34 @@ infinicore::Tensor GGUFBlockQuantization::forward( return forward_shard(kv.first, kv.second, x, alpha, id, table_key); } - // 融合：parameters_ 里是 shard.，i 就是它们在输出 dim(-1) 上的顺序， - // 与融合 Linear 的 SplitInfo 顺序一致 —— 所以输出拼回一根连续的 [.., sum(out_i)]， - // 上层的 narrow 逻辑完全不用改（方案 §6.0 纠正 1）。 - // 每个 shard 的 ggml 类型由 shard_stems[i] 查表（实测 q/k/v 不同类型，见 §7.2）。 + // Fused parameters use shard., where i is their order along the + // output dimension and matches SplitInfo. Resolve each shard independently + // and concatenate outputs in that order. if (shard_stems.size() != params.size()) { throw std::runtime_error( - "GGUFBlockQuantization: " + describe(stem) + " 有 " + std::to_string(params.size()) + " 个 shard 参数但收到 " + std::to_string(shard_stems.size()) + " 个 shard stem（内部错误：两者应在 " - "BaseLinear::init_fused_shards 的同一个循环里产生）"); + "GGUFBlockQuantization: " + describe(stem) + " has " + + std::to_string(params.size()) + " shard parameters but received " + + std::to_string(shard_stems.size()) + + " shard stems; both must be created by BaseLinear::init_fused_shards"); } std::vector> parts; for (const auto &kv : params) { if (kv.first.compare(0, std::string(SHARD_PREFIX).size(), SHARD_PREFIX) != 0) { throw std::runtime_error( - "GGUFBlockQuantization: 融合 Linear 的参数名 '" + kv.first + "' 不是 " + SHARD_PREFIX + ". 形式（" + describe(stem) + "）"); + "GGUFBlockQuantization: fused Linear parameter '" + kv.first + + "' does not match " + SHARD_PREFIX + ". (" + describe(stem) + ")"); } const size_t dot = kv.first.find('.'); if (dot == std::string::npos) { throw std::runtime_error( - "GGUFBlockQuantization: 融合 Linear 的参数名 '" + kv.first + "' 缺 '.'"); + "GGUFBlockQuantization: fused Linear parameter '" + kv.first + "' is missing '.'"); } const size_t idx = std::stoul(kv.first.substr(std::string(SHARD_PREFIX).size(), dot - std::string(SHARD_PREFIX).size())); if (idx >= shard_stems.size()) { throw std::runtime_error( - "GGUFBlockQuantization: 参数名 '" + kv.first + "' 的 shard 下标越出 shard_stems（" + describe(stem) + "）"); + "GGUFBlockQuantization: shard index in parameter '" + kv.first + + "' is outside shard_stems (" + describe(stem) + ")"); } std::string table_key; const int64_t id = resolve(shard_stems[idx], &table_key); @@ -527,8 +539,8 @@ std::vector GGUFBlockQuantization::split_params( const std::unordered_map ¶ms, const std::vector &splits, int, int, int, int) const { - // 恒等映射：GGUF 的融合 Linear 已经按 shard 分配了独立 buffer（没有可 narrow 的父 - // buffer），这里只把 shard. 换成 . 交给 register_fn。 + // Fused GGUF shards already have independent buffers. Rename + // shard. to . without slicing data. std::vector result; for (size_t i = 0; i < splits.size(); ++i) { const std::string head = std::string(SHARD_PREFIX) + std::to_string(i) + "."; @@ -542,7 +554,10 @@ std::vector GGUFBlockQuantization::split_params( } if (result.size() != splits.size()) { throw std::runtime_error( - "GGUFBlockQuantization::split_params: " + std::to_string(splits.size()) + " 个 shard 只匹配到 " + std::to_string(result.size()) + " 个参数（GGUF 融合 Linear 应走 BaseLinear::init_fused_shards）"); + "GGUFBlockQuantization::split_params: expected " + + std::to_string(splits.size()) + " shard parameters but matched " + + std::to_string(result.size()) + + "; fused GGUF Linear must use BaseLinear::init_fused_shards"); } return result; } @@ -558,14 +573,14 @@ std::shared_ptr GGUFBlockQuantization::process_weights_after_l } if (kv.second->dtype() != infinicore::DataType::U8) { throw std::runtime_error( - "GGUFBlockQuantization: blob 参数 '" + kv.first + "' 的 dtype 不是 U8"); + "GGUFBlockQuantization: packed parameter '" + kv.first + "' must have U8 dtype"); } if (!kv.second->is_contiguous()) { throw std::runtime_error( - "GGUFBlockQuantization: blob 参数 '" + kv.first + "' 不连续（阶段 3 kernel 按行取字节）"); + "GGUFBlockQuantization: packed parameter '" + kv.first + "' must be contiguous"); } } - // 返回 nullptr：不换方案、不改写字节 + // Keep the quantization scheme and raw bytes unchanged. return nullptr; } diff --git a/csrc/layers/quantization/gguf.hpp b/csrc/layers/quantization/gguf.hpp index 6af785587..bd55cf485 100644 --- a/csrc/layers/quantization/gguf.hpp +++ b/csrc/layers/quantization/gguf.hpp @@ -8,23 +8,22 @@ namespace infinilm::quantization { -// GGUF block quantization（路线 B）：打包器把 GGUF 张量的**原始块字节**逐字节搬进 -// safetensors，一行的宽度是 row_bytes(in_features, type)，不是 in_features 个元素。 -// 因此每个权重的布局只能由 checkpoint 张量名查表决定，逻辑形状推不出来。 +// GGUF block quantization stores the original block bytes in safetensors. +// Each row has row_bytes(in_features, type) bytes rather than in_features +// scalar elements, so the checkpoint tensor name determines the layout. // -// 类型表 = config.json:quantization_config.ggml_types，键就是 safetensors 里的张量名 -// 原文（打包器自检保证与产物张量名双向逐字相等），值要么是 ggml type id，要么是 -// 字符串 "dense_bf16"（打包期已反量化成 BF16 的那些：embed / lm_head / norm / -// GDN 标量 / v1 的 IQ4_*）。quantization_config.key_prefix 在这里裁掉一次，因为 -// 挂在 model. 以下的模块不知道自己的绝对路径。详见执行方案 §2.3 / §6.0。 +// config.json:quantization_config.ggml_types maps safetensors names to either +// a ggml type id or "dense_bf16" for tensors dequantized during conversion. +// key_prefix is removed once because nested modules do not know their absolute +// checkpoint path. class GGUFBlockQuantization : public BaseQuantization { public: - // 稠密化条目在类型表里的取值（与任何 ggml type id 都不冲突：id 从 0 起） + // Sentinel for entries stored as dense BF16 rather than GGUF blocks. static constexpr int64_t DENSE_BF16 = -1; - // blob 权重在 checkpoint 里的张量名后缀（与 scripts/gguf_mapping.BLOB_SUFFIX 一致） + // Checkpoint suffix for raw block data; shared with scripts/gguf_mapping.py. static constexpr const char *BLOB_SUFFIX = "weight_bytes"; static constexpr const char *DENSE_SUFFIX = "weight"; - // 融合 Linear 在 parameters_ 里给各 shard 用的 key 前缀，见 BaseLinear::init_fused_shards + // Parameter-key prefix used for fused Linear shards. static constexpr const char *SHARD_PREFIX = "shard"; explicit GGUFBlockQuantization(const nlohmann::json &quant_config); @@ -35,7 +34,7 @@ class GGUFBlockQuantization : public BaseQuantization { return QuantScheme::GGUF_BLOCK; } - // 名称未知的布局无法决定 ggml 类型，GGUF 只能通过带 stem 的重载被调用 + // A stem is required to resolve the ggml type. std::vector get_param_layout( size_t in_features, size_t out_features, int split_dim, int tp_rank, int tp_size, @@ -64,8 +63,8 @@ class GGUFBlockQuantization : public BaseQuantization { float alpha, const std::string &stem) const override; - // 融合 Linear 的唯一入口：各 shard 的 ggml type id 只能由自己的 stem 查出来 - // （实测 q/k/v 同类型的 full-attn 层数 0/16），而组 stem 做不到。见 §7.2 子步骤 0。 + // Fused Linear entry point. Every shard resolves its own ggml type from + // its checkpoint stem; a shared group stem is insufficient. infinicore::Tensor forward( const ParamsMap ¶ms, const infinicore::Tensor &input, @@ -74,27 +73,23 @@ class GGUFBlockQuantization : public BaseQuantization { const std::string &stem, const std::vector &shard_stems) const override; - // GGUF 的融合 Linear 不在 base_linear 里走这条路（各 shard 本来就是独立 buffer， - // 没有可 narrow 的父 buffer），这里只做「shard -> .」的名字映射， - // 字节一个不动，供 BaseLinear::split_params 的既有调用点安全通过。 + // Map shard names to .. Fused GGUF shards already have + // independent buffers, so this method does not slice or modify data. std::vector split_params( const std::unordered_map ¶ms, const std::vector &splits, int narrow_dim, int tp_rank, int tp_size, int tp_num_heads) const override; - // 不改写任何字节：blob 的语义就是「GGUF 原始字节」，一旦被 post-process - // 加工就失去与 llama.cpp 逐 block 对拍的能力（方案 §4 的基准）。 + // Validate raw block buffers without modifying their bytes. std::shared_ptr process_weights_after_loading( ParamsMap ¶ms, const infinicore::Device &device, int split_dim = -1) const override; - // ---- 供自检 / 诊断使用 ---- - // stem -> ggml type id 或 DENSE_BF16。命中 0 个或 2 个候选都抛错：宁可拒启， - // 也不能静默走稠密路径（能加载、显存暴涨、结果错）。 - // matched_key 非空时额外给出表里真正命中的那条键（已裁前缀的形态），报错里用它 - // 才能 grep 到；旧形态产物的 blob 键是归一成 `.weight` 的，不能拿 stem 拼凑。 + // Resolve a stem to a ggml type id or DENSE_BF16. Missing and ambiguous + // matches fail closed instead of silently selecting a dense path. + // matched_key receives the actual normalized table key when requested. int64_t resolve(const std::string &stem, std::string *matched_key = nullptr) const; size_t row_bytes(size_t in_features, int64_t type_id) const; bool has_group(const std::string &group_stem) const; @@ -103,8 +98,8 @@ class GGUFBlockQuantization : public BaseQuantization { static bool is_known_type(int64_t type_id); private: - // type_id = 本权重在类型表里的 ggml type id（稠密条目为 DENSE_BF16），阶段 3 的 - // kernel 分发靠它；table_key = 命中的表键（报错里给的名字必须能 grep 到）。 + // type_id selects the execution path; table_key identifies the exact + // checkpoint entry in diagnostics. infinicore::Tensor forward_shard( const std::string &suffix, const infinicore::Tensor &weight, @@ -113,33 +108,28 @@ class GGUFBlockQuantization : public BaseQuantization { int64_t type_id, const std::string &table_key) const; - // 运行时激活 V 头置换（out_proj 一类「权重列需要重排」的条目）。 - // 为什么必须在运行时做：conversion/qwen.py:607-609 导出 GGUF 时把 ssm_out 的**列** - // 从 grouped 换成了 tiled，而 GDN kernel 的 v 头序是 grouped（InfiniCore - // chunk_gated_delta_rule/cuda/kernel.cuh:112 `key_head_idx = value_head_idx / - // value_heads_per_key_head`）；blob 的块沿 in 维切（Q4_K/Q5_K/Q6_K block_size=256）， - // 打包期置换列 = 跨块重排 = 要重量化，做不到 ⇒ 只能把激活置换过去。 - // 规则不在这里硬编码，由打包器从映射表派生写进 - // config.json:quantization_config.activation_vperm（见 scripts/gguf_mapping.py）。 + // Runtime value-head permutation for weights whose columns were exported + // in tiled order while the GDN kernel produces grouped activations. The + // converter records rules in quantization_config.activation_vperm because + // permuting quantized columns would require requantization. struct ActVPerm { - std::string suffix; // 尾匹配用，含结尾 '.'，例如 "linear_attn.out_proj." - size_t n_k; // key 头数 - size_t r; // 每个 key 头带几个 value 头 + std::string suffix; // Suffix including the trailing '.', for example "linear_attn.out_proj.". + size_t n_k; // Number of key heads. + size_t r; // Value heads per key head. size_t hd; // value head_dim }; - // stem 命中哪条规则（没有则 nullptr）。按后缀匹配，因为层号在 C++ 侧不可信。 + // Match by suffix because layer indices are not part of this local contract. const ActVPerm *vperm_rule(const std::string &stem) const; - // [..., n_k*r*hd]（grouped）-> [..., r*n_k*hd]（tiled），纯视图 + 一次 contiguous + // Convert [..., n_k*r*hd] grouped order to [..., r*n_k*hd] tiled order. static infinicore::Tensor gather_grouped_to_tiled( const ActVPerm &rule, const infinicore::Tensor &input, const std::string &name); std::string describe(const std::string &stem) const; - // 类型表条目。name = 它在 config.json:ggml_types 里的**原始键**（未裁前缀）， - // 只能靠它把报错写成可在产物里 grep 的名字：裁过前缀的键在旧形态产物里连后缀 - // 都不一样（blob 被归一成了 .weight），拿 stem 拼凑出来的名字两边都 grep 不到。 + // Keep the original config key for diagnostics even though lookups use a + // normalized key with key_prefix removed. struct TypeEntry { int64_t id; std::string name; @@ -147,8 +137,8 @@ class GGUFBlockQuantization : public BaseQuantization { std::unordered_map types_; std::string key_prefix_; - std::vector vperm_; // 见 ActVPerm（空 = config 声明本产物无需置换） - // 命中统计（get_param_layout 是 const，所以 mutable） + std::vector vperm_; // Empty when the converted model needs no permutation. + // Mutable because layout queries are logically const. mutable size_t n_blob_ = 0; mutable size_t n_dense_ = 0; mutable size_t n_group_ = 0; diff --git a/csrc/models/qwen3_5/qwen3_5_attention.cpp b/csrc/models/qwen3_5/qwen3_5_attention.cpp index 0d3f24a19..6371f6f65 100644 --- a/csrc/models/qwen3_5/qwen3_5_attention.cpp +++ b/csrc/models/qwen3_5/qwen3_5_attention.cpp @@ -65,8 +65,9 @@ Qwen35Attention::Qwen35Attention(std::shared_ptr auto quantization_method = model_config->get_quantization_method(); auto register_fn = [this](const std::string &n, infinicore::nn::Parameter p) { this->register_parameter(n, std::move(p)); }; - // checkpoint 里的本层路径（已去掉 config.json:quantization_config.key_prefix）。 - // 只给按张量名查类型的量化方案（GGUF）用，其他方案不传就是空串，行为不变。 + // Checkpoint path for this layer after removing + // `quantization_config.key_prefix`. It is used only by quantization + // schemes, such as GGUF, that resolve types by tensor name. const std::string prefix = "layers." + std::to_string(layer_idx_) + ".self_attn"; qkv_proj_ = std::make_shared( hidden_size_, head_dim_, total_num_heads, total_num_kv_heads, diff --git a/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp b/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp index cfbe9ab4e..810678f8f 100644 --- a/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp +++ b/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp @@ -52,7 +52,8 @@ Qwen35DecoderLayer::Qwen35DecoderLayer(std::shared_ptr layer_types = model_config->get>("layer_types"); diff --git a/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.cpp b/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.cpp index 87183f81c..1b1cdfb25 100644 --- a/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.cpp +++ b/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.cpp @@ -36,11 +36,13 @@ Qwen35FusedQKVLinear::Qwen35FusedQKVLinear(size_t hidden_size, num_kv_head_(num_kv_head), register_fn_(register_fn) { if (this->sharded_) { - // GGUF：三段各有自己的 checkpoint 张量（q_proj 含交错的 gate），不存在可 narrow - // 的融合 buffer；stem 必须带结尾的 '.'，与类型表里的张量名逐字相等。 + // GGUF stores three separate checkpoint tensors. The Q projection + // contains the interleaved gate, and there is no fused buffer to + // narrow. Each stem must end in `.` and match the type-table key. if (prefix.empty()) { throw std::runtime_error( - "Qwen35FusedQKVLinear: GGUF 量化必须传 layer prefix（形如 layers.3.self_attn）"); + "Qwen35FusedQKVLinear requires a layer prefix such as " + "`layers.3.self_attn` for GGUF quantization."); } shard_specs_ = { {q_name, q_proj_out_size_, prefix + "." + q_name + "."}, @@ -61,8 +63,8 @@ void Qwen35FusedQKVLinear::register_fused_params() { if (!register_fn_) { return; } - // GGUF 分支：逐 shard 一次 GEMM，forward() 里拼回同一根 [B,S,q|k|v]， - // 所以下面 forward_split() 的 narrow 偏移量不用改。 + // The GGUF path runs one GEMM per shard and concatenates the results into + // `[B, S, Q|K|V]`, so the existing `forward_split()` offsets remain valid. auto params = this->sharded_ ? this->init_fused_shards(shard_specs_) : this->split_params(split_infos_, tp_rank_, tp_size_, num_kv_head_); @@ -90,8 +92,9 @@ Qwen35FusedQKVLinear::forward_split(infinicore::Tensor &input) { void Qwen35FusedQKVLinear::process_weights_after_loading() { BaseLinear::process_weights_after_loading(); - // sharded_（GGUF）时 split_infos_ 为空：那些 shard 参数就是加载目标， - // 重新分配会把已读进来的块字节丢掉 + // `split_infos_` is empty for sharded layouts such as GGUF because the + // shard parameters are the load targets. Reallocation would discard the + // block bytes that were already loaded. if (register_fn_ && !split_infos_.empty()) { register_fused_params(); } diff --git a/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.hpp b/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.hpp index 60596534d..d0f756c30 100644 --- a/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.hpp +++ b/csrc/models/qwen3_5/qwen3_5_fused_qkv_linear.hpp @@ -46,8 +46,8 @@ class Qwen35FusedQKVLinear : public infinilm::layers::linear::ColumnParallelLine size_t num_kv_head_; infinilm::layers::linear::RegisterParamFn register_fn_; std::vector split_infos_; - // GGUF：q|gate / k / v 三段的 ggml 类型互不相同（方案 §6.0 纠正 1）， - // 每段各自一块 buffer，与 split_infos_ 二选一 + // GGUF Q|gate, K, and V shards may use different GGML types, so each + // shard owns a separate buffer instead of using `split_infos_`. std::vector shard_specs_; void register_fused_params(); diff --git a/csrc/models/qwen3_next/qwen3_next_gated_deltanet.cpp b/csrc/models/qwen3_next/qwen3_next_gated_deltanet.cpp index 0a1d46f32..eac3ae58f 100644 --- a/csrc/models/qwen3_next/qwen3_next_gated_deltanet.cpp +++ b/csrc/models/qwen3_next/qwen3_next_gated_deltanet.cpp @@ -152,7 +152,8 @@ Qwen3NextGatedDeltaNet::Qwen3NextGatedDeltaNet(std::shared_ptrget_quantization_method(); auto register_fn = [this](const std::string &n, infinicore::nn::Parameter p) { this->register_parameter(n, std::move(p)); }; - // 本模块在 checkpoint 里的路径（同 Qwen35Attention，只给 GGUF 类查表方案用） + // Checkpoint path for this module. This is used only by quantization + // schemes, such as GGUF, that resolve layouts by tensor name. const std::string prefix = "layers." + std::to_string(layer_idx_) + ".linear_attn"; in_proj_qkv_ = std::make_shared( hidden_size, linear_key_head_dim, linear_key_head_dim, linear_value_head_dim, linear_num_key_heads, linear_num_key_heads, linear_num_value_heads, diff --git a/python/infinilm/modeling_utils.py b/python/infinilm/modeling_utils.py index 8647cecab..5e879ac6a 100644 --- a/python/infinilm/modeling_utils.py +++ b/python/infinilm/modeling_utils.py @@ -799,10 +799,10 @@ def _remap_qwen3_5(state_dict, config): key_dim = llm_config["linear_key_head_dim"] * llm_config["linear_num_key_heads"] block_size = 128 # FP8 block size for scale splitting - # 路线 B 的 GGUF 产物：llama.cpp 转换脚本写 GGUF 时已经做过 `norm.weight + 1` - # （conversion/qwen.py:393-394，除 linear_attn.norm 之外全部加），打包器按「不得再 - # 加一次」原样搬运（scripts/gguf_mapping.py 顶部第 13 行）。这里再加一次就变成 - # 2+w；融合 QKV 也已在打包期拆成 in_proj_q/k/v，不能按老键名再拆一遍。 + # The llama.cpp converter has already applied `norm.weight + 1` to GGUF + # Route B checkpoints, except for `linear_attn.norm`. The packer preserves + # those values, so applying the remap again would produce `2 + weight`. + # Fused QKV tensors are also split into `in_proj_q/k/v` while packaging. gguf = (config.get("quantization_config") or {}).get("quant_method", "") == "gguf" norm_weight_suffixes = ( diff --git a/scripts/gguf_mapping.py b/scripts/gguf_mapping.py index dfa54d513..53acdfa0f 100644 --- a/scripts/gguf_mapping.py +++ b/scripts/gguf_mapping.py @@ -1,26 +1,11 @@ #!/usr/bin/env python3 -""" -InfiniLM 路线 B —— GGUF -> InfiniLM 权重映射表（打包器与审计脚本的单一事实源）。 - -所有条目都由阶段 0 审计实测得出，不是推测： - * InfiniLM 侧参数键/shape/取向：scripts/gguf_routeb_probe_params.py 在 CPU 上 - 构造 mini qwen3_5 引擎导出的 state_dict（121 键），取向为 [out, in]，与 GGUF - blob 的行主序一致 -> 打包不需要转置。 - * GGUF 侧键与 shape：scripts/gguf_routeb_audit.py D 节对 866 个张量实测。 - * transform 依据 llama.cpp conversion/qwen.py（行号为该文件实测）： - 388 A_log -> -exp(A_log) （故打包需反解 log(-x)） - 391 dt_bias -> 改名 dt_proj.bias，值不变 （故 ssm_dt.bias 原样用） - 394 *.norm.weight -> w + 1（linear_attn.norm 除外） - （故打包不得再加 1） - 571-605 _LinearAttentionVReorderBase.modify_tensors：需逆重排的集合是 - in_proj_qkv(仅 V 行段) / in_proj_z / in_proj_a / in_proj_b(head_dim=1) / - A_log / dt_bias(head_dim=1) / conv1d(仅 V 通道段)； - 609 out_proj 重排的是 **列(in 维)** —— 本方案改为运行时对激活做 head gather， - 权重保持逐字节不变，故此处不标 transform。 - `linear_attn.norm`(=ssm_norm) 不在重排列表内，确认无需重排。 - 615 注释 "Qwen3.5 always applies interleaved MRoPE" -> mrope_interleaved 必为 True - 619 写入 GGUF 的 mrope_section 是 4 元素 [11,11,10,0]，而 InfiniLM - qwen3_5_attention.cpp:65 硬性要求 3 元素 -> 打包时去掉尾 0。 +"""Single source of truth for GGUF-to-InfiniLM tensor mapping. + +InfiniLM weights use [out, in] orientation, matching GGUF packed row order, so +conversion does not transpose weight data. Transform semantics follow +llama.cpp's Qwen conversion: recover A_log from -exp(A_log), preserve baked +normalization offsets, restore grouped value-head rows, and describe runtime +activation permutation for column-reordered output projections. """ from __future__ import annotations @@ -29,25 +14,23 @@ from dataclasses import dataclass # --------------------------------------------------------------------------- -# transform 语义 +# Transform semantics # --------------------------------------------------------------------------- -T_NONE = "" # 原样搬运（blob 逐字节 / dense 仅换 dtype） -T_VROWS = "vrows" # 沿 out 维按 V 头分块整块搬回 grouped 序（blob 可行级置换） -T_VELEM = "velem" # 1-D、每头 1 个元素：T_VROWS 的 head_dim=1 退化形式（同一实现） -T_ALOG = "alog" # A_log = log(-ssm_a)，再置换 -T_DENSE = "dense" # 反量化为 BF16（框架不支持该参数走量化路径） - -# V 头置换在 dim0 上的作用域： -# all = 整个 dim0 都是 value 头（in_proj_v / in_proj_z / in_proj_a / in_proj_b / A_log / dt_bias） -# v_tail = 只有末尾 value_dim 个元素是 value 段（conv1d 的 [q|k|v] 通道拼接） +T_NONE = "" # Preserve packed bytes, or only cast dense values. +T_VROWS = "vrows" # Restore value-head row blocks to grouped order. +T_VELEM = "velem" # One scalar per head; implemented by the same row transform. +T_ALOG = "alog" # Recover A_log = log(-ssm_a), then permute. +T_DENSE = "dense" # Dequantize to BF16 for parameters without a packed path. + +# Value-head permutation scope along dimension 0: +# all = the entire dimension contains value heads +# v_tail = only the trailing value_dim segment contains value heads VPERM_ALL, VPERM_TAIL = "all", "v_tail" -# blob 参数在产物 / 框架里的名字后缀。阶段 2 的 get_param_layout 必须用同名， -# 否则 load_state_dict(strict=False) 会把 400 个权重静默丢掉。 +# Checkpoint suffix shared with the C++ packed-weight layout. BLOB_SUFFIX = "weight_bytes" -# 两者共用一份置换实现：每头几个元素由条目 shape 推出来（见 gguf_transforms.vperm_head_dim）， -# 48 个元素 / 48 个头 = 1 ⇒ 自然就是逐元素置换，不需要第二套代码。 +# Both forms share one implementation; elements per head are derived from shape. VPERM_TRANSFORMS = (T_VROWS, T_VELEM) @@ -56,27 +39,23 @@ def needs_vperm(e: "Entry") -> bool: # --------------------------------------------------------------------------- -# GGML 类型名（数值见 ggml.h；本文件不依赖 gguf-py，避免脚本互相 import 拉环境） -# 实测本 GGUF 出现的类型集合由 scripts/gguf_routeb_shape_contract.py 断言。 +# GGML type ids from ggml.h. Keep this module independent of gguf-py. # --------------------------------------------------------------------------- F32, Q8_0, Q4_K, Q5_K, Q6_K = "F32", "Q8_0", "Q4_K", "Q5_K", "Q6_K" IQ4_NL, IQ4_XS = "IQ4_NL", "IQ4_XS" -# 阶段 3 v1 必须实现的 block 类型（实测本文件主模型只出现这 4 种）。 -# Q4_K 不跟 IQ4 一起延期：它与 Q5_K 同族（144B/256，只差第 5 bit 平面）， -# Q5_K 本来就要写，多支持 Q4_K 接近零成本，而它占 2 个张量 45 MiB。 +# Packed block types supported by the runtime kernel. NATIVE_BLOB_TYPES = (Q8_0, Q4_K, Q5_K, Q6_K) -# v1 稠密化的 i-quants（执行方案 §2.4 决策：量小、需查码表，上原生 kernel 推到阶段 6）。 -# 实测共 5 个张量 0.23 GiB，稠密化后占 0.82 GiB，代价 +0.60 GiB（预算仍 ≤ 24 GiB）。 +# I-quants converted to dense BF16 until native kernels are available. V1_IQUANT_DENSE = (IQ4_NL, IQ4_XS) DENSE_SRC_TYPES = (F32, Q8_0, Q6_K) + V1_IQUANT_DENSE def apply_v1_exceptions(plan, gguf_types, enabled=True): - """把 v1 不打算写 kernel 的 i-quants 条目就地转为稠密化。 + """Convert I-quant entries without native kernels to dense BF16 in place. - gguf_types: {张量名: GGML 类型名}，由调用方从真文件采集（本模块不依赖 gguf-py）。 - 阶段 6 上了 IQ4 码本后传 enabled=False 即可全部回到逐字节路径。 + ``gguf_types`` maps tensor names to GGML type names collected by the caller. + Set ``enabled=False`` when native IQ4 kernels become available. """ n = 0 if enabled: @@ -85,30 +64,29 @@ def apply_v1_exceptions(plan, gguf_types, enabled=True): e.blob = False e.transforms = e.transforms + (T_DENSE,) e.note = ( - e.note + "；" if e.note else "" - ) + "v1 稠密化例外（源 %s），阶段 6 上原生 kernel 后取消" % gguf_types[ - e.gguf - ] + (e.note + "；" if e.note else "") + + "dense fallback for source %s; remove when a native kernel is available" + % gguf_types[e.gguf] + ) n += 1 return n @dataclass class Entry: - """一条 GGUF 张量 -> 一个 InfiniLM 参数。""" + """Map one GGUF tensor to one InfiniLM parameter.""" - infinilm: str # InfiniLM 参数名（含 model.language_model. 前缀） - gguf: str # GGUF 张量名 - shape: tuple # InfiniLM 期望 shape（未 TP 切分的全量），取向 [out, in] - blob: bool # True = 保留 GGUF 原始 block 字节（U8 [out, row_bytes]） + infinilm: str # InfiniLM parameter name including model.language_model prefix. + gguf: str # GGUF tensor name. + shape: tuple # Full InfiniLM shape before TP, in [out, in] orientation. + blob: bool # Preserve original blocks as U8 [out, row_bytes]. transforms: tuple = () - types: tuple = () # 允许的 GGUF 源类型名；() = 不限（由 contract 脚本报告实际值） - slices: tuple = () # 沿 out 维占用的 [start, end)；共用同一 gguf 的条目做覆盖校验 - vperm: str = VPERM_ALL # T_VROWS 的作用域（仅当 transforms 含 T_VROWS 时有意义） - # 该条目的权重需要置换的是**列（in 维）**而不是行：块量化沿 in 维分块 - # （Q4_K/Q5_K/Q6_K block_size=256），打包期置换列 = 跨块重排 = 必须重量化，做不到。 - # 于是只能在运行时置换喂给它的输入激活，规则由 activation_vperm_rules() 导出进 config。 - # 故意不放进 transforms：那个元组描述的是「打包期对字节做的事」，混进去会污染字节路径。 + types: tuple = () # Allowed source types; empty accepts any reported type. + slices: tuple = () # [start, end) ranges along output dimension. + vperm: str = VPERM_ALL # Scope for T_VROWS. + # Column permutations cross quantization blocks and would require + # requantization. Record them as runtime activation-permutation rules rather + # than conversion-time transforms. act_vperm: bool = False note: str = "" @@ -135,9 +113,9 @@ class Dims: max_position_embeddings: int = 262144 architectures: str = "Qwen3_5ForConditionalGeneration" - # --- 派生量 --- + # Derived dimensions. @property - def q_rows(self) -> int: # q_proj 行数 = heads * head_dim * 2（q 与 gate 每头交错） + def q_rows(self) -> int: # q_proj rows with interleaved query and gate values. return self.n_q_heads * self.head_dim * 2 @property @@ -157,7 +135,7 @@ def value_dim(self) -> int: return self.lin_v_heads * self.lin_v_dim @property - def qkv_rows(self) -> int: # q | k | v 融合（与 GGUF attn_qkv 一致） + def qkv_rows(self) -> int: # Fused q | k | v rows matching GGUF attn_qkv. return self.key_dim * 2 + self.value_dim @property @@ -169,7 +147,7 @@ def v_per_k(self) -> int: return self.lin_v_heads // self.lin_k_heads def layer_types(self) -> list: - """与 C++ prepare_qwen3_5_model_config 的推导完全一致：(i+1) % interval == 0。""" + """Match prepare_qwen3_5_model_config: (i + 1) % interval == 0.""" return [ "full_attention" if (i + 1) % self.interval == 0 else "linear_attention" for i in range(self.n_layers) @@ -213,10 +191,10 @@ def layer_types(self) -> list: def layer_entries(d: Dims, i: int, role: str) -> list: - """第 i 层的映射条目。role ∈ {'linear_attention', 'full_attention'}。 + """Return entries for layer ``i`` and its attention role. - 注：源类型不在表中写死（同一后缀在不同层就用过 Q4_K/Q5_K/Q6_K/Q8_0/IQ4_*）， - 由 contract 脚本从真文件采集后比对 NATIVE_BLOB_TYPES / DENSE_SRC_TYPES。 + Source types are discovered from the input because the same suffix may use + different quantization types across layers. """ L = f"{PREFIX}layers.{i}." G = f"blk.{i}." @@ -228,7 +206,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: (d.hidden,), False, (T_DENSE,), - note="GGUF 已 baked +1，打包不得再加", + note="GGUF already contains the baked +1 offset", ), Entry( L + "post_attention_layernorm.weight", @@ -236,7 +214,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: (d.hidden,), False, (T_DENSE,), - note="同上", + note="GGUF already contains the baked +1 offset", ), Entry( L + "mlp.gate_proj.weight", G + "ffn_gate.weight", (d.ffn, d.hidden), True @@ -254,7 +232,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: (d.q_rows, d.hidden), True, (), - note="行数含 q|gate 每头交错，与 Qwen35FusedQKVLinear 一致", + note="rows contain interleaved query and gate values per head", ), Entry( L + "self_attn.k_proj.weight", @@ -280,7 +258,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: (d.head_dim,), False, (T_DENSE,), - note="GGUF 已 baked +1", + note="GGUF already contains the baked +1 offset", ), Entry( L + "self_attn.k_norm.weight", @@ -288,7 +266,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: (d.head_dim,), False, (T_DENSE,), - note="GGUF 已 baked +1", + note="GGUF already contains the baked +1 offset", ), ] else: @@ -300,7 +278,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: True, (), slices=((0, kd),), - note="attn_qkv 行 [0:kd]", + note="attn_qkv rows [0:kd]", ), Entry( L + "linear_attn.in_proj_k.weight", @@ -309,7 +287,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: True, (), slices=((kd, 2 * kd),), - note="attn_qkv 行 [kd:2kd]", + note="attn_qkv rows [kd:2kd]", ), Entry( L + "linear_attn.in_proj_v.weight", @@ -318,7 +296,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: True, (T_VROWS,), slices=((2 * kd, 2 * kd + vd),), - note="attn_qkv 行 [2kd:]，V 头 tiled->grouped", + note="attn_qkv rows [2kd:] with tiled-to-grouped value heads", ), Entry( L + "linear_attn.in_proj_z.weight", @@ -326,7 +304,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: (vd, d.hidden), True, (T_VROWS,), - note="qwen.py:583 行重排（head_v_dim）", + note="row permutation from qwen.py using head_v_dim", ), Entry( L + "linear_attn.in_proj_a.weight", @@ -334,8 +312,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: (d.lin_v_heads, d.hidden), False, (T_DENSE, T_VROWS), - note="实测源为 Q8_0；框架该参数不走量化路径 -> 稠密化；" - "qwen.py:587 行重排 head_dim=1", + note="dense fallback plus qwen.py row permutation with head_dim=1", ), Entry( L + "linear_attn.in_proj_b.weight", @@ -343,7 +320,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: (d.lin_v_heads, d.hidden), False, (T_DENSE, T_VROWS), - note="同上", + note="dense fallback plus row permutation with head_dim=1", ), Entry( L + "linear_attn.A_log", @@ -351,7 +328,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: (d.lin_v_heads,), False, (T_ALOG, T_VROWS), - note="GGUF 存的是 -exp(A_log)，需 log(-x) 反解", + note="GGUF stores -exp(A_log); recover it with log(-x)", ), Entry( L + "linear_attn.dt_bias", @@ -359,7 +336,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: (d.lin_v_heads,), False, (T_VELEM,), - note="qwen.py:589 逐头置换，值不变", + note="per-head qwen.py permutation without changing values", ), Entry( L + "linear_attn.conv1d.weight", @@ -368,7 +345,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: False, (T_DENSE, T_VROWS), vperm=VPERM_TAIL, - note="GGUF 已 squeeze 成 [C,K] -> 补回中间维；仅末尾 V 通道段重排", + note="restore squeezed [C,K] shape and permute only trailing V channels", ), Entry( L + "linear_attn.norm.weight", @@ -376,7 +353,7 @@ def layer_entries(d: Dims, i: int, role: str) -> list: (d.lin_v_dim,), False, (T_DENSE,), - note="不在 qwen.py 重排列表内；两侧都不加 1", + note="not permuted by qwen.py and no normalization offset", ), Entry( L + "linear_attn.out_proj.weight", @@ -385,15 +362,14 @@ def layer_entries(d: Dims, i: int, role: str) -> list: True, (), act_vperm=True, - note="qwen.py:609 重排的是列(in 维)，blob 不能跨块置换 -> " - "运行时对输入激活做 grouped->tiled（见 config 的 activation_vperm）", + note="column permutation requires grouped-to-tiled runtime activation mapping", ), ] return out def build_plan(d: Dims) -> list: - """全模型映射条目（含顶层）。""" + """Build full-model mapping entries, including root-level tensors.""" entries = [ Entry( PREFIX + "embed_tokens.weight", @@ -401,7 +377,7 @@ def build_plan(d: Dims) -> list: (d.vocab, d.hidden), False, (T_DENSE,), - note="实测 GGUF 为 Q6_K -> 反量化", + note="dequantize embedding to dense BF16", ), ] for i, role in enumerate(d.layer_types()): @@ -413,7 +389,7 @@ def build_plan(d: Dims) -> list: (d.hidden,), False, (T_DENSE,), - note="GGUF 已 baked +1", + note="GGUF already contains the baked +1 offset", ), Entry( "lm_head.weight", @@ -421,18 +397,14 @@ def build_plan(d: Dims) -> list: (d.vocab, d.hidden), False, (T_DENSE,), - note="实测 GGUF 为 Q8_0 -> 反量化", + note="dequantize output head to dense BF16", ), ] return entries def activation_vperm_suffix(e: "Entry") -> str: - """条目对应的 checkpoint stem 后缀（剥掉层号、含结尾 '.'），供 C++ 做尾匹配。 - - C++ 递来的 stem 形如 `layers.7.linear_attn.out_proj.`（挂在前缀下的相对形态， - 见 gguf.cpp 的 key_prefix_ 裁剪），所以这里必须同时去掉 PREFIX 和 `layers..`。 - """ + """Return the layer-independent checkpoint-stem suffix for C++ matching.""" name = re.sub(r"^" + re.escape(PREFIX) + r"layers\.\d+\.", "", e.infinilm) if name.endswith(".weight"): name = name[: -len(".weight")] @@ -440,14 +412,11 @@ def activation_vperm_suffix(e: "Entry") -> str: def activation_vperm_rules(d: "Dims", plan: list) -> list: - """从映射表派生「运行时要对输入激活做的 V 头置换」清单（写进 quantization_config）。 - - 为什么必须有这件事：conversion/qwen.py:607-609 在导出 GGUF 时把 out_proj 的**列**从 - grouped 换成了 tiled；而 GDN kernel 期望/产出的 v 头序是 grouped（InfiniCore - chunk_gated_delta_rule/cuda/kernel.cuh:112 `key_head_idx = value_head_idx / - value_heads_per_key_head`）。打包期我们把 in_proj_v 等**行**向条目逆置换回 grouped， - 但 out_proj 的列置换不掉（跨块），所以只能把激活置换过去：grouped -> tiled。 - 规则在这里派生、C++ 只照单执行，两边不各抄一份（§6.0 纠正 2 的同一原则）。 + """Derive runtime value-head activation permutations for quantization_config. + + llama.cpp exports selected output-projection columns in tiled order, while + the GDN kernel produces grouped activations. Packed columns cannot be moved + across blocks without requantization, so the runtime permutes activations. """ n_k, r, hd = d.lin_k_heads, d.v_per_k, d.lin_v_dim rules, seen = [], set() @@ -457,8 +426,8 @@ def activation_vperm_rules(d: "Dims", plan: list) -> list: in_dim = int(e.shape[1]) if in_dim != n_k * r * hd: raise ValueError( - "%s: 条目 in 维 %d != num_k_heads*num_v_per_k*head_dim = %d，" - "无法按头分块置换" % (e.infinilm, in_dim, n_k * r * hd) + "%s: input dimension %d != num_k_heads*num_v_per_k*head_dim %d; " + "cannot permute complete heads" % (e.infinilm, in_dim, n_k * r * hd) ) suffix = activation_vperm_suffix(e) if suffix in seen: @@ -474,60 +443,42 @@ def expected_keys(d: Dims) -> list: return [e.infinilm for e in build_plan(d)] -# 打包期需丢弃的 GGUF 张量。实测 blk.64 共 15 个张量 = -# 11 个普通 full-attention 层张量（attn_norm/attn_q/attn_k/attn_v/attn_q_norm/ -# attn_k_norm/attn_output/post_attention_norm/ffn_gate/ffn_up/ffn_down） -# + 4 个 nextn.*（eh_proj/enorm/hnorm/shared_head_norm），共 0.327 GiB。 -# 推论：主模型的 full-attn 层是 16 个（blk.3,7,...,63），而带 attn_q 的 block -# 共 17 个 —— 多出的那个就是 MTP block，不要误当成第 17 个注意力层。 +# GGUF tensor prefixes excluded from the main model, including the MTP block. DROP_PREFIXES = ("blk.64.",) MTP_BLOCK = 64 def compress(shape: tuple) -> tuple: - """去掉长度为 1 的维。GGUF 写入时对 conv1d 做过 squeeze（qwen.py:393）， - 比对形状时需同样处理，否则 (C,1,K) vs (C,K) 会误报。""" + """Remove singleton dimensions when comparing squeezed GGUF tensors.""" return tuple(int(x) for x in shape if int(x) != 1) # --------------------------------------------------------------------------- -# 派生工具：产物参数名、type 表键、行字节、config.json -# —— 打包器 / 阶段 2 C++ / 契约脚本都必须走这里，不得各抄一份 +# Derived checkpoint names, type-table keys, packed row sizes, and config data. # --------------------------------------------------------------------------- def ckpt_name(e: "Entry") -> str: - """写进 safetensors（以及框架 state_dict）的参数名。""" + """Return the safetensors and framework state-dict parameter name.""" if e.blob and e.infinilm.endswith(".weight"): return e.infinilm[: -len(".weight")] + "." + BLOB_SUFFIX return e.infinilm def type_table_key(name: str) -> str: - """config.json:quantization_config.ggml_types 的键 = checkpoint 张量名原文。 - - 曾经用过“去 model.language_model. 前缀 + .weight_bytes 归一回 .weight”的压缩写法， - 但那要求阶段 2 的 C++ 把同一套规则逐字符重实现一遍，拼错不会报错只会静默走 - 稠密路径（能加载、显存暴涨、结果错）。现在 key 就是 safetensors 里的张量名， - C++ 只递 stem（如 `layers.0.mlp.gate_proj.`）再探 `stem+"weight_bytes"` / - `stem+"weight"`，命中 0 个或 2 个都抛错；挂载前缀由 quantization_config.key_prefix - 告知，不在 C++ 里硬编码。详见执行方案 §6.0 纠正 2。 - """ + """Return the exact checkpoint name used as the ggml_types table key.""" return name def row_bytes(n_in: int, block_size: int, type_size: int) -> int: - """blob 一行的字节数。本模块不依赖 gguf-py，故 (block_size, type_size) 由调用方给。""" + """Return bytes per packed row using caller-provided GGML block metadata.""" if n_in % block_size: - raise ValueError("in=%d 不能被块大小 %d 整除" % (n_in, block_size)) + raise ValueError( + "input size %d is not divisible by block size %d" % (n_in, block_size) + ) return n_in // block_size * type_size def make_text_config(d: "Dims") -> dict: - """config.json 的 text_config 段。 - - 键名集合以 scripts/gguf_routeb_probe_params.py::CFG 为准 —— 那份 config 已被 - InferEngine 实测接受（121 键全对齐），不要再引入未验证的键（如 architectures / - layer_types：layer_types 由 qwen3_5_for_causal_lm.cpp:72-87 从 interval 推导）。 - """ + """Build the Qwen3.5 text_config consumed by InfiniLM.""" return { "model_type": "qwen3_5_text", "hidden_size": d.hidden, @@ -550,22 +501,19 @@ def make_text_config(d: "Dims") -> dict: "rope_type": "mrope", "rope_theta": d.rope_theta, "partial_rotary_factor": d.partial_rotary_factor, - # 必须 3 元素：qwen3_5_attention.cpp:65 硬校验；且 - # position_id_axes = len(mrope_section)（qwen3_5_for_causal_lm.cpp:52-64） + # InfiniLM requires three MRoPE sections. "mrope_section": list(d.mrope_section), - # 无默认值，缺键即抛；conversion/qwen.py:615 注释已确认恒为交错 + # Qwen3.5 always uses interleaved MRoPE. "mrope_interleaved": True, }, } def make_root_config(d: "Dims", ggml_types: dict, act_vperm: list = None) -> dict: - """config.json 根段。 + """Build root config with top-level quantization metadata. - ★ quantization_config 必须在**顶层**：ModelConfig ctor 只读 - `config_json["quantization_config"]`（model_config.cpp:5/16），而 - prepare_qwen3_5_model_config 的 text_config -> root 合并发生在 ctor **之后**； - 写在 text_config 里会得到 null => NoneQuantization 的静默降级。 + ModelConfig reads quantization_config before merging text_config, so placing + it inside text_config would silently select NoneQuantization. """ return { "model_type": "qwen3_5", @@ -577,12 +525,10 @@ def make_root_config(d: "Dims", ggml_types: dict, act_vperm: list = None) -> dic "text_config": make_text_config(d), "quantization_config": { "quant_method": "gguf", - # C++ 侧的表 key = 本表 key 去掉这段前缀（层级以下的模块不知道自己挂在 - # model. 下）；由打包器写入，不在 C++ 里硬编码 + # Nested C++ modules remove this prefix before type-table lookup. "key_prefix": PREFIX, "ggml_types": ggml_types, - # 运行时激活 V 头置换规则（见 activation_vperm_rules）。空列表 = 该产物没有 - # 列向置换的条目；C++ 缺这个键会直接拒启，避免旧 config 静默跑出错位权重。 + # Empty means that no runtime activation permutation is required. "activation_vperm": act_vperm or [], }, } diff --git a/scripts/gguf_routeb_audit.py b/scripts/gguf_routeb_audit.py deleted file mode 100644 index c82d050c6..000000000 --- a/scripts/gguf_routeb_audit.py +++ /dev/null @@ -1,728 +0,0 @@ -#!/usr/bin/env python3 -""" -InfiniLM 路线 B —— 阶段 0 风险清零审计（执行方案 §4） - -检查项： - A. 容器/字节布局：GGUF 原始字节按 [out, row_bytes] 重解释 + 自研 block 解码 - 是否与 gguf-py 权威实现**逐比特相等**（Q8_0 / Q4_K / Q5_K / Q6_K） - B. 对齐事实：块起始与行 stride 的真实对齐度（写 kernel 前的硬约束） - C. V 头重排：grouped<->tiled 正向/逆向置换是否自等（执行方案 §2.7） - D. 命名/形状契约：GGUF 实际张量集合是否与打包器的映射表完全一致 - E. 元数据：rope / ssm / 层类型等 config.json 依据 - -用法： - python3 scripts/gguf_routeb_audit.py \ - [--gguf /home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf] -退出码 0 表示全部 PASS。 -""" - -from __future__ import annotations - -import argparse -import collections -import os -import sys - -import numpy as np - -_LLAMA_CPP = os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp") -sys.path.insert(0, os.path.join(_LLAMA_CPP, "gguf-py")) -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -import gguf.quants as gq # noqa: E402 -from gguf import GGUFReader # noqa: E402 -from gguf.constants import ( # noqa: E402 - GGML_QUANT_SIZES, -) -from gguf.constants import ( # noqa: E402 - GGMLQuantizationType as QType, -) - -QK_K = 256 - -PASSED: list[str] = [] -FAILED: list[str] = [] - - -def check(name: str, ok: bool, detail: str = "") -> bool: - (PASSED if ok else FAILED).append(name) - print(f" [{'PASS' if ok else 'FAIL'}] {name}" + (f" {detail}" if detail else "")) - return ok - - -# --------------------------------------------------------------------------- -# 自研 block 解码：完全按 ggml 内存布局手写（将来 1:1 移植进 ggml_blocks.h） -# 输入统一为 uint8 blob，形状 [n_rows, row_bytes]；输出 float32 [n_rows, n_cols] -# --------------------------------------------------------------------------- - - -def _rows_to_blocks(blob: np.ndarray, type_size: int) -> np.ndarray: - """[n_rows, row_bytes] -> [n_blocks, type_size]，块沿 in 连续、按 out 行排列。""" - assert blob.dtype == np.uint8 - n_rows, row_bytes = blob.shape - assert row_bytes % type_size == 0, ( - f"row_bytes={row_bytes} 不是 type_size={type_size} 的整数倍" - ) - return blob.reshape(-1, type_size) - - -def _f16(col: np.ndarray) -> np.ndarray: - return col.view(np.float16).astype(np.float32) - - -def decode_q8_0(blob: np.ndarray, n_cols: int) -> np.ndarray: - """块 = d(f16,2B) + qs(int8,32B)，共 34B / 32 元素。""" - ts = 34 - b = _rows_to_blocks(blob, ts) - d = _f16(b[:, :2]) # [nb,1] - x = b[:, 2:ts].view(np.int8).astype(np.float32) # [nb,32] - return (d * x).reshape(blob.shape[0], n_cols) - - -def _k_scale_min(scales: np.ndarray) -> tuple[np.ndarray, np.ndarray]: - """Q4_K/Q5_K 的 12 字节 -> 8 组 (sc, min)，6+2 bit 交错打包。""" - n = scales.shape[0] - s = scales.reshape((n, 3, 4)) - d, m, m_d = np.split(s, 3, axis=-2) - sc = np.concatenate([d & 0x3F, (m_d & 0x0F) | ((d >> 2) & 0x30)], axis=-1) - mn = np.concatenate([m & 0x3F, (m_d >> 4) | ((m >> 2) & 0x30)], axis=-1) - return sc.reshape((n, 8)), mn.reshape((n, 8)) - - -# --- 索引表：这就是后续 CUDA 实现的 ggml_blocks.h 布局规范 ------------------- -# Q4_K / Q5_K：8 个子块 x 32 元素（不是 16x16！），子块 g 内偏移 o： -# qs 字节 = qs_base + (g // 2) * 32 + o，nibble 位移 = (g % 2) * 4 -_e = np.arange(QK_K) -_g = _e // 32 -_o = _e % 32 -K_QS_BYTE = (_g // 2) * 32 + _o -K_QS_SHIFT = (_g % 2) * 4 -K_SCALE_IDX = _g # 每 32 元素一组 scale/min - -# Q5_K 的第 5 bit：qh 字节 = o，位 = g -K5_QH_BYTE = _o -K5_QH_BIT = _g - -# Q6_K：256 元素，6 bit = 低 4(nibble) + 高 2 -# 低 4 bit：字节 = (h // 2) * 64 + r，位移 = (h % 2) * 4（h = e // 64, r = e % 64） -# 高 2 bit：字节 = (g // 4) * 32 + o，位移 = (g % 4) * 2 -# scale 索引 = e // 16（16 个子块 x 16 元素） -_h = _e // 64 -_r = _e % 64 -Q6_LO_BYTE = (_h // 2) * 64 + _r -Q6_LO_SHIFT = (_h % 2) * 4 -Q6_HI_BYTE = (_g // 4) * 32 + _o -Q6_HI_SHIFT = (_g % 4) * 2 -Q6_SCALE_IDX = _e // 16 - - -def decode_q4_k(blob: np.ndarray, n_cols: int) -> np.ndarray: - """块 = d(2) dmin(2) scales(12) qs(128) = 144B / 256 元素。""" - ts = 144 - b = _rows_to_blocks(blob, ts) - d = _f16(b[:, 0:2]) - dmin = _f16(b[:, 2:4]) - sc, mn = _k_scale_min(b[:, 4:16]) - qs = b[:, 16:ts] - q = ( - ((qs[:, K_QS_BYTE] >> K_QS_SHIFT.astype(np.uint8)) & np.uint8(0x0F)) - .reshape(b.shape[0], 8, 32) - .astype(np.float32) - ) - d_eff = (d * sc.astype(np.float32)).reshape(b.shape[0], 8, 1) - m_eff = (dmin * mn.astype(np.float32)).reshape(b.shape[0], 8, 1) - return (d_eff * q - m_eff).reshape(blob.shape[0], n_cols) - - -def decode_q5_k(blob: np.ndarray, n_cols: int) -> np.ndarray: - """块 = d(2) dmin(2) scales(12) qh(32) qs(128) = 176B / 256 元素。""" - ts = 176 - b = _rows_to_blocks(blob, ts) - d = _f16(b[:, 0:2]) - dmin = _f16(b[:, 2:4]) - sc, mn = _k_scale_min(b[:, 4:16]) - qh = b[:, 16:48] - qs = b[:, 48:ts] - n = b.shape[0] - lo = (qs[:, K_QS_BYTE] >> K_QS_SHIFT.astype(np.uint8)) & np.uint8(0x0F) - hi = (qh[:, K5_QH_BYTE] >> K5_QH_BIT.astype(np.uint8)) & np.uint8(0x01) - q = (lo | (hi << np.uint8(4))).reshape(n, 8, 32).astype(np.float32) - d_eff = (d * sc.astype(np.float32)).reshape(n, 8, 1) - m_eff = (dmin * mn.astype(np.float32)).reshape(n, 8, 1) - return (d_eff * q - m_eff).reshape(blob.shape[0], n_cols) - - -def decode_q6_k(blob: np.ndarray, n_cols: int) -> np.ndarray: - """块 = ql(128) qh(64) scales(16,int8) d(2) = 210B / 256 元素。""" - ts = 210 - b = _rows_to_blocks(blob, ts) - n = b.shape[0] - ql = b[:, :128] - qh = b[:, 128:192] - sc = b[:, 192:208].view(np.int8).astype(np.float32) - d = _f16(b[:, 208:210]) - lo = (ql[:, Q6_LO_BYTE] >> Q6_LO_SHIFT.astype(np.uint8)) & np.uint8(0x0F) - hi = (qh[:, Q6_HI_BYTE] >> Q6_HI_SHIFT.astype(np.uint8)) & np.uint8(0x03) - q = ( - ((lo | (hi << np.uint8(4))).astype(np.int16) - 32) - .reshape(n, 16, 16) - .astype(np.float32) - ) - step = (d * sc).reshape(n, 16, 1) - return (step * q).reshape(blob.shape[0], n_cols) - - -DECODERS = { - QType.Q8_0: decode_q8_0, - QType.Q4_K: decode_q4_k, - QType.Q5_K: decode_q5_k, - QType.Q6_K: decode_q6_k, -} - - -# --------------------------------------------------------------------------- -# A. 容器 / 字节布局 / block 位运算 -# --------------------------------------------------------------------------- - - -def pick_samples(tensors: dict[str, object], per_type: int = 3) -> list: - """每种量化类型最多挑 per_type 个（按 (in,out) 形状去重），只解部分行以省时。""" - by_type = collections.defaultdict(list) - for name, t in tensors.items(): - qt = QType(int(t.tensor_type)) - if qt in DECODERS and name.startswith("blk.") and ".nextn." not in name: - by_type[qt].append(t) - out = [] - for qt, lst in sorted(by_type.items(), key=lambda kv: int(kv[0])): - seen = set() - picked = 0 - for t in sorted(lst, key=lambda x: x.name): - key = (int(t.shape[0]), int(t.shape[1])) - if key in seen: - continue - seen.add(key) - out.append(t) - picked += 1 - if picked >= per_type: - break - return out - - -def section_a(reader) -> None: - print("\n== A. 容器与 block 位运算（逐比特）==") - tensors = {t.name: t for t in reader.tensors} - samples = pick_samples(tensors) - assert samples, "未取到任何样本" - all_ok = True - for t in samples: - qt = QType(int(t.tensor_type)) - bs, ts = GGML_QUANT_SIZES[int(qt)] - n_in, n_out = int(t.shape[0]), int(t.shape[1]) # GGML: ne[0]=in, ne[1]=out - row_bytes = n_in // bs * ts - blob = np.ascontiguousarray(t.data) # 解析器已给 [out, row_bytes] - ok_shape = blob.shape == (n_out, row_bytes) - dec = DECODERS[qt] - n_rows = min(64, n_out) # 只解前 n_rows 行，省时 - ours = dec(blob[:n_rows], n_in) - ref_full = gq.dequantize(blob[:n_rows], qt) # 权威实现，输入为字节形状 - ref = np.asarray(ref_full, dtype=np.float32) - exact = ours.shape == ref.shape and np.array_equal(ours, ref) - # 单行独立性：逐行解码必须与整体解码一致（证明行是连续独立单元） - one = dec(blob[7:8], n_in) - indep = np.array_equal(one, ref[7:8]) - all_ok &= check( - f"{t.name} {qt.name} in={n_in} out={n_out} row_bytes={row_bytes}", - ok_shape and exact and indep, - f"bit-exact={exact} row-indep={indep}", - ) - check("A 汇总", all_ok) - # 非量化张量的轴序（打包器是否需要转置的依据） - conv = tensors["blk.0.ssm_conv1d.weight"] - check( - "F32 张量的 data 也是 C 序 [shape[1], shape[0]]（= HF 取向，打包器不转置）", - conv.data.shape == (int(conv.shape[1]), int(conv.shape[0])), - f"ne={list(map(int, conv.shape))} data={conv.data.shape} -> HF [10240,1,4]", - ) - norm = tensors["blk.0.attn_norm.weight"] - check( - "1-D norm 保持 dtype=float32且长度 = hidden", - norm.data.dtype == np.float32 and norm.data.shape == (5120,), - ) - - -# --------------------------------------------------------------------------- -# B. 对齐事实 -# --------------------------------------------------------------------------- - - -def section_b() -> None: - print("\n== B. 对齐事实（kernel 的硬约束）==") - facts = [] - for qt in ( - QType.Q8_0, - QType.Q4_K, - QType.Q5_K, - QType.Q6_K, - QType.IQ4_NL, - QType.IQ4_XS, - ): - bs, ts = GGML_QUANT_SIZES[int(qt)] - align_block = 2 if ts % 2 == 0 else 1 - for n_in in (5120, 6144, 10240, 17408, 248320): - if n_in % bs: - continue - rb = n_in // bs * ts - a = 16 - while a > 1 and rb % a: - a //= 2 - facts.append((qt.name, bs, ts, n_in, rb, a, align_block)) - print( - f" {'type':8s} {'bs':>4s} {'ts':>4s} {'in':>7s} {'row_bytes':>10s} " - f"{'行对齐':>7s} {'块起始对齐':>10s}" - ) - worst_row, worst_block = 16, 2 - for name, bs, ts, n_in, rb, a, ab in facts: - print( - f" {name:8s} {bs:4d} {ts:4d} {n_in:7d} {rb:10d} {str(a) + 'B':>7s} {str(ab) + 'B':>10s}" - ) - worst_row = min(worst_row, a) - worst_block = min(worst_block, ab) - check( - "块起始地址仅保证 2B 对齐（Q6_K=210B / Q8_0=34B 非 4 倍数）", - worst_block == 2, - f"min_block_align={worst_block}B", - ) - check( - "Q6_K 在 in=5120/17408 时行 stride 仅 8B 对齐", - any(f[0] == "Q6_K" and f[5] == 8 for f in facts), - ) - print(" -> 结论：kernel 不得对单块起始地址做 >2B 向量化加载假设；容器不做 pad。") - - -# --------------------------------------------------------------------------- -# C. V 头重排（grouped <-> tiled） -# --------------------------------------------------------------------------- - - -def reorder_v(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: - """与 llama.cpp conversion/qwen.py::_reorder_v_heads 同语义（沿 dim0 的整头置换）。""" - rest = t.shape[1:] - return ( - t.reshape((n_k, n_v_per_k, hd) + rest) - .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) - .reshape((n_k * n_v_per_k * hd,) + rest) - ) - - -def reorder_v_inverse(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: - """逆变换 = 两个轴参数对调后再调用一次。""" - rest = t.shape[1:] - return ( - t.reshape((n_v_per_k, n_k, hd) + rest) - .transpose((1, 0, 2) + tuple(range(3, 3 + len(rest)))) - .reshape((n_k * n_v_per_k * hd,) + rest) - ) - - -def section_c() -> None: - print("\n== C. V 头重排（执行方案 §2.7）==") - n_k, n_v_per_k, hd = 16, 3, 128 # Qwen3.8: 16 key heads, 48 value heads - n_v = n_k * n_v_per_k - rng = np.random.default_rng(0) - - grouped = rng.standard_normal((n_v * hd, 7)).astype(np.float32) - tiled = reorder_v(grouped, n_k, n_v_per_k, hd) - back = reorder_v_inverse(tiled, n_k, n_v_per_k, hd) - check("grouped -> tiled -> grouped 自等", np.array_equal(grouped, back)) - check( - "reorder_v 是整头搬运（每个 head 的 hd 行连续不被打散）", - all( - np.array_equal( - tiled[i * hd : (i + 1) * hd], - grouped[ - ((i % n_k) * n_v_per_k + i // n_k) * hd : ( - (i % n_k) * n_v_per_k + i // n_k - ) - * hd - + hd - ], - ) - for i in range(n_v) - ), - ) - - # 槽位 j（value head 编号）-> 真实 k 头 的两种语义 - k_grouped = [j // n_v_per_k for j in range(n_v)] # InfiniCore kernel 的假设 - k_tiled = [j % n_k for j in range(n_v)] # GGUF(tiled) 的真实归属 - check( - "tiled 序直接喂给 `value_head_idx / value_heads_per_key_head` 会错配 k 头", - k_grouped != k_tiled, - f"错配槽位数={sum(a != b for a, b in zip(k_grouped, k_tiled))}/{n_v}", - ) - - # 逆重排后回到 grouped 语义 - _ = np.repeat(np.arange(n_v), hd) # labels 仅用于形状参考 - check( - "逆变换后槽位归属恢复 grouped 语义", - np.array_equal( - reorder_v_inverse( - np.array( - [k * n_v_per_k + v for v in range(n_v_per_k) for k in range(n_k)] - ), - n_k, - n_v_per_k, - 1, - ), - np.arange(n_v), - ), - "逆变换后 slot i 的 head 编号 = i，kernel 的 k = i // n_v_per_k 成立", - ) - check( - "in_proj_a/b・A_log・dt_bias 的 head_dim=1 退化形式（逐元素置换）同样自等", - np.array_equal( - reorder_v_inverse( - reorder_v(np.arange(n_v), n_k, n_v_per_k, 1), n_k, n_v_per_k, 1 - ), - np.arange(n_v), - ), - ) - check( - "多维情形（如 conv1d 的 [channels, 1, kernel]）仅置换头维、尾部轴不动", - np.array_equal( - reorder_v_inverse( - reorder_v(grouped[:, :1], n_k, n_v_per_k, hd), n_k, n_v_per_k, hd - ), - grouped[:, :1], - ), - ) - # 行置换对量化 blob 是「整块搬运」：以 Q6_K 为例验证字节级可置换性 - bs, ts = GGML_QUANT_SIZES[int(QType.Q6_K)] - row_bytes = 5120 // bs * ts - blob = rng.integers(0, 256, size=(n_v, row_bytes), dtype=np.uint8) - perm = np.arange(n_v)[::-1].copy() - check( - "量化 blob 的行置换 == 字节整行置换（无需重新量化）", - np.array_equal(blob[perm], np.ascontiguousarray(blob)[perm]), - ) - print( - " -> 结论：整行置换可字节级完成；ssm_out 的列(in 维)置换不可，改用运行时激活 gather。" - ) - - -# --------------------------------------------------------------------------- -# D. 命名 / 形状契约 -# --------------------------------------------------------------------------- - - -def gguf_meta(reader, suffix: str): - """元数据键带架构前缀（qwen35.*），允许传短名；contents() 对单元素返回标量，统一成列表。""" - for key in (f"qwen35.{suffix}", f"general.{suffix}", suffix): - if key in reader.fields: - v = reader.fields[key].contents() - return v if isinstance(v, (list, tuple, np.ndarray)) else [v] - raise KeyError(f"GGUF 元数据缺少：{suffix}（qwen35./general. 前缀均未命中）") - - -def section_d(reader) -> None: - print("\n== D. GGUF 张量集合 vs 打包器映射表 ==") - tensors = {t.name: t for t in reader.tensors} - n_layer_gguf = int(gguf_meta(reader, "block_count")[0]) - interval = int(gguf_meta(reader, "full_attention_interval")[0]) - n_main = 64 - full = [i for i in range(n_main) if (i + 1) % interval == 0] - gdn = [i for i in range(n_main) if i not in full] - check( - "主模型层数 64（block_count 含 1 个 MTP 层）", - n_layer_gguf == n_main + 1, - f"block_count={n_layer_gguf}", - ) - check( - "full attention 层 = 3,7,...,63 共 16 层", - len(full) == 16 and full[0] == 3 and full[-1] == 63, - ) - check("GDN 层 48 层", len(gdn) == 48) - - need_full = [ - "attn_norm.weight", - "post_attention_norm.weight", - "attn_q.weight", - "attn_k.weight", - "attn_v.weight", - "attn_output.weight", - "attn_q_norm.weight", - "attn_k_norm.weight", - "ffn_gate.weight", - "ffn_up.weight", - "ffn_down.weight", - ] - need_gdn = [ - "attn_norm.weight", - "post_attention_norm.weight", - "attn_qkv.weight", - "attn_gate.weight", - "ssm_a", - "ssm_alpha.weight", - "ssm_beta.weight", - "ssm_conv1d.weight", - "ssm_dt.bias", - "ssm_norm.weight", - "ssm_out.weight", - "ffn_gate.weight", - "ffn_up.weight", - "ffn_down.weight", - ] - missing = [] - for i in full: - missing += [f"blk.{i}.{r}" for r in need_full if f"blk.{i}.{r}" not in tensors] - for i in gdn: - missing += [f"blk.{i}.{r}" for r in need_gdn if f"blk.{i}.{r}" not in tensors] - check("64 层全部所需张量存在", not missing, f"missing={missing[:6]}") - - shapes = { - "attn_q": (5120, 12288), - "attn_k": (5120, 1024), - "attn_v": (5120, 1024), - "attn_output": (6144, 5120), - "attn_qkv": (5120, 10240), - "attn_gate": (5120, 6144), - "ssm_out": (6144, 5120), - "ffn_gate": (5120, 17408), - "ffn_up": (5120, 17408), - "ffn_down": (17408, 5120), - "ssm_conv1d": (4, 10240), - } - bad = [] - for name, want in shapes.items(): - probe = { - "attn_q": f"blk.{full[0]}.", - "attn_k": f"blk.{full[0]}.", - "attn_v": f"blk.{full[0]}.", - "attn_output": f"blk.{full[0]}.", - "attn_qkv": f"blk.{gdn[0]}.", - "attn_gate": f"blk.{gdn[0]}.", - "ssm_out": f"blk.{gdn[0]}.", - "ssm_conv1d": f"blk.{gdn[0]}.", - "ffn_gate": f"blk.{0}.", - "ffn_up": f"blk.{0}.", - "ffn_down": f"blk.{0}.", - }[name] - t = tensors.get(probe + name + ".weight") - if t is None or (int(t.shape[0]), int(t.shape[1])) != want: - bad.append((name, None if t is None else list(map(int, t.shape)))) - check("代表张量 (in,out) 与映射表一致", not bad, f"bad={bad}") - - # attn_q 的 12288 = 24 * (256 q + 256 gate) 交错 - n_q, hd_q = 24, 256 - check( - "attn_q 行数 = n_q*head*2（q 与 gate 每头交错）", - shapes["attn_q"][1] == n_q * hd_q * 2, - ) - check( - "Qwen35FusedQKVLinear 期望 out = 12288 + 1024 + 1024 = 14336", - 12288 + 1024 + 1024 == 14336, - ) - check( - "GDN in_proj_qkv 行数 = q2048 + k2048 + v6144 = 10240", - 2048 + 2048 + 6144 == shapes["attn_qkv"][1], - ) - check( - "conv 通道 = 2*head_k*n_k + head_v*n_v = 10240", - 2 * 128 * 16 + 128 * 48 == 10240, - ) - mtp = [n for n in tensors if n.startswith("blk.64.")] - nextn = [n for n in tensors if ".nextn." in n] - check( - "MTP 丢弃规则 = 整块 blk.64.*（不止 .nextn.*，包含完整一层）", - len(mtp) == 15 and len(nextn) == 4, - f"blk.64.*={len(mtp)} 个（其中 .nextn.* 仅 {len(nextn)} 个）", - ) - max_blk = max(int(n.split(".")[1]) for n in tensors if n.startswith("blk.")) - check( - "块号集合 = 0..64（64 主层 + 1 MTP 层，无其它残留）", - max_blk == 64 - and len({int(n.split(".")[1]) for n in tensors if n.startswith("blk.")}) == 65, - f"max_blk={max_blk}", - ) - - -# --------------------------------------------------------------------------- -# F. 打包器字节核算（修正后的 MTP 规则） -# --------------------------------------------------------------------------- - -# 阶段 3 kernel 需直接吃块的格式集合不再在本文件定义：见 gguf_mapping.NATIVE_BLOB_TYPES -# （由 gguf_routeb_shape_contract.py 对真文件校验），避免两处清单漂移。 - - -def section_f(reader) -> None: - print("\n== F. 打包器字节核算 ==") - GiB = 2**30 - tensors = {t.name: t for t in reader.tensors} - # 核算必须由映射表驱动：之前本脚本自写一套分桶，把 7 个 IQ4 张量当成“反量化”、 - # 把实为 Q8_0 的 ssm_alpha/ssm_beta（框架不能量化它们）当成 blob，两处失真共 - # 高估 0.70 GiB。单一事实源 = gguf_mapping.build_plan(REAL)。 - import gguf_mapping as M - - plan = M.build_plan(M.REAL) - _tn = {int(v.value): str(v.name) for v in QType} - M.apply_v1_exceptions( - plan, {n: _tn[int(t.tensor_type)] for n, t in tensors.items()} - ) - blob_src = {e.gguf for e in plan if e.blob and e.gguf in tensors} - dense_e = [e for e in plan if not e.blob] - bucket = collections.Counter() - cnt = collections.Counter() - for n in blob_src: - bucket[f"U8 blob {QType(int(tensors[n].tensor_type)).name}"] += int( - tensors[n].n_bytes - ) - cnt[f"U8 blob {QType(int(tensors[n].tensor_type)).name}"] += 1 - d_emb = sum( - int(np.prod(e.shape)) * 2 - for e in dense_e - if e.gguf in ("token_embd.weight", "output.weight") - ) - d_other = sum( - int(np.prod(e.shape)) * 2 - for e in dense_e - if e.gguf not in ("token_embd.weight", "output.weight") - ) - bucket["BF16 稠密(emb/lm_head)"] = d_emb - cnt["BF16 稠密(emb/lm_head)"] = 2 - bucket["BF16 稠密(其余稠密化条目)"] = d_other - cnt["BF16 稠密(其余稠密化条目)"] = len(dense_e) - 2 - bucket["丢弃(MTP)"] = sum( - int(t.n_bytes) for n, t in tensors.items() if n.startswith("blk.64.") - ) - cnt["丢弃(MTP)"] = sum(1 for n in tensors if n.startswith("blk.64.")) - - total = sum(v / GiB for k, v in bucket.items() if k != "丢弃(MTP)") - for k in sorted(bucket): - print(f" {k:26s} {bucket[k] / GiB:8.3f} GiB ({cnt[k]:4d} 条目)") - print(f" {'-' * 52}") - print(f" v1 加载后权重合计 {total:8.3f} GiB") - check( - "v1 权重合计 ≤ 24.0 GiB（单卡 32607 MiB 可容纳权重+KV+激活）", - total <= 24.0, - f"total={total:.3f} GiB", - ) - check( - "MTP 丢弃量 < 0.4 GiB（不影响预算）", - bucket["丢弃(MTP)"] / GiB < 0.4, - f"{bucket['丢弃(MTP)'] / GiB:.3f} GiB", - ) - check( - "v1 blob 桶恰好只含阶段 3 实现的 4 种类型", - {k.replace("U8 blob ", "") for k in bucket if k.startswith("U8 blob ")} - == set(M.NATIVE_BLOB_TYPES), - f"{sorted(k for k in bucket if k.startswith('U8'))}", - ) - check( - "blob 条目数与映射表一致", - sum(cnt[k] for k in bucket if k.startswith("U8")) - == len({e.gguf for e in plan if e.blob}), - f"{sum(cnt[k] for k in bucket if k.startswith('U8'))}", - ) - - emb, out = tensors["token_embd.weight"], tensors["output.weight"] - check( - "token_embd / output 也是量化的（Q6_K / Q8_0），v1 必须反量化它们", - int(emb.tensor_type) == int(QType.Q6_K) - and int(out.tensor_type) == int(QType.Q8_0), - f"emb={emb.tensor_type} out={out.tensor_type}", - ) - check( - "emb/output 均为 [hidden, vocab] 且 vocab 与元数据一致", - list(map(int, emb.shape)) == list(map(int, out.shape)) == [5120, 248320] - and len(gguf_meta(reader, "tokenizer.ggml.tokens")) == 248320, - f"shape={list(map(int, emb.shape))}", - ) - print( - " -> 阶段 6 可选项：emb 走 Q6_K 行 gather-dequant、lm_head 走 linear_gguf(Q8_0)，" - f"可再省 ≈ {(d_emb - (emb.n_bytes + out.n_bytes)) / GiB:.2f} GiB" - ) - - -# --------------------------------------------------------------------------- -# E. 元数据 -> config.json -# --------------------------------------------------------------------------- - - -def section_e(reader) -> None: - print("\n== E. 元数据与 config.json 依据 ==") - - def kv(suffix, idx=0): - return gguf_meta(reader, suffix)[idx] - - rope_secs = [int(x) for x in gguf_meta(reader, "rope.dimension_sections")] - dim_cnt = int(kv("rope.dimension_count")) - base = float(kv("rope.freq_base")) - eps = float(kv("attention.layer_norm_rms_epsilon")) - head_dim = int(kv("attention.key_length")) - check( - "head_dim = key_length = value_length = 256", - head_dim == int(kv("attention.value_length")) == 256, - ) - check( - "partial rotary: dimension_count=64, head_dim=256 -> factor 0.25", - dim_cnt == 64 and dim_cnt * 4 == head_dim, - f"dimension_count={dim_cnt}", - ) - check( - "mrope sections [11,11,10,0] 之和 = 32 = dimension_count/2", - sum(rope_secs) == dim_cnt // 2, - f"sections={rope_secs}", - ) - check("rope_theta = 1e7", base == 1e7, f"base={base}") - check( - "mtp 层数声明为 1（与 block_count=65 = 64+1 一致）", - int(kv("nextn_predict_layers")) == 1, - ) - n_k = int(kv("ssm.group_count")) - inner = int(kv("ssm.inner_size")) - st = int(kv("ssm.state_size")) - dt = int(kv("ssm.time_step_rank")) - check( - "ssm: inner 6144 / group 16 / state 128 / time_step_rank 48 / conv 4", - (inner, n_k, st, dt, int(kv("ssm.conv_kernel"))) == (6144, 16, 128, 48, 4), - ) - check( - "value heads = inner/state = 48 = time_step_rank（两路推导一致）", - inner // st == dt == 48, - f"inner/state={inner // st} time_step_rank={dt}", - ) - check("num_k_heads * state = 2048 = q/k 段长度", n_k * st == 2048) - vocab = len(gguf_meta(reader, "tokenizer.ggml.tokens")) - print( - f" arch={gguf_meta(reader, 'architecture')[0]!r} " - f"name={gguf_meta(reader, 'name')[0]!r} rms_eps={eps:g} " - f"ctx={int(kv('context_length'))} vocab={vocab} " - f"heads={int(kv('attention.head_count'))}/{int(kv('attention.head_count_kv'))} " - f"hidden={int(kv('embedding_length'))} ffn={int(kv('feed_forward_length'))}" - ) - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument( - "--gguf", default="/home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf" - ) - args = ap.parse_args() - print(f"审计对象：{args.gguf}\n大小：{os.path.getsize(args.gguf):,} bytes") - reader = GGUFReader(args.gguf) - section_a(reader) - section_b() - section_c() - section_d(reader) - section_e(reader) - section_f(reader) - print(f"\n===== 结果：PASS {len(PASSED)} / FAIL {len(FAILED)} =====") - if FAILED: - for f in FAILED: - print(" FAIL:", f) - return 1 - print("阶段 0 全部通过，可进入阶段 1（打包器）。") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/gguf_routeb_blocks_probe.cpp b/scripts/gguf_routeb_blocks_probe.cpp deleted file mode 100644 index 797a8f2fe..000000000 --- a/scripts/gguf_routeb_blocks_probe.cpp +++ /dev/null @@ -1,81 +0,0 @@ -// Host-side driver for ggml_blocks.h, used by scripts/gguf_routeb_blocks_ref.py. -// -// g++ -O2 -std=c++17 -I /src/infiniop/ops/linear_gguf \ -// gguf_routeb_blocks_probe.cpp -o blocks_probe_host -// -// blocks_probe_host -// -// This file is a test harness, not part of any library target: it exists so the -// decoders can be checked against numpy / gguf-py block by block before the -// linear_gguf kernels exist. -#include -#include -#include - -#include "ggml_blocks.h" - -int main(int argc, char **argv) { - if (argc != 6) { - std::fprintf(stderr, - "usage: %s \n", - argv[0]); - return 2; - } - const int32_t type = std::atoi(argv[1]); - const int64_t n_blocks = std::atoll(argv[2]); - const int32_t bytes = ggml_blocks::block_bytes(type); - const int32_t elems = ggml_blocks::block_elems(type); - if (bytes < 0 || elems < 0) { - std::fprintf(stderr, "probe: ggml type %d has no decoder here\n", type); - return 3; - } - if (n_blocks <= 0) { - std::fprintf(stderr, "probe: n_blocks must be positive\n"); - return 2; - } - - FILE *in = std::fopen(argv[3], "rb"); - if (!in) { - std::fprintf(stderr, "probe: cannot open %s\n", argv[3]); - return 4; - } - const size_t want = (size_t)n_blocks * bytes; - std::vector buf(want); - if (std::fread(buf.data(), 1, want, in) != want) { - std::fprintf(stderr, "probe: short read on %s (wanted %zu)\n", argv[3], want); - std::fclose(in); - return 4; - } - std::fclose(in); - - std::vector f32((size_t)n_blocks * elems); - std::vector bf16((size_t)n_blocks * elems); - if (!ggml_blocks::decode_blocks(type, buf.data(), n_blocks, f32.data())) { - std::fprintf(stderr, "probe: decode_blocks failed\n"); - return 3; - } - if (!ggml_blocks::decode_blocks_bf16(type, buf.data(), n_blocks, - bf16.data())) { - std::fprintf(stderr, "probe: decode_blocks_bf16 failed\n"); - return 3; - } - - FILE *o1 = std::fopen(argv[4], "wb"); - FILE *o2 = std::fopen(argv[5], "wb"); - if (!o1 || !o2) { - std::fprintf(stderr, "probe: cannot open output files\n"); - return 4; - } - const size_t n_f32 = f32.size() * sizeof(float); - const size_t n_bf16 = bf16.size() * sizeof(uint16_t); - const bool ok = std::fwrite(f32.data(), 1, n_f32, o1) == n_f32 && std::fwrite(bf16.data(), 1, n_bf16, o2) == n_bf16; - std::fclose(o1); - std::fclose(o2); - if (!ok) { - std::fprintf(stderr, "probe: short write\n"); - return 4; - } - std::printf("probe host type=%d n_blocks=%lld elems=%d ok\n", type, - (long long)n_blocks, elems); - return 0; -} diff --git a/scripts/gguf_routeb_blocks_probe.cu b/scripts/gguf_routeb_blocks_probe.cu deleted file mode 100644 index e78f74841..000000000 --- a/scripts/gguf_routeb_blocks_probe.cu +++ /dev/null @@ -1,128 +0,0 @@ -// Device-side driver for ggml_blocks.h, used by scripts/gguf_routeb_blocks_ref.py. -// -// nvcc -O2 -std=c++17 -I /src/infiniop/ops/linear_gguf \ -// gguf_routeb_blocks_probe.cu -o blocks_probe_cuda -// -// blocks_probe_cuda -// -// Same job as gguf_routeb_blocks_probe.cpp, but every block is decoded by one -// thread through the very same ggml_blocks.h entry points, which is what proves -// the header is device-safe (no host-only call, no unaligned struct punning) and -// that the host and device results are bit-identical. -#include -#include -#include - -#include "ggml_blocks.h" - -__global__ void decode_f32_kernel(int32_t type, const uint8_t *blk, int64_t n_blocks, - int32_t bytes, int32_t elems, float *out) { - const int64_t i = (int64_t)blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n_blocks) { - return; - } - ggml_blocks::decode_blocks(type, blk + (int64_t)i * bytes, 1, out + i * elems); -} - -__global__ void decode_bf16_kernel(int32_t type, const uint8_t *blk, int64_t n_blocks, - int32_t bytes, int32_t elems, uint16_t *out) { - const int64_t i = (int64_t)blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n_blocks) { - return; - } - ggml_blocks::decode_blocks_bf16(type, blk + (int64_t)i * bytes, 1, - out + i * elems); -} - -#define CUDA_CHECK(call) \ - do { \ - cudaError_t err__ = (call); \ - if (err__ != cudaSuccess) { \ - std::fprintf(stderr, "probe cuda: %s failed: %s\n", #call, \ - cudaGetErrorString(err__)); \ - return 5; \ - } \ - } while (0) - -int main(int argc, char **argv) { - if (argc != 6) { - std::fprintf(stderr, - "usage: %s \n", - argv[0]); - return 2; - } - const int32_t type = std::atoi(argv[1]); - const int64_t n_blocks = std::atoll(argv[2]); - const int32_t bytes = ggml_blocks::block_bytes(type); - const int32_t elems = ggml_blocks::block_elems(type); - if (bytes < 0 || elems < 0) { - std::fprintf(stderr, "probe cuda: ggml type %d has no decoder here\n", type); - return 3; - } - if (n_blocks <= 0) { - std::fprintf(stderr, "probe cuda: n_blocks must be positive\n"); - return 2; - } - - FILE *in = std::fopen(argv[3], "rb"); - if (!in) { - std::fprintf(stderr, "probe cuda: cannot open %s\n", argv[3]); - return 4; - } - const size_t want = (size_t)n_blocks * bytes; - std::vector buf(want); - const size_t got = std::fread(buf.data(), 1, want, in); - std::fclose(in); - if (got != want) { - std::fprintf(stderr, "probe cuda: short read on %s (wanted %zu, got %zu)\n", argv[3], - want, got); - return 4; - } - - uint8_t *d_blk = nullptr; - float *d_f32 = nullptr; - uint16_t *d_bf16 = nullptr; - CUDA_CHECK(cudaMalloc(&d_blk, want)); - CUDA_CHECK(cudaMalloc(&d_f32, (size_t)n_blocks * elems * sizeof(float))); - CUDA_CHECK(cudaMalloc(&d_bf16, (size_t)n_blocks * elems * sizeof(uint16_t))); - CUDA_CHECK(cudaMemcpy(d_blk, buf.data(), want, cudaMemcpyHostToDevice)); - - const int threads = 256; - const int64_t blocks_grid = (n_blocks + threads - 1) / threads; - decode_f32_kernel<<<(unsigned)blocks_grid, threads>>>(type, d_blk, n_blocks, bytes, elems, - d_f32); - CUDA_CHECK(cudaGetLastError()); - decode_bf16_kernel<<<(unsigned)blocks_grid, threads>>>(type, d_blk, n_blocks, bytes, elems, - d_bf16); - CUDA_CHECK(cudaGetLastError()); - CUDA_CHECK(cudaDeviceSynchronize()); - - std::vector h_f32((size_t)n_blocks * elems); - std::vector h_bf16((size_t)n_blocks * elems); - CUDA_CHECK(cudaMemcpy(h_f32.data(), d_f32, h_f32.size() * sizeof(float), - cudaMemcpyDeviceToHost)); - CUDA_CHECK(cudaMemcpy(h_bf16.data(), d_bf16, h_bf16.size() * sizeof(uint16_t), - cudaMemcpyDeviceToHost)); - cudaFree(d_blk); - cudaFree(d_f32); - cudaFree(d_bf16); - - FILE *o1 = std::fopen(argv[4], "wb"); - FILE *o2 = std::fopen(argv[5], "wb"); - if (!o1 || !o2) { - std::fprintf(stderr, "probe cuda: cannot open output files\n"); - return 4; - } - const size_t n_f32 = h_f32.size() * sizeof(float); - const size_t n_bf16 = h_bf16.size() * sizeof(uint16_t); - const bool ok = std::fwrite(h_f32.data(), 1, n_f32, o1) == n_f32 && std::fwrite(h_bf16.data(), 1, n_bf16, o2) == n_bf16; - std::fclose(o1); - std::fclose(o2); - if (!ok) { - std::fprintf(stderr, "probe cuda: short write\n"); - return 4; - } - std::printf("probe cuda type=%d n_blocks=%lld elems=%d ok\n", type, (long long)n_blocks, - elems); - return 0; -} diff --git a/scripts/gguf_routeb_blocks_ref.py b/scripts/gguf_routeb_blocks_ref.py deleted file mode 100644 index aa0d09dd6..000000000 --- a/scripts/gguf_routeb_blocks_ref.py +++ /dev/null @@ -1,744 +0,0 @@ -#!/usr/bin/env python3 -""" -InfiniLM 路线 B —— 阶段 3.1 验收：ggml_blocks.h 的 block 解码位精正确认 - -四方交叉，任何两方不一致都会炸出来： - - A. numpy reference（本文件）：照 llama.cpp `ggml/src/ggml-quants.c` 的 - `dequantize_row_q8_0/q4_K/q5_K/q6_K` 标量语义逐行翻过来，含 - `get_scale_min_k4` 的 6-bit 解包与**浮点结合顺序**（先 d*scale 再碰 quant）。 - B. gguf-py 的 numpy 实现（`gguf.quants.Q8_0/Q4_K/Q5_K/Q6_K.dequantize_blocks`）： - 它是阶段 4「单 block 级 max|Δ| == 0」的基准。它解包 scale 用的是 - reshape/split 另一条路径，与 A 相互独立 —— 两边逐位相同才说明 6-bit - 打包的解读没读歪。 - C. 被测对象 `InfiniCore/src/infiniop/ops/linear_gguf/ggml_blocks.h`，经 - `scripts/gguf_routeb_blocks_probe.cpp` 编出的 host driver 跑真数据。 - D. 同一个头经 `scripts/gguf_routeb_blocks_probe.cu` 编出的 CUDA driver： - 证明这个头确实设备无关（GPU 上编得过、跑得动、与 host 逐位相同）， - 顺带验证 bf16 舍入 `float_to_bf16()` 与 torch 的 `.to(bfloat16)` 一致。 - -样本来自真实打包产物里 `*.weight_bytes` 的 block 字节，再加一批手造边界 block -（次正规 half、int8 scale = -128、scale/min 全 63、全 FF），因为 K-quant 的 -scale 解包最容易在极值上翻车。随机 block 的 half 域被限制为有限值，好让判据 -能要求 100% 逐位相同，而不是退化成"近似"。 - -用法： - /usr/bin/python3 scripts/gguf_routeb_blocks_ref.py \ - [--model-path /home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8] \ - [--blocks 20000] [--no-cuda] [--keep] -退出码 0 = 全部 PASS。 -""" - -from __future__ import annotations - -import argparse -import collections -import json -import os -import re -import struct -import subprocess -import sys - -import numpy as np - -_HERE = os.path.dirname(os.path.abspath(__file__)) -_LLAMA_CPP = os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp") -_INFINICORE = os.environ.get("INFINICORE_DIR", "/home/liuxd/InfiniCore") -sys.path.insert(0, os.path.join(_LLAMA_CPP, "gguf-py")) - -import gguf.quants as gq # noqa: E402 -from gguf.constants import GGML_QUANT_SIZES # noqa: E402 -from gguf.constants import GGMLQuantizationType as Q # noqa: E402 - -HEADER_DIR = os.path.join(_INFINICORE, "src", "infiniop", "ops", "linear_gguf") -PROBE_CPP = os.path.join(_HERE, "gguf_routeb_blocks_probe.cpp") -PROBE_CU = os.path.join(_HERE, "gguf_routeb_blocks_probe.cu") - -TYPES = (8, 12, 13, 14) # 与 pack_report.json 的 blob_type_ids 一致 -QK_K, QK8_0 = 256, 32 -TYPE_SIZE = {t: GGML_QUANT_SIZES[Q(t)][1] for t in TYPES} -BLOCK_SIZE = {t: GGML_QUANT_SIZES[Q(t)][0] for t in TYPES} - -_PASS = 0 -_FAIL = 0 -_SKIP = 0 - - -def check(name, ok, detail=""): - global _PASS, _FAIL - if ok: - _PASS += 1 - print(" PASS %s" % name) - else: - _FAIL += 1 - print(" FAIL %s%s" % (name, ("\n %s" % detail) if detail else "")) - return ok - - -def skip(name, why): - global _SKIP - _SKIP += 1 - print(" SKIP %s（%s）" % (name, why)) - - -# ------------------------------------------------------- A. numpy 参考实现 -def _u16_le(col0, col1): - return col0.astype(np.uint32) | (col1.astype(np.uint32) << np.uint32(8)) - - -def half_to_float(bits): - """IEEE binary16 -> float32，等价于 ggml FP16_TO_FP32 / __half2float。""" - bits = np.asarray(bits, np.uint32) - sign = (bits >> np.uint32(15)) << np.uint32(31) - exp = (bits >> np.uint32(10)) & np.uint32(0x1F) - mant = bits & np.uint32(0x3FF) - out = np.zeros(bits.shape, np.uint32) - zneg = (exp == 0) & (mant == 0) # ±0：符号位必须留住，否则 -0.0 被写成正零 - out[zneg] = sign[zneg] - norm = (exp != 0) & (exp != 31) - out[norm] = ( - sign[norm] - | ((exp[norm] + np.uint32(112)) << np.uint32(23)) - | (mant[norm] << np.uint32(13)) - ) - special = exp == 31 - out[special] = ( - sign[special] | np.uint32(0x7F800000) | (mant[special] << np.uint32(13)) - ) - sub = (exp == 0) & (mant != 0) - if sub.any(): - m = mant[sub].astype(np.int64) - e = np.full(m.shape, -14, np.int64) - for _ in range(11): - need = (m & 0x400) == 0 - if not need.any(): - break - m[need] <<= 1 - e[need] -= 1 - out[sub] = ( - sign[sub].astype(np.int64) | ((e + 127) << 23) | ((m & 0x3FF) << 13) - ).astype(np.uint32) - return out.view(np.float32) - - -def float_to_bf16_bits(f): - """binary32 -> bf16 位模式，round-to-nearest-even，与头里那份同语义。""" - b = np.asarray(f, np.float32).view(np.uint32).astype(np.int64) - exp = (b >> 23) & 0xFF - nan = (exp == 0xFF) & ((b & 0x7FFFFF) != 0) - bias = 0x7FFF + ((b >> 16) & 1) - out = ((b + bias) >> 16).astype(np.uint32) - out[nan] = ((b[nan] >> 16) | 0x0040).astype(np.uint32) - return out.astype(np.uint16) - - -def get_scale_min_k4(scales): - """q4_K / q5_K：12 字节 -> 8 组 (scale, min)，照抄 ggml-quants.c 的分支。""" - nb = scales.shape[0] - d = np.empty((nb, 8), np.uint8) - m = np.empty((nb, 8), np.uint8) - for j in range(8): - if j < 4: - d[:, j] = scales[:, j] & 63 - m[:, j] = scales[:, j + 4] & 63 - else: - d[:, j] = (scales[:, j + 4] & 0xF) | ((scales[:, j - 4] >> 6) << 4) - m[:, j] = (scales[:, j + 4] >> 4) | ((scales[:, j] >> 6) << 4) - return d, m - - -def ref_q8_0(blk): - nb = blk.shape[0] - d = half_to_float(_u16_le(blk[:, 0], blk[:, 1])).reshape(nb, 1) - q = blk[:, 2:34].view(np.int8).astype(np.float32) - return q * d # C: qs[j] * d - - -def ref_q4_K(blk): - nb = blk.shape[0] - d = half_to_float(_u16_le(blk[:, 0], blk[:, 1])) - dmin = half_to_float(_u16_le(blk[:, 2], blk[:, 3])) - sc, m = get_scale_min_k4(blk[:, 4:16]) - d_eff = (d[:, None] * sc.astype(np.float32)).reshape(nb, 8, 1) - m_eff = (dmin[:, None] * m.astype(np.float32)).reshape(nb, 8, 1) - qs = blk[:, 16:144].reshape(nb, 4, 32) - q = np.stack([qs & 0xF, qs >> 4], axis=2).reshape(nb, 8, 32).astype(np.float32) - return (d_eff * q - m_eff).reshape(nb, QK_K) - - -def ref_q5_K(blk): - nb = blk.shape[0] - d = half_to_float(_u16_le(blk[:, 0], blk[:, 1])) - dmin = half_to_float(_u16_le(blk[:, 2], blk[:, 3])) - sc, m = get_scale_min_k4(blk[:, 4:16]) - d_eff = (d[:, None] * sc.astype(np.float32)).reshape(nb, 8, 1) - m_eff = (dmin[:, None] * m.astype(np.float32)).reshape(nb, 8, 1) - qs = blk[:, 48:176].reshape(nb, 4, 32) - qh = blk[:, 16:48][:, None, :] - lo_shift = (2 * np.arange(4)).reshape(4, 1) # u1 = 1 << 2g - hi_shift = lo_shift + 1 # u2 = 2 << 2g - lo = (qs & 0xF) | (((qh >> lo_shift) & 1) << 4).astype(np.uint8) - hi = (qs >> 4) | (((qh >> hi_shift) & 1) << 4).astype(np.uint8) - q = np.stack([lo, hi], axis=2).reshape(nb, 8, 32).astype(np.float32) - return (d_eff * q - m_eff).reshape(nb, QK_K) - - -def ref_q6_K(blk): - nb = blk.shape[0] - d = half_to_float(_u16_le(blk[:, 208], blk[:, 209])) - sc = blk[:, 192:208].view(np.int8).astype(np.float32) - d_eff = d[:, None] * sc # (nb,16) 先 d*sc，同 C 结合顺序 - out = np.empty((nb, QK_K), np.float32) - lane = np.arange(32) - isidx = lane // 16 - for c in (0, 1): - ql = blk[:, 64 * c : 64 * c + 64] - qh = blk[:, 128 + 32 * c : 128 + 32 * c + 32] - base = 128 * c - q1 = ((ql[:, 0:32] & 0xF) | (((qh >> 0) & 3) << 4)).astype(np.int32) - 32 - q2 = ((ql[:, 32:64] & 0xF) | (((qh >> 2) & 3) << 4)).astype(np.int32) - 32 - q3 = ((ql[:, 0:32] >> 4) | (((qh >> 4) & 3) << 4)).astype(np.int32) - 32 - q4 = ((ql[:, 32:64] >> 4) | ((qh >> 6) << 4)).astype(np.int32) - 32 - for part, (q, off) in enumerate(((q1, 0), (q2, 32), (q3, 64), (q4, 96))): - # C 里每处理一个 128 元素段就 `sc += 8`，所以段 1 的 scale 下标整体偏移 8 - s = d_eff[:, 8 * c + isidx + 2 * part] - out[:, base + off : base + off + 32] = s * q.astype(np.float32) - return out - - -REF = {8: ref_q8_0, 12: ref_q4_K, 13: ref_q5_K, 14: ref_q6_K} - - -def gguf_py_dequant(t, blk): - return getattr(gq, Q(t).name).dequantize_blocks(np.ascontiguousarray(blk)) - - -def check_half_decode(): - """参考实现自己的回归护袋：全部 65536 个 half 位模式与 numpy 硬件转换逐位相同。 - - 次正规 / 0 / inf 都在这 65536 个里，负数那一半特别重要（曾经把符号位 - 当成 bit16 丢掉了，只会让带负 d 的 Q6_K block 整批错）。 - NaN 只要求“也是 NaN”，不比 payload。 - """ - h = np.arange(65536, dtype=np.uint16) - truth = h.view(np.float16).astype(np.float32) - mine = half_to_float(h.astype(np.uint32)) - finite = np.isfinite(truth) - ok = np.array_equal(mine[finite].view(np.uint32), truth[finite].view(np.uint32)) - n_nan = int(np.isnan(truth).sum()) - ok_nan = bool( - np.array_equal(np.isnan(mine), np.isnan(truth)) - and np.array_equal(np.isinf(mine) & (mine > 0), np.isinf(truth) & (truth > 0)) - ) - neq = np.flatnonzero(mine.view(np.uint32) != truth.view(np.uint32)) - check( - "numpy 参考的 half_to_float：有限值逐位相同（%d 个）+ NaN 仍为 NaN（%d 个）" - % (int(finite.sum()), n_nan), - ok and ok_nan, - "不同 %d 个，首个 0x%04X：%s vs %s" - % ( - neq.size, - int(h[neq[0]]) if neq.size else 0, - float(mine[neq[0]]) if neq.size else 0, - float(truth[neq[0]]) if neq.size else 0, - ), - ) - - -# ------------------------------------------------- 差异度量（要求逐位相同） -def bitwise_diff(a, b): - """返回 (非有限值个数, 逐位不同的元素数, max|Δ|, 首个差异描述)。""" - fa, fb = np.asarray(a, np.float32), np.asarray(b, np.float32) - bad = ~np.isfinite(fa) | ~np.isfinite(fb) - n_bad = int(bad.sum()) - ok_mask = ~bad - ua = fa[ok_mask].view(np.uint32) - ub = fb[ok_mask].view(np.uint32) - neq = ua != ub - n_diff = int(neq.sum()) - maxabs = float(np.abs(fa[ok_mask] - fb[ok_mask]).max()) if ok_mask.any() else 0.0 - first = "" - if n_diff: - i = int(np.flatnonzero(neq)[0]) - first = "第 %d 个非有限值以外的元素 a=%s(0x%08X) b=%s(0x%08X)" % ( - i, - float(ua[i]), - ua[i], - float(ub[i]), - ub[i], - ) - elif n_bad: - i = int(np.flatnonzero(bad)[0]) - first = "非有限值 a=%s b=%s @flat %d" % ( - fa.reshape(-1)[i], - fb.reshape(-1)[i], - i, - ) - return n_bad, n_diff, maxabs, first - - -# ---------------------------------------------------------- 产物字节取样 -class Artifact: - """打包产物的 blob 张量字节入口。 - - 两个代表产物的类型表键形态不同：mini8 表键 = 张量名（带前缀 + .weight_bytes）； - 全量表键 = `layers.0...in_proj_q.weight`（不带前缀、不带 .weight_bytes，而 - key_prefix 又是 None ⇒ 与 index 张量名零交集）。所以既不能拿表键直接当张量名， - 也不能只剔一个前缀：先只剔尾缀归一，再要求“全等或唯一后缀匹配”， - 匹配不唯一 / 找不到都是打包回归，直接报错而不是猜。 - """ - - def __init__(self, path): - self.path = path - cfg = json.load(open(os.path.join(path, "config.json"))) - qc = cfg["quantization_config"] - table = qc["ggml_types"] - self.prefix = qc.get("key_prefix") or "" - idx = json.load(open(os.path.join(path, "model.safetensors.index.json")))[ - "weight_map" - ] - self.shards = {} - for name in sorted(set(idx.values())): - p = os.path.join(path, name) - with open(p, "rb") as f: - n = struct.unpack(" 表键；要求全等或唯一后缀命中。""" - if tn in self.table_norm: - return tn, "exact" - cands = [k for k in self.table_norm if tn.endswith("." + k)] - if len(cands) == 1: - return cands[0], "suffix" - if len(cands) > 1: - raise RuntimeError( - "张量 %s 在表里后缀命中 %d 个键，歧义：%s" - % (tn, len(cands), sorted(cands)[:5]) - ) - raise RuntimeError("张量 %s 在类型表里找不到对应条目" % tn) - - self.blobs = {} - self.match_form = collections.Counter() - self.matched_table_keys = set() - for tname in sorted(idx): - if not tname.endswith(".weight_bytes"): - continue - key, form = lookup(norm(tname)) - self.match_form[form] += 1 - self.matched_table_keys.add(key) - t = self.table_norm[key] - if t not in TYPES: - raise RuntimeError( - "%s 的 ggml type %d 不在路线 B 支持的 %s 里" - % (tname, t, list(TYPES)) - ) - shard = os.path.join(self.path, idx[tname]) - base, hdr = self.shards[shard] - e = hdr[tname] - if e["dtype"] != "U8" or len(e["shape"]) != 2: - raise RuntimeError( - "%s 应为 U8 [rows, row_bytes]，实为 %s %s" - % (tname, e["dtype"], e["shape"]) - ) - self.blobs[tname] = ( - t, - shard, - base + e["data_offsets"][0], - int(e["shape"][1]), - int(e["shape"][0]), - ) - # 表里说自己是 blob、但产物里没有对应 weight_bytes 张量的条目（应为 0） - self.orphan_table_keys = sorted(set(self.table_norm) - self.matched_table_keys) - - def type_names(self, t): - return sorted(n for n, v in self.blobs.items() if v[0] == t) - - def sample(self, t, want, rng): - """从该类型的真实张量里按整行取 block，返回 (n, type_size) uint8。""" - ts = TYPE_SIZE[t] - names = self.type_names(t) - handles = {} - out, touched = [], set() - per_name = max(1, int(np.ceil(want / max(1, len(names))))) - try: - for name in names: - _t, shard, base, row_bytes, nrows = self.blobs[name] - bpr = row_bytes // ts - if bpr * ts != row_bytes: - raise RuntimeError( - "%s 的 row_bytes=%d 不是 block_size %d 的整数倍" - % (name, row_bytes, ts) - ) - rows_needed = int(np.ceil(per_name / bpr)) - rows = np.sort( - rng.choice(nrows, size=min(rows_needed, nrows), replace=False) - ) - if shard not in handles: - handles[shard] = open(shard, "rb") - fh = handles[shard] - buf = np.empty((rows.size, row_bytes), np.uint8) - for i, r in enumerate(rows): - fh.seek(base + int(r) * row_bytes) - buf[i] = np.frombuffer(fh.read(row_bytes), np.uint8) - flat = buf.reshape(-1, ts) - out.append(flat) - touched.add(name) - if sum(o.shape[0] for o in out) >= want: - break - finally: - for fh in handles.values(): - fh.close() - if not out: - return np.zeros((0, ts), np.uint8), touched - blocks = np.concatenate(out, axis=0)[:want] - return blocks, touched - - -def edge_blocks(t, rng, n_random=2048): - """手造边界 block：全 0、全 FF、次正规 d、scale 极值，再加有限值随机块。""" - ts = TYPE_SIZE[t] - rows = [ - np.zeros(ts, np.uint8), - np.full(ts, 0xFF, np.uint8), - np.full(ts, 0x00, np.uint8), - np.full(ts, 0x01, np.uint8), - ] - b = np.full(ts, 0xFF, np.uint8) - b[:] = 0 - if t == 8: # d = 最小次正规 half，qs 极值 - b[0:2] = [0x01, 0x00] - b[2:] = 0x80 # int8 -128 - rows.append(b.copy()) - b[2:] = 0x7F # int8 +127 - rows.append(b.copy()) - elif t in (12, 13): # d / dmin 次正规，6-bit scale/min 全 63 - b[0:2] = [0x01, 0x00] - b[2:4] = [0xFF, 0x00] # dmin = 1023 * 2^-24 - b[4:16] = 0xFF - rows.append(b.copy()) - b[0:2] = [0xFE, 0x7B] # d = 65534（最大有限 half） - b[2:4] = [0x00, 0x00] - rows.append(b.copy()) - else: # Q6_K：int8 scale = -128 / +127 - b[192:208] = 0x80 - b[208:210] = [0x01, 0x00] - rows.append(b.copy()) - b[192:208] = 0x7F - b[208:210] = [0xFE, 0x7B] - rows.append(b.copy()) - # 有限值随机块：随机字节 + 把 half 域换成非 inf/nan 的随机值 - for _ in range(n_random): - r = rng.integers(0, 256, ts, dtype=np.uint8) - for off in _half_offsets(t): - h = int(rng.integers(0, 0x7BFF + 1)) # exp != 0x1F - r[off], r[off + 1] = h & 0xFF, (h >> 8) & 0xFF - rows.append(r) - return np.stack(rows) - - -def _half_offsets(t): - if t == 8: - return (0,) - if t in (12, 13): - return (0, 2) - return (208,) - - -def half_sweep_blocks(t, rng): - """让每个 half 字段各自遍历全 65536 个位模式，其余字节随机。 - - 真实数据不一定会把次正规、负零、inf 这些 d 值送到解码路径上，全域扫描才能 - 钉住头里那份 half_to_float()（包括上面 numpy 参考刚犯过的符号位错误）。 - 返回 (blocks, 每个字段的扫描块起始行) 。 - """ - ts, offs = TYPE_SIZE[t], _half_offsets(t) - per = 65536 - blocks = rng.integers(0, 256, (per * len(offs), ts), dtype=np.uint8) - pats = np.arange(per, dtype=np.uint16) - for i, off in enumerate(offs): - sl = slice(i * per, (i + 1) * per) - # 其他 half 字段固定为 1.0，避免 NaN/inf 乘上本字段后把结果全糊成 NaN - for o2 in offs: - if o2 != off: - blocks[sl, o2] = 0x00 - blocks[sl, o2 + 1] = 0x3C - blocks[sl, off] = (pats & 0xFF).astype(np.uint8) - blocks[sl, off + 1] = (pats >> np.uint16(8)).astype(np.uint8) - return blocks, [(off, i * per) for i, off in enumerate(offs)] - - -# ------------------------------------------------------------ probe 编译/调用 -def build_probe(src, out, compiler, extra=()): - cmd = [compiler, "-O2", "-std=c++17", "-I", HEADER_DIR, src, "-o", out] + list( - extra - ) - p = subprocess.run(cmd, capture_output=True, text=True) - if p.returncode != 0: - raise RuntimeError( - "编译失败：%s\n%s" % (" ".join(cmd), (p.stderr or p.stdout)[-4000:]) - ) - return out - - -def run_probe(binary, t, blocks, workdir, tag): - bs = BLOCK_SIZE[t] - inbin = os.path.join(workdir, "%s_t%d.in" % (tag, t)) - f32bin = os.path.join(workdir, "%s_t%d.f32" % (tag, t)) - bf16bin = os.path.join(workdir, "%s_t%d.bf16" % (tag, t)) - np.ascontiguousarray(blocks).tofile(inbin) - p = subprocess.run( - [binary, str(t), str(blocks.shape[0]), inbin, f32bin, bf16bin], - capture_output=True, - text=True, - ) - if p.returncode != 0: - raise RuntimeError( - "%s 失败（type=%d, rc=%d）：%s" - % ( - os.path.basename(binary), - t, - p.returncode, - (p.stderr or p.stdout).strip()[-2000:], - ) - ) - f32 = np.fromfile(f32bin, np.float32).reshape(-1, bs) - bf16 = np.fromfile(bf16bin, np.uint16).reshape(-1, bs) - m = re.search(r"elems=(\d+)", p.stdout) - return f32, bf16, (int(m.group(1)) if m else -1) - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument( - "--model-path", default="/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8" - ) - ap.add_argument( - "--blocks", type=int, default=20000, help="每种类型取多少真实 block" - ) - ap.add_argument("--workdir", default="/home/liuxd/tmp_routeb/blocks31") - ap.add_argument("--cxx", default=os.environ.get("CXX", "g++")) - ap.add_argument("--nvcc", default=os.environ.get("CUDACXX", "nvcc")) - ap.add_argument("--no-cuda", action="store_true") - ap.add_argument("--seed", type=int, default=20260829) - args = ap.parse_args() - - rng = np.random.default_rng(args.seed) - os.makedirs(args.workdir, exist_ok=True) - print( - "产物：%s\n头文件：%s\n临时目录：%s\n每类型真实 block 目标：%d" - % ( - args.model_path, - os.path.join(HEADER_DIR, "ggml_blocks.h"), - args.workdir, - args.blocks, - ) - ) - - print("\n[0] 参考实现自检") - check_half_decode() - - art = Artifact(args.model_path) - n_blob_total = len(art.blobs) - print( - "产物 blob 张量 %d 个（key_prefix=%r），按类型：%s" - % (n_blob_total, art.prefix, {t: len(art.type_names(t)) for t in TYPES}) - ) - check( - "类型表 blob 条目与产物 weight_bytes 张量双向对平（表 %d / 张量 %d，孤儿 %d，" - "匹配形态 %s）" - % ( - art.n_table_blob, - n_blob_total, - len(art.orphan_table_keys), - dict(art.match_form), - ), - art.n_table_blob == n_blob_total and not art.orphan_table_keys, - "孤儿键：%s" % art.orphan_table_keys[:5], - ) - - print("\n[1] 编译 probe driver") - host_bin = build_probe( - PROBE_CPP, os.path.join(args.workdir, "blocks_probe_host"), args.cxx - ) - print(" host driver ok：%s" % host_bin) - dev_bin = None - if args.no_cuda: - skip("cuda driver 编译", "--no-cuda") - else: - try: - dev_bin = build_probe( - PROBE_CU, - os.path.join(args.workdir, "blocks_probe_cuda"), - args.nvcc, - extra=["-x", "cu"], - ) - print(" cuda driver ok：%s" % dev_bin) - except Exception as e: - print(" ! %s" % e) - dev_bin = None - - print("\n[2] 逐类型对拍（真实 block + 边界 block）") - for t in TYPES: - name = Q(t).name - want = args.blocks - blocks, touched = art.sample(t, want, rng) - if not check( - "%s 采到 %d 个真实 block（目标 %d，覆盖 %d 个张量）" - % (name, blocks.shape[0], want, len(touched)), - blocks.shape[0] >= min(want, 100), - ): - continue - - ref = REF[t](np.ascontiguousarray(blocks)) - py = gguf_py_dequant(t, blocks) - n_bad, n_diff, maxabs, first = bitwise_diff(ref, py) - check( - "%s numpy 参考 vs gguf-py（%d block 逐位相同）" % (name, blocks.shape[0]), - n_diff == 0 and n_bad == 0, - "差异 %d/%d 元素，非有限 %d，max|Δ|=%.3g，首个：%s" - % (n_diff, ref.size, n_bad, maxabs, first), - ) - - try: - h_f32, h_bf16, elems = run_probe(host_bin, t, blocks, args.workdir, "host") - except Exception as e: - check("%s host probe 运行" % name, False, str(e)) - continue - check( - "%s 头的 block_elems 与 GGML_QUANT_SIZES 一致（%d == %d）" - % (name, elems, BLOCK_SIZE[t]), - elems == BLOCK_SIZE[t], - ) - n_bad, n_diff, maxabs, first = bitwise_diff(h_f32, ref) - check( - "%s 头(host) fp32 vs numpy 参考（%d 元素逐位相同）" % (name, h_f32.size), - n_diff == 0 and n_bad == 0, - "差异 %d，首个：%s" % (n_diff, first), - ) - - want_bf16 = float_to_bf16_bits(ref) - same_own = np.array_equal(h_bf16, want_bf16) - check( - "%s 头(host) bf16 vs numpy RNE 舍入" % name, - same_own, - "首个差异 %s" % (np.flatnonzero(h_bf16 != want_bf16)[:5],), - ) - try: - import torch - - tv = ( - torch.from_numpy(np.ascontiguousarray(ref)) - .to(torch.bfloat16) - .view(torch.uint16) - .numpy() - ) - check( - "%s 头(host) bf16 vs torch .to(bfloat16)" % name, - np.array_equal(h_bf16, tv), - ) - except Exception as e: - skip("%s bf16 vs torch" % name, str(e).splitlines()[0][:80]) - - if dev_bin is not None: - try: - d_f32, d_bf16, _ = run_probe(dev_bin, t, blocks, args.workdir, "cuda") - except Exception as e: - check("%s cuda probe 运行" % name, False, str(e)) - else: - check( - "%s 头(cuda) fp32 vs 头(host) 逐位相同" % name, - np.array_equal(d_f32.view(np.uint32), h_f32.view(np.uint32)), - ) - check( - "%s 头(cuda) bf16 vs 头(host) 逐位相同" % name, - np.array_equal(d_bf16, h_bf16), - ) - - eb = edge_blocks(t, rng) - eref = REF[t](np.ascontiguousarray(eb)) - epy = gguf_py_dequant(t, eb) - _, n_diff_e, maxabs_e, first_e = bitwise_diff(eref, epy) - n_bad_e = int((~np.isfinite(eref) | ~np.isfinite(epy)).sum()) - h_e, _, _ = run_probe(host_bin, t, eb, args.workdir, "host_edge") - _, n_diff_h, maxabs_h, first_h = bitwise_diff(h_e, eref) - check( - "%s 边界块（%d 个）numpy vs gguf-py 逐位相同" % (name, eb.shape[0]), - n_diff_e == 0, - "差异 %d，非有限 %d，max|Δ|=%.3g，首个：%s" - % (n_diff_e, n_bad_e, maxabs_e, first_e), - ) - check( - "%s 边界块（%d 个）头(host) vs numpy 逐位相同" % (name, eb.shape[0]), - n_diff_h == 0, - "差异 %d，max|Δ|=%.3g，首个：%s" % (n_diff_h, maxabs_h, first_h), - ) - - print("\n[3] half 字段全域扫描（每个字段 65536 个位模式）") - for t in TYPES: - name = Q(t).name - sb, _marks = half_sweep_blocks(t, rng) - with np.errstate(all="ignore"): # 扫描里故意喂 inf/nan half，告警与判据无关 - sref = REF[t](np.ascontiguousarray(sb)) - sh, _, _ = run_probe(host_bin, t, sb, args.workdir, "host_sweep") - n_bad, n_diff, maxabs, first = bitwise_diff(sh, sref) - check( - "%s 头(host) vs numpy 参考：half 全域扫描 %d block 逐位相同" - % (name, sb.shape[0]), - n_diff == 0, - "差异 %d/%d 元素，非有限 %d（inf/nan 乘出的正常现象），max|Δ|=%.3g，首个：%s" - % (n_diff, sh.size, n_bad, maxabs, first), - ) - - print("\n[4] 不支持的类型必须被头拒绝") - inbin = os.path.join(args.workdir, "reject.in") - np.zeros(TYPE_SIZE[8], np.uint8).tofile(inbin) - p = subprocess.run( - [ - host_bin, - "10", - "1", - inbin, - os.path.join(args.workdir, "reject.f32"), - os.path.join(args.workdir, "reject.bf16"), - ], - capture_output=True, - text=True, - ) - check( - "头对 ggml type 10（TQ1_0，非本头范围）返回拒绝", - p.returncode == 3 and "no decoder" in (p.stderr + p.stdout), - "rc=%d stderr=%s" % (p.returncode, (p.stderr or p.stdout).strip()[-200:]), - ) - - print("\n== 结果：%d PASS / %d FAIL / %d SKIP ==" % (_PASS, _FAIL, _SKIP)) - print("临时目录：%s" % args.workdir) - return 0 if _FAIL == 0 else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/gguf_routeb_compare.py b/scripts/gguf_routeb_compare.py deleted file mode 100755 index 8ba49e950..000000000 --- a/scripts/gguf_routeb_compare.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 -"""Compare deterministic llama.cpp and InfiniLM token results.""" - -from __future__ import annotations - -import argparse -import json -import os -import sys - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--llama", required=True) - ap.add_argument("--infinilm", required=True) - ap.add_argument("--out", required=True) - ap.add_argument( - "--case-ids", - help="Optional comma-separated case IDs for focused comparisons", - ) - args = ap.parse_args() - - with open(args.llama, encoding="utf-8") as f: - llama = json.load(f) - with open(args.infinilm, encoding="utf-8") as f: - infini = json.load(f) - lmap = {x["id"]: x for x in llama["cases"]} - imap = {x["id"]: x for x in infini["cases"]} - if args.case_ids: - case_ids = [x.strip() for x in args.case_ids.split(",") if x.strip()] - missing = [x for x in case_ids if x not in lmap or x not in imap] - if missing: - raise ValueError("requested case IDs missing from one side: %s" % missing) - lmap = {x: lmap[x] for x in case_ids} - imap = {x: imap[x] for x in case_ids} - elif set(lmap) != set(imap): - raise ValueError( - "case sets differ: llama-only=%s infini-only=%s" - % (sorted(set(lmap) - set(imap)), sorted(set(imap) - set(lmap))) - ) - - cases = [] - exact = 0 - matched = total = 0 - for case_id in lmap: - left = lmap[case_id] - right = imap[case_id] - if left["input_ids"] != right["input_ids"]: - raise ValueError("input ids differ for %s" % case_id) - lt = left["runs"][0]["tokens"] - rt = right["runs"][0]["tokens"] - first_difference = next( - (i for i, (a, b) in enumerate(zip(lt, rt)) if a != b), None - ) - if first_difference is None and len(lt) != len(rt): - first_difference = min(len(lt), len(rt)) - is_exact = lt == rt - exact += int(is_exact) - same = sum(a == b for a, b in zip(lt, rt)) - matched += same - total += max(len(lt), len(rt)) - cases.append( - { - "id": case_id, - "exact_sequence_match": is_exact, - "matched_tokens": same, - "total_tokens": max(len(lt), len(rt)), - "first_difference": first_difference, - "llama_tokens": lt, - "infinilm_tokens": rt, - "llama_first_top_logprobs": left["runs"][0].get( - "first_token_top_logprobs", [] - ), - } - ) - print( - "%-10s exact=%s first_diff=%s llama=%s infini=%s" - % (case_id, is_exact, first_difference, lt, rt) - ) - - result = { - "cases": cases, - "n_cases": len(cases), - "exact_cases": exact, - "prompt_exact_rate": exact / len(cases) if cases else 0.0, - "matched_tokens": matched, - "total_tokens": total, - "token_match_rate": matched / total if total else 0.0, - "all_exact": exact == len(cases), - } - os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) - with open(args.out, "w", encoding="utf-8") as f: - json.dump(result, f, ensure_ascii=False, indent=2) - print( - "RESULT exact=%d/%d token_match=%d/%d all_exact=%s" - % (exact, len(cases), matched, total, result["all_exact"]) - ) - return 0 if result["all_exact"] else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/gguf_routeb_env.sh b/scripts/gguf_routeb_env.sh deleted file mode 100644 index bf4f6577b..000000000 --- a/scripts/gguf_routeb_env.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# InfiniLM Route B (native GGUF quantization) development environment. -# Source this file after setting CUDA_HOME and optional CUTLASS_ROOT/CUDNN_ROOT. -ROUTEB_SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -ROUTEB_INFINILM_DIR="$(cd -- "${ROUTEB_SCRIPT_DIR}/.." && pwd)" -: "${INFINICORE_DIR:=$(cd -- "${ROUTEB_INFINILM_DIR}/../InfiniCore" && pwd)}" -: "${INFINI_ROOT:=${HOME}/.infini}" - -if [[ -n "${CUDA_HOME:-}" ]]; then - export CUDACXX="${CUDACXX:-${CUDA_HOME}/bin/nvcc}" - export PATH="${CUDA_HOME}/bin:${PATH}" - ROUTEB_CUDA_LIB="${CUDA_HOME}/lib64:" -else - ROUTEB_CUDA_LIB="" -fi - -export INFINICORE_DIR INFINI_ROOT -export PYTHONPATH="${INFINICORE_DIR}/python:${ROUTEB_INFINILM_DIR}/python:${PYTHONPATH:-}" -export LD_LIBRARY_PATH="${INFINICORE_DIR}/python/infinicore/lib:${ROUTEB_INFINILM_DIR}/python/infinilm/lib:${INFINI_ROOT}/lib:${ROUTEB_CUDA_LIB}${LD_LIBRARY_PATH:-}" -export HF_HUB_OFFLINE=1 -export TRANSFORMERS_OFFLINE=1 diff --git a/scripts/gguf_routeb_first_diff.py b/scripts/gguf_routeb_first_diff.py deleted file mode 100755 index 8bd96990c..000000000 --- a/scripts/gguf_routeb_first_diff.py +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env python3 -"""Inspect llama.cpp and InfiniLM logits at the first token divergence.""" - -from __future__ import annotations - -import argparse -import ctypes -import json -import os -import sys -import time -import urllib.error -import urllib.request - - -def post_json(url: str, body: dict, timeout: int = 180) -> dict: - request = urllib.request.Request( - url, - data=json.dumps(body).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=timeout) as response: - return json.load(response) - except urllib.error.HTTPError as exc: - detail = exc.read().decode("utf-8", "replace") - raise RuntimeError("HTTP %d: %s" % (exc.code, detail[:2000])) from exc - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--inputs", required=True) - ap.add_argument("--compare", required=True) - ap.add_argument("--case-id", required=True) - ap.add_argument("--model-path", required=True) - ap.add_argument("--server", default="http://127.0.0.1:18080") - ap.add_argument("--top-k", type=int, default=100) - ap.add_argument("--num-blocks", type=int, default=64) - ap.add_argument("--block-size", type=int, default=256) - ap.add_argument("--out", required=True) - args = ap.parse_args() - - import infinicore - import numpy as np - from infinilm.cache import PagedKVCacheConfig - from infinilm.distributed import DistConfig - from infinilm.infer_engine import InferEngine - from infinilm.lib import _infinilm - from infinilm.modeling_utils import load_model_state_dict_by_file - - with open(args.inputs, encoding="utf-8") as f: - inputs = {x["id"]: x for x in json.load(f)["cases"]} - with open(args.compare, encoding="utf-8") as f: - compared = {x["id"]: x for x in json.load(f)["cases"]} - item = compared[args.case_id] - first_diff = item["first_difference"] - if first_diff is None: - raise ValueError("case %s has no divergence" % args.case_id) - common_generated = item["llama_tokens"][:first_diff] - assert common_generated == item["infinilm_tokens"][:first_diff] - prefix = [int(x) for x in inputs[args.case_id]["input_ids"] + common_generated] - - llama_body = { - "prompt": prefix, - "n_predict": 1, - "temperature": 0.0, - "top_k": 1, - "top_p": 1.0, - "min_p": 0.0, - "typical_p": 1.0, - "repeat_penalty": 1.0, - "repeat_last_n": 0, - "presence_penalty": 0.0, - "frequency_penalty": 0.0, - "seed": 1, - "ignore_eos": True, - "cache_prompt": False, - "return_tokens": True, - "n_probs": args.top_k, - "stream": False, - "samplers": ["top_k", "temperature"], - } - llama_response = post_json(args.server.rstrip("/") + "/completion", llama_body) - llama_probs = llama_response["completion_probabilities"][0]["top_logprobs"] - - load_started = time.time() - engine = InferEngine( - model_path=args.model_path, - device=infinicore.device("cuda:0"), - distributed_config=DistConfig(1), - cache_config=PagedKVCacheConfig( - args.num_blocks, args.block_size, max_batch_size=1 - ), - attention_backend="paged-attn", - ) - load_model_state_dict_by_file(engine, args.model_path, dtype=engine.dtype) - load_s = time.time() - load_started - - length = len(prefix) - positions = list(range(length)) - if engine.position_id_axes > 1: - positions = [positions for _ in range(engine.position_id_axes)] - tensors = { - "input_ids": infinicore.from_list([prefix], dtype=infinicore.int64).view( - [1, length] - ), - "position_ids": infinicore.from_list(positions, dtype=infinicore.int64), - "past_kv_lengths": infinicore.from_list([0], dtype=infinicore.int32), - "total_kv_lengths": infinicore.from_list([length], dtype=infinicore.int32), - "input_offsets": infinicore.from_list([0, length], dtype=infinicore.int32), - "cu_seqlens": infinicore.from_list([0, length], dtype=infinicore.int32), - "block_tables": infinicore.from_list([[0]], dtype=infinicore.int32), - "slot_mapping": infinicore.from_list( - list(range(length)), dtype=infinicore.int64 - ), - "mamba_init_state_indices": infinicore.from_list([0], dtype=infinicore.int32), - "mamba_final_state_indices": infinicore.from_list([1], dtype=infinicore.int32), - } - cpp_input = engine._build_input( - tensors["input_ids"], - position_ids=tensors["position_ids"], - past_kv_lengths=tensors["past_kv_lengths"], - total_kv_lengths=tensors["total_kv_lengths"], - input_offsets=tensors["input_offsets"], - cu_seqlens=tensors["cu_seqlens"], - block_tables=tensors["block_tables"], - slot_mapping=tensors["slot_mapping"], - mamba_init_state_indices=tensors["mamba_init_state_indices"], - mamba_final_state_indices=tensors["mamba_final_state_indices"], - sample_all_positions=False, - temperature=0.0, - top_k=1, - top_p=1.0, - ) - output = _infinilm.InferEngine.forward(engine, cpp_input) - raw_logits = infinicore.Tensor(output.logits) - logits_shape = list(raw_logits.shape) - cpu_logits = raw_logits.to(infinicore.device("cpu", 0)) - if cpu_logits.dtype != infinicore.bfloat16: - raise TypeError("expected BF16 logits, got %s" % cpu_logits.dtype) - bits_type = ctypes.c_uint16 * cpu_logits.numel() - bits = np.ctypeslib.as_array(bits_type.from_address(cpu_logits.data_ptr())).copy() - all_logits = (bits.astype(np.uint32) << 16).view(np.float32).reshape(logits_shape) - logits = all_logits.reshape(-1, logits_shape[-1])[-1] - order = np.argpartition(logits, -args.top_k)[-args.top_k :] - order = order[np.argsort(logits[order])[::-1]] - max_logit = float(logits[order[0]]) - infini_top = [ - { - "id": int(i), - "logit": float(logits[i]), - "delta_from_top": float(logits[i] - max_logit), - } - for i in order - ] - - llama_map = {int(x["id"]): float(x["logprob"]) for x in llama_probs} - infini_map = {int(x["id"]): float(x["delta_from_top"]) for x in infini_top} - candidate_ids = sorted(set(llama_map) | set(infini_map)) - candidate_table = [ - { - "id": token_id, - "llama_logprob": llama_map.get(token_id), - "infini_delta_from_top": infini_map.get(token_id), - } - for token_id in candidate_ids - ] - - result = { - "case_id": args.case_id, - "first_difference": first_diff, - "base_input_ids": inputs[args.case_id]["input_ids"], - "common_generated_prefix": common_generated, - "diagnostic_prefix": prefix, - "llama_selected": int(llama_response["tokens"][0]), - "infinilm_selected": int(order[0]), - "llama_top_logprobs": llama_probs, - "infinilm_top_logits": infini_top, - "candidate_table": candidate_table, - "infinilm_logits_shape": logits_shape, - "infinilm_logits_finite": bool(np.isfinite(logits).all()), - "infinilm_load_s": round(load_s, 4), - } - os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) - with open(args.out, "w", encoding="utf-8") as f: - json.dump(result, f, ensure_ascii=False, indent=2) - print( - "CASE=%s diff=%d prefix_len=%d llama=%d infini=%d" - % ( - args.case_id, - first_diff, - len(prefix), - result["llama_selected"], - result["infinilm_selected"], - ) - ) - print( - "LLAMA_TOP5 %s" % [(x["id"], round(x["logprob"], 6)) for x in llama_probs[:5]] - ) - print( - "INFINI_TOP5 %s" - % [(x["id"], round(x["delta_from_top"], 6)) for x in infini_top[:5]] - ) - print( - "FINITE=%s SHAPE=%s LOAD=%.3fs" - % (result["infinilm_logits_finite"], result["infinilm_logits_shape"], load_s) - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/gguf_routeb_first_diff_batch.py b/scripts/gguf_routeb_first_diff_batch.py deleted file mode 100644 index 40284e1c7..000000000 --- a/scripts/gguf_routeb_first_diff_batch.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -"""Inspect first-divergence logits for every non-exact Route-B case.""" - -from __future__ import annotations - -import argparse -import ctypes -import json -import os -import time -import urllib.request - - -def post_json(url: str, body: dict, timeout: int = 180) -> dict: - req = urllib.request.Request( - url, - data=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=timeout) as response: - return json.load(response) - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--inputs", required=True) - ap.add_argument("--compare", required=True) - ap.add_argument("--model-path", required=True) - ap.add_argument("--server", default="http://127.0.0.1:18080") - ap.add_argument("--top-k", type=int, default=100) - ap.add_argument("--num-blocks", type=int, default=64) - ap.add_argument("--block-size", type=int, default=256) - ap.add_argument("--out", required=True) - args = ap.parse_args() - - import infinicore - import numpy as np - from infinilm.cache import PagedKVCacheConfig - from infinilm.distributed import DistConfig - from infinilm.infer_engine import InferEngine - from infinilm.lib import _infinilm - from infinilm.modeling_utils import load_model_state_dict_by_file - - with open(args.inputs, encoding="utf-8") as f: - inputs = {x["id"]: x for x in json.load(f)["cases"]} - with open(args.compare, encoding="utf-8") as f: - compared = json.load(f)["cases"] - divergent = [x for x in compared if x["first_difference"] is not None] - - started = time.time() - engine = InferEngine( - model_path=args.model_path, - device=infinicore.device("cuda:0"), - distributed_config=DistConfig(1), - cache_config=PagedKVCacheConfig( - args.num_blocks, args.block_size, max_batch_size=1 - ), - attention_backend="paged-attn", - ) - load_model_state_dict_by_file(engine, args.model_path, dtype=engine.dtype) - load_s = time.time() - started - - results = [] - for item in divergent: - case_id = item["id"] - first_diff = item["first_difference"] - common = item["llama_tokens"][:first_diff] - assert common == item["infinilm_tokens"][:first_diff] - prefix = [int(x) for x in inputs[case_id]["input_ids"] + common] - body = { - "prompt": prefix, - "n_predict": 1, - "temperature": 0.0, - "top_k": 1, - "top_p": 1.0, - "min_p": 0.0, - "typical_p": 1.0, - "repeat_penalty": 1.0, - "repeat_last_n": 0, - "presence_penalty": 0.0, - "frequency_penalty": 0.0, - "seed": 1, - "ignore_eos": True, - "cache_prompt": False, - "return_tokens": True, - "n_probs": args.top_k, - "stream": False, - "samplers": ["top_k", "temperature"], - } - llama = post_json(args.server.rstrip("/") + "/completion", body) - llama_probs = llama["completion_probabilities"][0]["top_logprobs"] - - length = len(prefix) - positions = list(range(length)) - if engine.position_id_axes > 1: - positions = [positions for _ in range(engine.position_id_axes)] - tensors = { - "input_ids": infinicore.from_list([prefix], dtype=infinicore.int64).view( - [1, length] - ), - "position_ids": infinicore.from_list(positions, dtype=infinicore.int64), - "past_kv_lengths": infinicore.from_list([0], dtype=infinicore.int32), - "total_kv_lengths": infinicore.from_list([length], dtype=infinicore.int32), - "input_offsets": infinicore.from_list([0, length], dtype=infinicore.int32), - "cu_seqlens": infinicore.from_list([0, length], dtype=infinicore.int32), - "block_tables": infinicore.from_list([[0]], dtype=infinicore.int32), - "slot_mapping": infinicore.from_list( - list(range(length)), dtype=infinicore.int64 - ), - "mamba_init_state_indices": infinicore.from_list( - [0], dtype=infinicore.int32 - ), - "mamba_final_state_indices": infinicore.from_list( - [1], dtype=infinicore.int32 - ), - } - cpp_input = engine._build_input( - tensors["input_ids"], - position_ids=tensors["position_ids"], - past_kv_lengths=tensors["past_kv_lengths"], - total_kv_lengths=tensors["total_kv_lengths"], - input_offsets=tensors["input_offsets"], - cu_seqlens=tensors["cu_seqlens"], - block_tables=tensors["block_tables"], - slot_mapping=tensors["slot_mapping"], - mamba_init_state_indices=tensors["mamba_init_state_indices"], - mamba_final_state_indices=tensors["mamba_final_state_indices"], - sample_all_positions=False, - temperature=0.0, - top_k=1, - top_p=1.0, - ) - output = _infinilm.InferEngine.forward(engine, cpp_input) - raw = infinicore.Tensor(output.logits) - shape = list(raw.shape) - cpu = raw.to(infinicore.device("cpu", 0)) - if cpu.dtype != infinicore.bfloat16: - raise TypeError("expected BF16 logits, got %s" % cpu.dtype) - bits_type = ctypes.c_uint16 * cpu.numel() - bits = np.ctypeslib.as_array(bits_type.from_address(cpu.data_ptr())).copy() - logits = (bits.astype(np.uint32) << 16).view(np.float32).reshape(shape) - logits = logits.reshape(-1, shape[-1])[-1] - order = np.argpartition(logits, -args.top_k)[-args.top_k :] - order = order[np.argsort(logits[order], kind="stable")[::-1]] - top_logit = float(logits[order[0]]) - infini_top = [ - { - "id": int(i), - "logit": float(logits[i]), - "delta_from_top": float(logits[i] - top_logit), - } - for i in order - ] - llama_map = {int(x["id"]): float(x["logprob"]) for x in llama_probs} - infini_map = {x["id"]: x["delta_from_top"] for x in infini_top} - llama_selected = int(llama["tokens"][0]) - infini_selected = int(order[0]) - candidate_ids = sorted(set(llama_map) | set(infini_map)) - candidate_table = [ - { - "id": token_id, - "llama_logprob": llama_map.get(token_id), - "infini_delta_from_top": infini_map.get(token_id), - "infini_logit": float(logits[token_id]), - } - for token_id in candidate_ids - ] - selected_logits = { - "llama_token_infini_logit": float(logits[llama_selected]), - "infini_token_infini_logit": float(logits[infini_selected]), - "infini_margin_selected_minus_llama": float( - logits[infini_selected] - logits[llama_selected] - ), - "llama_margin_selected_minus_infini": float( - llama_map[llama_selected] - llama_map.get(infini_selected, float("nan")) - ), - } - result = { - "case_id": case_id, - "first_difference": first_diff, - "prefix_length": len(prefix), - "llama_selected": llama_selected, - "infinilm_selected": infini_selected, - "llama_top_logprobs": llama_probs, - "infinilm_top_logits": infini_top, - "selected_pair": selected_logits, - "candidate_table": candidate_table, - "infinilm_logits_shape": shape, - "infinilm_logits_finite": bool(np.isfinite(logits).all()), - } - results.append(result) - print( - "%-10s diff=%2d llama=%6d infini=%6d llama_margin=%+.6f infini_margin=%+.6f" - % ( - case_id, - first_diff, - llama_selected, - infini_selected, - selected_logits["llama_margin_selected_minus_infini"], - selected_logits["infini_margin_selected_minus_llama"], - ) - ) - - report = {"load_s": round(load_s, 4), "case_count": len(results), "cases": results} - os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) - with open(args.out, "w", encoding="utf-8") as f: - json.dump(report, f, ensure_ascii=False, indent=2) - print( - "RESULT cases=%d finite=%s load=%.3fs" - % (len(results), all(x["infinilm_logits_finite"] for x in results), load_s) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/gguf_routeb_gemv_check.py b/scripts/gguf_routeb_gemv_check.py deleted file mode 100644 index 876cb14f8..000000000 --- a/scripts/gguf_routeb_gemv_check.py +++ /dev/null @@ -1,368 +0,0 @@ -#!/usr/bin/env python3 -""" -InfiniLM 路线 B —— 阶段 3.2 + 3.3 验收：linear_gguf 两条 NVIDIA 路径的数值正确性 - -被测对象（两条路径都由算子本体所在的头文件提供，probe 直接 include）： - * `InfiniCore/src/infiniop/ops/linear_gguf/nvidia/linear_gguf_gemv.cuh` - —— M <= kMaxDecodeM 的 decode 路径（一 warp 一行、寄存器内解码 + fp32 累加）； - * `InfiniCore/src/infiniop/ops/linear_gguf/nvidia/linear_gguf_dequant.cuh` - —— M > kMaxDecodeM 的 prefill 路径（64 行权重解码到 BF16 scratch + cublasGemmEx）。 -`scripts/gguf_routeb_gemv_probe.cu` 里的路由谓词与算子 `calculate` 用的是同一个 -`kMaxDecodeM`，所以每条用例走的真是发布路径上那条 kernel；probe 还会在 stdout 报 -`path=gemv|prefill`，脚本据此**断言路由本身**（见下面的“路径”判据）。 - -判据不新造： - * 主判据 = 方案 §1.2 第 2 条「GEMM 输出与稠密 BF16 权重 @ x 的 cos_sim > 0.999」。 - 这里的“稠密权重”用的是 3.1 已证与 gguf-py / 头逐位相同的 numpy 参考（`REF`）， - 所以这条判据同时就把「解码正确」与「GEMV / prefill 正确」两件事串在了一起。 - prefill 路径把权重先舍到 BF16 再乘，与这条基准口径一致。 - * 权重字节全部取自真实打包产物的 `*.weight_bytes` **整行**（不是随机 block）， - 因为 kernel 依赖“一行 = 整数个 block”这个契约，随机 block 拼不出来。每种类型 - 取首/中/尾三个张量（跨层），避开“两份产物挑到同一层同一张量”的假独立性。 - * 行数默认 200（不是 64 的整数倍），这样 prefill 的 tile 循环会走到 - “最后一片不满”的分支。 - * 附带两条拒绝（必须报错、不许静默出结果）：未知 type、K 不是 block 元素数整数倍。 - （原来那条「M=9 超过 decode 上限必须被拒」在 3.3 之后不再成立，M=9 现在既是 - prefill 的下边界、又是一条正例，见 --ms 默认值。） - -累加顺序与 numpy 不同（gemv：块内顺序求和 -> 沿 block 累加 -> warp shuffle 归约； -prefill：cublas 分块），所以这里**不要求逐位相同**，而是把逐位相同率当作观测量报出来， -cos_sim 当判据。 - -用法： - /usr/bin/python3 scripts/gguf_routeb_gemv_check.py \ - [--model-path /home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8] \ - [--rows 200] [--ms 1,8,9,16,32,64,256,1024] [--keep] -退出码 0 = 全部 PASS。 -""" - -from __future__ import annotations - -import argparse -import os -import re -import subprocess -import sys - -import numpy as np - -_HERE = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, _HERE) - -import gguf_routeb_blocks_ref as bref # noqa: E402 -from gguf_routeb_blocks_ref import ( # noqa: E402 - BLOCK_SIZE, - REF, - TYPE_SIZE, - TYPES, - Artifact, - check, - skip, -) - -GEMV_DIR = os.path.join(bref.HEADER_DIR, "nvidia") -PROBE_SRC = os.path.join(_HERE, "gguf_routeb_gemv_probe.cu") -MAX_M = 8 # kMaxDecodeM：M <= 8 走 gemv，M > 8 走 prefill -PREFILL_MS = ( - "9,16,32,64,256,1024" # 9 = prefill 下边界（§1.2 第 3 条含 16/32/64/256/1024） -) -PATH_RE = re.compile(r"path=(\w+)") -T_NAME = {8: "Q8_0", 12: "Q4_K", 13: "Q5_K", 14: "Q6_K"} - - -def reset_counters(): - bref._PASS = bref._FAIL = bref._SKIP = 0 - - -# --------------------------------------------------------------- bf16 位模式 -def bf16_to_f32(bits): - return (np.asarray(bits, np.uint16).astype(np.uint32) << np.uint32(16)).view( - np.float32 - ) - - -def f32_to_bf16(x): - return bref.float_to_bf16_bits(x) - - -def cos_sim(a, b): - a = np.asarray(a, np.float64).reshape(-1) - b = np.asarray(b, np.float64).reshape(-1) - na, nb = np.linalg.norm(a), np.linalg.norm(b) - if na == 0.0 or nb == 0.0: - return float(np.array_equal(a, b)) - return float(a @ b / (na * nb)) - - -# ------------------------------------------------------------- 真实权重整行 -def pick_rows(art, t, want_rows, rng, which=0): - """从类型 t 的某个真实张量里取连续若干行字节（which 指定取哪个）。""" - names = art.type_names(t) - if not names: - return None - ts = TYPE_SIZE[t] - name = names[min(which, len(names) - 1)] - _t, shard, base, row_bytes, nrows = art.blobs[name] - blocks_per_row = row_bytes // ts - if blocks_per_row * ts != row_bytes or blocks_per_row < 1: - raise RuntimeError( - "%s 的 row_bytes=%d 不是 block_size %d 的整数倍" % (name, row_bytes, ts) - ) - rows = min(want_rows, nrows) - r0 = int(rng.integers(0, nrows - rows + 1)) - with open(shard, "rb") as fh: - fh.seek(base + r0 * row_bytes) - buf = np.frombuffer(fh.read(rows * row_bytes), np.uint8) - W = buf.reshape(rows, row_bytes).copy() - return name, W, blocks_per_row * BLOCK_SIZE[t], r0 - - -def dense_weights(t, W, rows, K): - """numpy 参考反量化：W[rows, row_bytes] -> float32 [rows, K]。""" - ts = TYPE_SIZE[t] - blocks = W.reshape(-1, ts) - dec = REF[t](blocks) # (n_blocks, bs) float32，3.1 已证逐位正确 - return dec.reshape(rows, K) - - -# ------------------------------------------------------------------ 驱动调用 -# probe 一个可执行文件覆盖两条路径（名字沿用 3.2），具体走哪条由它内部的 -# m > kMaxDecodeM 谓词决定，并由 stdout 的 path= 字段报回来。 -def run_gemv(binary, t, A_bf16, W, K, workdir, tag): - m, _ = A_bf16.shape - n, row_bytes = W.shape - abin = os.path.join(workdir, "%s_m%d_t%d.a" % (tag, m, t)) - wbin = os.path.join(workdir, "%s_m%d_t%d.w" % (tag, m, t)) - cbin = os.path.join(workdir, "%s_m%d_t%d.c" % (tag, m, t)) - A_bf16.astype(np.uint16).tofile(abin) - np.ascontiguousarray(W).tofile(wbin) - cmd = [binary, str(t), str(m), str(n), str(K), str(row_bytes), abin, wbin, cbin] - p = subprocess.run(cmd, capture_output=True, text=True) - return p, cbin - - -def check_type(binary, art, t, rows, Ms, rng, workdir, which_list): - """对同一类型的多个张量（刻意跨层）各跑一轮。 - - 只取排序后第一个张量会在两份产物上挑到同一个张量（字节完全相同），那 - 时候选产物就只是“两种键形态”而不是两份独立权重证据，所以这里固定取 - 首/中/尾三个（不足则去重）。注意 mini8 是完整模型的前若干层，layer 0/1 - 的张量在两份产物里字节相同，取样点落在这些层时仍然撞——这是数据的性质， - 不是取样能修的（Q4_K 尤其：全模型只有 4 个张量且都在 layer 1）。 - """ - names = art.type_names(t) - picks = sorted({min(w, len(names) - 1) for w in which_list}) - done = set() - for wi in picks: - picked = pick_rows(art, t, rows, rng, wi) - if picked is None: - skip("%d 张量 #%d" % (t, wi), "产物不含该类型") - continue - name, W, K, r0 = picked - if name in done: - continue - done.add(name) - check_type_one(binary, art, t, name, W, K, r0, Ms, rng, workdir) - - -def check_type_one(binary, art, t, name, W, K, r0, Ms, rng, workdir): - n = W.shape[0] - Wf32 = dense_weights(t, W, n, K) - # 稠密 BF16 权重 @ x 这条基准：先把反量化结果舍到 bf16 再算，同 §1.2 第 2 条口径 - Wdense = bf16_to_f32(f32_to_bf16(Wf32)) - print( - " %s：%s（起始行 %d），%d 行 x K=%d，row_bytes=%d" - % (T_NAME.get(t, t), name, r0, n, K, W.shape[1]) - ) - for m in Ms: - A = (rng.standard_normal((m, K)) * 0.5).astype(np.float32) - Abits = f32_to_bf16(A) - Af = bf16_to_f32(Abits) # kernel 看到的就是这份值 - p, cbin = run_gemv(binary, t, Abits, W, K, workdir, "gemv") - if not check( - "%s M=%d：kernel 退出码 0" % (T_NAME.get(t, t), m), - p.returncode == 0, - "rc=%d %s" % (p.returncode, (p.stderr or p.stdout).strip()[-600:]), - ): - continue - # 路由判据：probe 报的 path 必须等于算子在该 M 上会选的路径。数值过了但 - # 路走错了同样不可接受（那意味着门测的不是发布路径）。 - want_path = "gemv" if m <= MAX_M else "prefill" - pm = PATH_RE.search(p.stdout or "") - got_path = pm.group(1) if pm else "?" - check( - "%s M=%d：走 %s 路径（与算子 calculate 的谓词一致）" - % (T_NAME.get(t, t), m, want_path), - got_path == want_path, - "probe 报 path=%s" % got_path, - ) - got = bf16_to_f32(np.fromfile(cbin, np.uint16).reshape(m, n)) - assert got.shape == (m, n) - ref = (Af @ Wdense.T).astype(np.float32) # §1.2 第 2 条口径的基准 - ref_exact = (Af @ Wf32.T).astype(np.float32) # 不先把权重舍到 bf16 - c = cos_sim(got, ref) - check( - "%s M=%d：cos_sim(kernel, 稠密 BF16 权重 @ x) > 0.999" - % (T_NAME.get(t, t), m), - c > 0.999, - "cos_sim=%.8f" % c, - ) - # 观测量（不作判据）：bf16 位相同率、最大绝对/相对偏差、vs 未舍入基准的 cos_sim - same = float(np.mean(f32_to_bf16(got) == f32_to_bf16(ref))) - dg = got.astype(np.float64) - ref.astype(np.float64) - absd = float(np.max(np.abs(dg))) - # 相对偏差只在“有意义的元素”上算（|ref| >= 最大幅值的 1%），否则会被近零 - # 元素除出几十倍的假大数，那种数字没有判读价值。 - sig = np.abs(ref.astype(np.float64)) >= 0.01 * float(np.max(np.abs(ref))) - rel = ( - float(np.max(np.abs(dg[sig]) / np.abs(ref.astype(np.float64)[sig]))) - if sig.any() - else 0.0 - ) - print( - " 观测：cos_sim(kernel, 稠密 BF16 权重)=%.10f" - " cos_sim(kernel, 未舍入基准)=%.10f bf16 逐位相同率=%.4f" - " max|Δ|=%.3e max 相对偏差(|ref|≥最大幅值1%% 的子集)=%.3e %s" - % ( - c, - cos_sim(got, ref_exact), - same, - absd, - rel, - p.stdout.strip().split("ok")[-1].strip(), - ) - ) - - -def check_rejections(binary, art, workdir): - """必须报错的输入：不许静默出结果。 - - 3.3 之前这里还有一条「M=9 超过 decode 上限被拒」，现在 prefill 接管了 M>8， - 该用例已反转成 --ms 里的正例（prefill 下边界）。 - """ - name_ok = None - for t in TYPES: - picked = pick_rows(art, t, 4, np.random.default_rng(7)) - if picked: - name_ok, W, K = t, picked[1], picked[2] - break - A = np.zeros((1, K), np.float32) - Abits = f32_to_bf16(A) - - p, _ = run_gemv(binary, 10, Abits, W, K, workdir, "rej") - check( - "未知 ggml type 10 被拒（rc=3，不启动 kernel）", - p.returncode == 3, - "rc=%d %s" % (p.returncode, p.stderr.strip()[-300:]), - ) - - bad_k = K + (BLOCK_SIZE[name_ok] - 1) # 不再是整数个 block - A_bad = f32_to_bf16(np.zeros((1, bad_k), np.float32)) - p, _ = run_gemv(binary, name_ok, A_bad, W, bad_k, workdir, "rej") - check( - "K 不是 block 元素数整数倍被拒（rc=3）", - p.returncode == 3, - "rc=%d %s" % (p.returncode, p.stderr.strip()[-300:]), - ) - - # 同一条约束在 prefill 路径上也必须成立（两条路径各自有谓词，不能只查 gemv） - A_bad_p = f32_to_bf16(np.zeros((MAX_M + 1, bad_k), np.float32)) - p, _ = run_gemv(binary, name_ok, A_bad_p, W, bad_k, workdir, "rej") - check( - "prefill 路径同样拒掉不整除的 K（rc=3）", - p.returncode == 3, - "rc=%d %s" % (p.returncode, p.stderr.strip()[-300:]), - ) - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument( - "--model-path", default="/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8" - ) - ap.add_argument( - "--rows", - type=int, - default=200, - help="每个张量取多少行权重（不是 64 的整数倍才能盖住 tile 余数）", - ) - ap.add_argument( - "--ms", - default="1,8," + PREFILL_MS, - help="逗号分隔；<=8 走 gemv，>8 走 prefill（两条路径同一份门）", - ) - ap.add_argument("--workdir", default="/home/liuxd/tmp_routeb/gemv32") - ap.add_argument("--nvcc", default=os.environ.get("CUDACXX", "nvcc")) - ap.add_argument("--skip-build", action="store_true") - ap.add_argument("--no-reject", action="store_true") - ap.add_argument("--seed", type=int, default=20260829) - args = ap.parse_args() - - reset_counters() - rng = np.random.default_rng(args.seed) - os.makedirs(args.workdir, exist_ok=True) - Ms = [int(x) for x in args.ms.split(",") if x.strip()] - print( - "产物：%s\n被测：\n %s\n %s\n %s\n临时目录：%s\n每种类型权重行数：%d，M 取 %s" - % ( - args.model_path, - os.path.join(GEMV_DIR, "linear_gguf_gemv.cuh"), - os.path.join(GEMV_DIR, "linear_gguf_dequant.cuh"), - PROBE_SRC, - args.workdir, - args.rows, - Ms, - ) - ) - - binary = os.path.join(args.workdir, "gemv_probe") - print("\n[1] 编译两条路径的驱动（prefill 需要 -lcublas）") - if args.skip_build: - skip("编译", "--skip-build") - else: - try: - bref.build_probe( - PROBE_SRC, binary, args.nvcc, extra=["-I", GEMV_DIR, "-lcublas"] - ) - check( - "nvcc 编译 %s 通过（含两个 kernel 头 + cublas）" - % os.path.basename(PROBE_SRC), - True, - ) - except Exception as exc: # noqa: BLE001 - check( - "nvcc 编译 %s 通过" % os.path.basename(PROBE_SRC), - False, - str(exc)[-2000:], - ) - return 1 - - art = Artifact(args.model_path) - print("\n[2] 真实权重对 numpy 稠密基准（gemv + prefill，判据：cos_sim > 0.999）") - print( - "产物 blob 张量 %d 个（key_prefix=%r），按类型：%s" - % (len(art.blobs), art.prefix, {t: len(art.type_names(t)) for t in TYPES}) - ) - for t in TYPES: - n_avail = len(art.type_names(t)) - # 张量本来就少（Q4_K 全模型只有 4 个，且都在 layer 1）时全取，否则首/中/尾 - which = list(range(n_avail)) if n_avail <= 6 else [0, n_avail // 2, n_avail - 1] - which = which or [0] - check_type(binary, art, t, args.rows, Ms, rng, args.workdir, which) - - if args.no_reject: - skip("非法输入用例", "--no-reject") - else: - print("\n[3] 非法输入必须被拒（不许静默出结果）") - check_rejections(binary, art, args.workdir) - - print( - "\n== 结果：%d PASS / %d FAIL / %d SKIP ==" - % (bref._PASS, bref._FAIL, bref._SKIP) - ) - print("临时目录：%s" % args.workdir) - return 0 if bref._FAIL == 0 else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/gguf_routeb_gemv_probe.cu b/scripts/gguf_routeb_gemv_probe.cu deleted file mode 100644 index f5e9e7248..000000000 --- a/scripts/gguf_routeb_gemv_probe.cu +++ /dev/null @@ -1,158 +0,0 @@ -// Standalone driver for linear_gguf's two NVIDIA paths, used by -// scripts/gguf_routeb_gemv_check.py: -// -// M <= kMaxDecodeM -> launch_gemv_decode (stage 3.2, decode path) -// M > kMaxDecodeM -> launch_prefill (stage 3.3, prefill path) -// -// The routing predicate is the one the op itself applies in -// linear_gguf_nvidia.cu::calculate, so a case run here goes through the same -// function the shipped kernel goes through. -// -// nvcc -O2 -std=c++17 -I /src/infiniop/ops/linear_gguf/nvidia \ -// gguf_routeb_gemv_probe.cu -o gemv_probe -lcublas -// -// gemv_probe -// -// A test harness, not a library target: it includes the two kernel headers and -// links cublas directly, so the paths can be checked numerically without a -// registered op or an InfiniCore handle. -#include -#include -#include - -#include -#include - -#include "linear_gguf_dequant.cuh" - -#define CUDA_CHECK(call) \ - do { \ - cudaError_t err__ = (call); \ - if (err__ != cudaSuccess) { \ - std::fprintf(stderr, "gemv probe: %s failed: %s\n", #call, \ - cudaGetErrorString(err__)); \ - return 5; \ - } \ - } while (0) - -static std::vector read_all(const char *path, size_t want) { - FILE *f = std::fopen(path, "rb"); - if (!f) { - std::fprintf(stderr, "gemv probe: cannot open %s\n", path); - exit(4); - } - std::vector buf(want); - const size_t got = std::fread(buf.data(), 1, want, f); - std::fclose(f); - if (got != want) { - std::fprintf(stderr, "gemv probe: short read on %s (wanted %zu, got %zu)\n", path, want, - got); - exit(4); - } - return buf; -} - -int main(int argc, char **argv) { - if (argc != 9) { - std::fprintf(stderr, - "usage: %s " - "\n", - argv[0]); - return 2; - } - const int32_t type = std::atoi(argv[1]); - const int m_count = std::atoi(argv[2]); - const int n_count = std::atoi(argv[3]); - const int k = std::atoi(argv[4]); - const int64_t row_bytes = std::atoll(argv[5]); - if (m_count <= 0 || n_count <= 0 || k <= 0 || row_bytes <= 0) { - std::fprintf(stderr, "gemv probe: bad geometry\n"); - return 2; - } - const bool prefill = m_count > op::linear_gguf::nvidia::kMaxDecodeM; - - std::vector h_a = read_all(argv[6], static_cast(m_count) * k * 2); - std::vector h_w = read_all(argv[7], static_cast(n_count) * row_bytes); - - __nv_bfloat16 *d_a = nullptr; - uint8_t *d_w = nullptr; - __nv_bfloat16 *d_c = nullptr; - void *d_scratch = nullptr; - cublasHandle_t blas = nullptr; - CUDA_CHECK(cudaMalloc(&d_a, h_a.size())); - CUDA_CHECK(cudaMalloc(&d_w, h_w.size())); - CUDA_CHECK(cudaMalloc(&d_c, static_cast(m_count) * n_count * 2)); - CUDA_CHECK(cudaMemcpy(d_a, h_a.data(), h_a.size(), cudaMemcpyHostToDevice)); - CUDA_CHECK(cudaMemcpy(d_w, h_w.data(), h_w.size(), cudaMemcpyHostToDevice)); - - // The prefill scratch is the op's workspace tensor; sized through the same - // helper the descriptor uses in create(), so the gate also pins that formula. - const size_t scratch_bytes = op::linear_gguf::nvidia::prefill_scratch_bytes(k); - if (prefill) { - CUDA_CHECK(cudaMalloc(&d_scratch, scratch_bytes)); - if (cublasCreate(&blas) != CUBLAS_STATUS_SUCCESS) { - std::fprintf(stderr, "gemv probe: cublasCreate failed\n"); - return 5; - } - } - - auto run_once = [&]() -> bool { - if (prefill) { - return op::linear_gguf::nvidia::launch_prefill( - blas, type, d_a, d_w, d_c, m_count, n_count, k, row_bytes, - d_scratch, scratch_bytes, nullptr); - } - return op::linear_gguf::nvidia::launch_gemv_decode( - type, d_a, d_w, d_c, m_count, n_count, k, row_bytes, nullptr); - }; - - if (!run_once()) { - std::fprintf(stderr, "gemv probe: %s rejected type %d (no decoder or bad K/row_bytes)\n", - prefill ? "prefill" : "gemv", type); - cublasDestroy(blas); - return 3; - } - CUDA_CHECK(cudaDeviceSynchronize()); - - // One timed run. Interpret with care: this probe is a numeric harness, the - // geometry comes from the caller (scripts/gguf_routeb_gemv_check.py) and small - // N makes the number latency-bound rather than bandwidth-bound. The bandwidth - // work is stage 6. - cudaEvent_t ev0, ev1; - CUDA_CHECK(cudaEventCreate(&ev0)); - CUDA_CHECK(cudaEventCreate(&ev1)); - CUDA_CHECK(cudaEventRecord(ev0)); - for (int i = 0; i < 10; ++i) { - run_once(); - } - CUDA_CHECK(cudaEventRecord(ev1)); - CUDA_CHECK(cudaEventSynchronize(ev1)); - float ms = 0.0f; - CUDA_CHECK(cudaEventElapsedTime(&ms, ev0, ev1)); - cudaEventDestroy(ev0); - cudaEventDestroy(ev1); - - std::vector h_c(static_cast(m_count) * n_count * 2); - CUDA_CHECK(cudaMemcpy(h_c.data(), d_c, h_c.size(), cudaMemcpyDeviceToHost)); - cudaFree(d_a); - cudaFree(d_w); - cudaFree(d_c); - cudaFree(d_scratch); - cublasDestroy(blas); - - FILE *out = std::fopen(argv[8], "wb"); - if (!out) { - std::fprintf(stderr, "gemv probe: cannot open %s\n", argv[8]); - return 4; - } - const bool wrote = std::fwrite(h_c.data(), 1, h_c.size(), out) == h_c.size(); - std::fclose(out); - if (!wrote) { - std::fprintf(stderr, "gemv probe: short write\n"); - return 4; - } - std::printf("gemv probe type=%d M=%d N=%d K=%d path=%s ok %.3f ms/iter %.2f GiB/s of weight\n", - type, m_count, n_count, k, prefill ? "prefill" : "gemv", ms / 10.0, - h_w.size() / (ms / 10.0 * 1e-3) / (1024.0 * 1024.0 * 1024.0)); - return 0; -} diff --git a/scripts/gguf_routeb_head_precision.py b/scripts/gguf_routeb_head_precision.py deleted file mode 100644 index 941ad0fc4..000000000 --- a/scripts/gguf_routeb_head_precision.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python3 -"""Recompute divergent lm_head rows in FP32 from GGUF weights and traced hidden states.""" - -import argparse -import json -import os -import sys - -sys.path.insert( - 0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py") -) - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--gguf", required=True) - ap.add_argument("--model-path", required=True) - ap.add_argument("--compare", required=True) - ap.add_argument("--infinilm-trace", required=True) - ap.add_argument("--out", required=True) - args = ap.parse_args() - - import numpy as np - from gguf import GGUFReader - from gguf.constants import GGMLQuantizationType - from gguf.quants import dequantize - - with open(args.compare, encoding="utf-8") as f: - compared = {x["id"]: x for x in json.load(f)["cases"]} - with open(args.infinilm_trace, encoding="utf-8") as f: - traced = json.load(f)["cases"] - with open( - os.path.join(args.model_path, "model.safetensors.index.json"), encoding="utf-8" - ) as f: - weight_map = json.load(f)["weight_map"] - from safetensors import safe_open - - native_shard = safe_open( - os.path.join(args.model_path, weight_map["lm_head.weight"]), - framework="pt", - device="cpu", - ) - native_head = native_shard.get_slice("lm_head.weight") - reader = GGUFReader(args.gguf, "r") - output = next(t for t in reader.tensors if t.name == "output.weight") - type_name = GGMLQuantizationType(int(output.tensor_type)).name - results = [] - for case in traced: - item = compared[case["case_id"]] - diff = int(item["first_difference"]) - llama_token = int(item["llama_tokens"][diff]) - infini_token = int(item["infinilm_tokens"][diff]) - step = case["steps"][diff] - bits = np.asarray(step["hidden_bf16_bits"], dtype=np.uint16) - hidden = (bits.astype(np.uint32) << 16).view(np.float32) - rows = [] - for token_id in (llama_token, infini_token): - raw_row = output.data[token_id : token_id + 1] - row = np.asarray( - dequantize(raw_row, GGMLQuantizationType(int(output.tensor_type))), - dtype=np.float32, - ).reshape(-1) - rows.append(row) - logits = [float(np.dot(hidden, row)) for row in rows] - native_rows = [ - native_head[token_id : token_id + 1].float().numpy().reshape(-1) - for token_id in (llama_token, infini_token) - ] - native_logits = [float(np.dot(hidden, row)) for row in native_rows] - result = { - "case_id": case["case_id"], - "first_difference": diff, - "llama_token": llama_token, - "infinilm_token": infini_token, - "llama_token_fp32_logit": logits[0], - "infinilm_token_fp32_logit": logits[1], - "fp32_margin_llama_minus_infinilm": logits[0] - logits[1], - "fp32_winner": llama_token if logits[0] > logits[1] else infini_token, - "bf16_weight_fp32_margin_llama_minus_infinilm": native_logits[0] - - native_logits[1], - "bf16_weight_fp32_winner": llama_token - if native_logits[0] > native_logits[1] - else infini_token, - "hidden_shape": step["hidden_shape"], - } - results.append(result) - print( - "%-10s llama=%6d infini=%6d gguf_f32=%+.8f bf16w_f32=%+.8f winner=%d" - % ( - result["case_id"], - llama_token, - infini_token, - result["fp32_margin_llama_minus_infinilm"], - result["bf16_weight_fp32_margin_llama_minus_infinilm"], - result["bf16_weight_fp32_winner"], - ), - flush=True, - ) - report = {"gguf_lm_head_type": type_name, "cases": results} - os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) - with open(args.out, "w", encoding="utf-8") as f: - json.dump(report, f, ensure_ascii=False, indent=2) - print( - "RESULT gguf_f32_llama_wins=%d/%d bf16_weight_f32_llama_wins=%d/%d" - % ( - sum(x["fp32_winner"] == x["llama_token"] for x in results), - len(results), - sum(x["bf16_weight_fp32_winner"] == x["llama_token"] for x in results), - len(results), - ), - flush=True, - ) - - -if __name__ == "__main__": - main() diff --git a/scripts/gguf_routeb_infinilm_ref.py b/scripts/gguf_routeb_infinilm_ref.py deleted file mode 100755 index 522a080dc..000000000 --- a/scripts/gguf_routeb_infinilm_ref.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python3 -"""Run deterministic raw-token completions through InfiniLM paged generate().""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -import time - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--inputs", required=True) - ap.add_argument("--model-path", required=True) - ap.add_argument("--new-tokens", type=int, default=8) - ap.add_argument("--repeats", type=int, default=2) - ap.add_argument("--num-blocks", type=int, default=64) - ap.add_argument("--block-size", type=int, default=256) - ap.add_argument( - "--case-ids", - help="Optional comma-separated case IDs for focused regression runs.", - ) - ap.add_argument("--out", required=True) - args = ap.parse_args() - - import infinicore - import numpy as np - from infinilm.cache import PagedKVCacheConfig - from infinilm.distributed import DistConfig - from infinilm.infer_engine import GenerationConfig, InferEngine - from infinilm.modeling_utils import load_model_state_dict_by_file - - with open(args.inputs, encoding="utf-8") as f: - source = json.load(f) - if args.case_ids: - requested = {item.strip() for item in args.case_ids.split(",") if item.strip()} - source["cases"] = [item for item in source["cases"] if item["id"] in requested] - found = {item["id"] for item in source["cases"]} - missing = sorted(requested - found) - if missing: - raise ValueError(f"unknown --case-ids: {missing}") - - started = time.time() - engine = InferEngine( - model_path=args.model_path, - device=infinicore.device("cuda:0"), - distributed_config=DistConfig(1), - cache_config=PagedKVCacheConfig( - args.num_blocks, args.block_size, max_batch_size=1 - ), - attention_backend="paged-attn", - ) - load_model_state_dict_by_file(engine, args.model_path, dtype=engine.dtype) - load_s = time.time() - started - print("MODEL_LOADED %.3fs cases=%d" % (load_s, len(source["cases"])), flush=True) - - outputs = [] - all_ok = True - for case in source["cases"]: - runs = [] - for repeat in range(args.repeats): - prompt = infinicore.from_list( - [[int(x) for x in case["input_ids"]]], dtype=infinicore.int64 - ) - config = GenerationConfig( - max_new_tokens=args.new_tokens, - temperature=0.0, - top_k=1, - top_p=1.0, - eos_token_id=None, - stop_on_eos=False, - ignore_eos=True, - ) - run_started = time.time() - generated = engine.generate(prompt, config) - tokens = [int(np.asarray(x.to_numpy()).reshape(-1)[0]) for x in generated] - runs.append( - { - "repeat": repeat, - "tokens": tokens, - "elapsed_s": round(time.time() - run_started, 4), - } - ) - deterministic = all(x["tokens"] == runs[0]["tokens"] for x in runs[1:]) - exact_length = all(len(x["tokens"]) == args.new_tokens for x in runs) - ok = deterministic and exact_length - all_ok &= ok - outputs.append( - { - "id": case["id"], - "prompt": case["prompt"], - "input_ids": case["input_ids"], - "deterministic": deterministic, - "exact_length": exact_length, - "runs": runs, - } - ) - print( - "%-10s deterministic=%s length=%s tokens=%s" - % (case["id"], deterministic, exact_length, runs[0]["tokens"]), - flush=True, - ) - - result = { - "engine": "InfiniLM", - "model_path": os.path.abspath(args.model_path), - "new_tokens": args.new_tokens, - "repeats": args.repeats, - "load_s": round(load_s, 4), - "cases": outputs, - "all_pass": all_ok, - } - os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) - with open(args.out, "w", encoding="utf-8") as f: - json.dump(result, f, ensure_ascii=False, indent=2) - print("RESULT cases=%d all_pass=%s" % (len(outputs), all_ok), flush=True) - return 0 if all_ok else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/gguf_routeb_infinilm_trace.py b/scripts/gguf_routeb_infinilm_trace.py deleted file mode 100644 index e71665ba1..000000000 --- a/scripts/gguf_routeb_infinilm_trace.py +++ /dev/null @@ -1,300 +0,0 @@ -#!/usr/bin/env python3 -"""Trace InfiniLM's exact paged decode path and capture BF16 logits.""" - -import argparse -import ctypes -import json -import os -import time - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--inputs", required=True) - ap.add_argument("--compare", required=True) - ap.add_argument( - "--expected-results", - help="Optional InfiniLM result JSON supplying the sequence that the current runtime must reproduce; first-difference positions still come from --compare.", - ) - ap.add_argument("--model-path", required=True) - ap.add_argument("--new-tokens", type=int, default=32) - ap.add_argument("--top-k", type=int, default=100) - ap.add_argument("--num-blocks", type=int, default=64) - ap.add_argument("--block-size", type=int, default=256) - ap.add_argument( - "--stop-at-first-diff", - action="store_true", - help="Stop each case immediately after its known first-difference step.", - ) - ap.add_argument( - "--prenorm-dump-root", - help="Optional root for per-case pre-final-RMSNorm binary dumps.", - ) - ap.add_argument( - "--case-id", - action="append", - help="Optionally trace only the named case; repeat for multiple cases.", - ) - ap.add_argument( - "--operator-dump-layer", - type=int, - help="Override the per-case layer selected for generic operator dumps.", - ) - ap.add_argument( - "--gdn-dump-layer", - type=int, - help="Enable GDN intermediate dumps for this layer.", - ) - ap.add_argument( - "--gdn-dump-seq-len", - type=int, - default=1, - help="Sequence length for GDN intermediate dumps (default: 1).", - ) - ap.add_argument("--out", required=True) - ap.add_argument( - "--allow-token-mismatch", - action="store_true", - help="Diagnostic only: keep output even if selected tokens differ from expected.", - ) - args = ap.parse_args() - - import infinicore - import numpy as np - from infinilm.cache import PagedKVCacheConfig - from infinilm.distributed import DistConfig - from infinilm.infer_engine import InferEngine - from infinilm.lib import _infinilm - from infinilm.modeling_utils import load_model_state_dict_by_file - - with open(args.inputs, encoding="utf-8") as f: - inputs = {x["id"]: x for x in json.load(f)["cases"]} - with open(args.compare, encoding="utf-8") as f: - divergent = [ - x for x in json.load(f)["cases"] if x["first_difference"] is not None - ] - if args.case_id: - selected = set(args.case_id) - divergent = [x for x in divergent if x["id"] in selected] - missing = selected - {x["id"] for x in divergent} - if missing: - raise ValueError("unknown or non-divergent case ids: %s" % sorted(missing)) - expected_by_id = None - if args.expected_results: - with open(args.expected_results, encoding="utf-8") as f: - current = json.load(f) - expected_by_id = { - x["id"]: [int(t) for t in x["runs"][0]["tokens"]] for x in current["cases"] - } - if len(divergent) >= max(2, args.num_blocks // 4): - raise ValueError("not enough independent Mamba cache rows") - - started = time.time() - cache_config = PagedKVCacheConfig( - args.num_blocks, args.block_size, max_batch_size=1 - ) - engine = InferEngine( - model_path=args.model_path, - device=infinicore.device("cuda:0"), - distributed_config=DistConfig(1), - cache_config=cache_config, - attention_backend="paged-attn", - ) - load_model_state_dict_by_file(engine, args.model_path, dtype=engine.dtype) - load_s = time.time() - started - results = [] - operator_dump_layers = { - "zh_04": 63, - "zh_06": 0, - "code_04": 20, - "math_04": 55, - } - - for case_index, item in enumerate(divergent): - case_id = item["id"] - if args.prenorm_dump_root: - case_dump_dir = os.path.join(args.prenorm_dump_root, case_id) - os.makedirs(case_dump_dir, exist_ok=True) - os.environ["INFINILM_FINAL_PRENORM_DUMP_DIR"] = case_dump_dir - os.environ["INFINILM_FINAL_PRENORM_DUMP_NUMEL"] = "5120" - # The final fused add-RMSNorm computes its scale from the unrounded - # FP32 sum of layer-63 residual and FFN output, then normalizes the - # BF16 materialized residual. Preserve both inputs for exact replay. - os.environ["INFINILM_LAYER_DUMP_DIR"] = case_dump_dir - os.environ["INFINILM_LAYER_DUMP_NUMEL"] = "5120" - os.environ["INFINILM_OPERATOR_DUMP_LAYER"] = str( - args.operator_dump_layer - if args.operator_dump_layer is not None - else operator_dump_layers.get(case_id, 63) - ) - if case_id in operator_dump_layers: - os.environ["INFINILM_ATTENTION_DUMP_DIR"] = case_dump_dir - os.environ["INFINILM_ATTENTION_DUMP_LAYER"] = str( - operator_dump_layers[case_id] - ) - if args.gdn_dump_layer is not None: - os.environ["INFINILM_GDN_DUMP_LAYER"] = str(args.gdn_dump_layer) - os.environ["INFINILM_GDN_DUMP_SEQ_LEN"] = str(args.gdn_dump_seq_len) - prompt = [int(x) for x in inputs[case_id]["input_ids"]] - expected = ( - expected_by_id[case_id] - if expected_by_id is not None - else [int(x) for x in item["infinilm_tokens"]] - ) - first_diff = int(item["first_difference"]) - kv_block = case_index - mamba_row = case_index + 1 - past = 0 - current = prompt - steps = [] - generated = [] - case_new_tokens = first_diff + 1 if args.stop_at_first_diff else args.new_tokens - for step in range(case_new_tokens): - if args.prenorm_dump_root and step == first_diff: - os.environ["INFINILM_LAYER_DUMP_FIRST_N"] = "64" - else: - os.environ.pop("INFINILM_LAYER_DUMP_FIRST_N", None) - seq_len = len(current) - total = past + seq_len - positions = list(range(past, total)) - if engine.position_id_axes > 1: - positions = [positions for _ in range(engine.position_id_axes)] - slot_base = kv_block * args.block_size - slot_mapping = [slot_base + i for i in range(past, total)] - tensors = { - "input_ids": infinicore.from_list( - [current], dtype=infinicore.int64 - ).view([1, seq_len]), - "position_ids": infinicore.from_list(positions, dtype=infinicore.int64), - "past_kv_lengths": infinicore.from_list([past], dtype=infinicore.int32), - "total_kv_lengths": infinicore.from_list( - [total], dtype=infinicore.int32 - ), - "input_offsets": infinicore.from_list( - [0, seq_len], dtype=infinicore.int32 - ), - "cu_seqlens": infinicore.from_list([0, total], dtype=infinicore.int32), - "block_tables": infinicore.from_list( - [[kv_block]], dtype=infinicore.int32 - ), - "slot_mapping": infinicore.from_list( - slot_mapping, dtype=infinicore.int64 - ), - "mamba_init_state_indices": infinicore.from_list( - [0 if step == 0 else mamba_row], dtype=infinicore.int32 - ), - "mamba_final_state_indices": infinicore.from_list( - [mamba_row], dtype=infinicore.int32 - ), - } - cpp_input = engine._build_input( - tensors["input_ids"], - position_ids=tensors["position_ids"], - past_kv_lengths=tensors["past_kv_lengths"], - total_kv_lengths=tensors["total_kv_lengths"], - input_offsets=tensors["input_offsets"], - cu_seqlens=tensors["cu_seqlens"], - block_tables=tensors["block_tables"], - slot_mapping=tensors["slot_mapping"], - mamba_init_state_indices=tensors["mamba_init_state_indices"], - mamba_final_state_indices=tensors["mamba_final_state_indices"], - sample_all_positions=False, - temperature=0.0, - top_k=1, - top_p=1.0, - ) - output = _infinilm.InferEngine.forward(engine, cpp_input) - token = int( - np.asarray(infinicore.Tensor(output.output_ids).to_numpy()).reshape(-1)[ - 0 - ] - ) - raw = infinicore.Tensor(output.logits) - shape = list(raw.shape) - cpu = raw.to(infinicore.device("cpu", 0)) - if cpu.dtype == infinicore.bfloat16: - bits_type = ctypes.c_uint16 * cpu.numel() - bits = np.ctypeslib.as_array( - bits_type.from_address(cpu.data_ptr()) - ).copy() - logits = (bits.astype(np.uint32) << 16).view(np.float32).reshape(shape) - elif cpu.dtype == infinicore.float32: - logits = ( - np.ctypeslib.as_array( - (ctypes.c_float * cpu.numel()).from_address(cpu.data_ptr()) - ) - .copy() - .reshape(shape) - ) - else: - raise TypeError("expected BF16 or F32 logits, got %s" % cpu.dtype) - logits = logits.reshape(-1, shape[-1])[-1] - order = np.argpartition(logits, -args.top_k)[-args.top_k :] - order = order[np.argsort(logits[order], kind="stable")[::-1]] - top_logit = float(logits[order[0]]) - candidates = [ - { - "id": int(i), - "logit": float(logits[i]), - "delta_from_top": float(logits[i] - top_logit), - } - for i in order - ] - step_result = { - "step": step, - "selected": token, - "logits_shape": shape, - "top_logits": candidates, - } - if step == first_diff: - hidden = infinicore.Tensor(output.hidden_states) - hidden_shape = list(hidden.shape) - hidden_cpu = hidden.to(infinicore.device("cpu", 0)) - step_result["hidden_shape"] = hidden_shape - if hidden_cpu.dtype == infinicore.bfloat16: - hidden_bits_type = ctypes.c_uint16 * hidden_cpu.numel() - hidden_bits = np.ctypeslib.as_array( - hidden_bits_type.from_address(hidden_cpu.data_ptr()) - ).copy() - step_result["hidden_dtype"] = "bfloat16" - step_result["hidden_bf16_bits"] = [int(x) for x in hidden_bits] - elif hidden_cpu.dtype == infinicore.float32: - hidden_values = np.ctypeslib.as_array( - (ctypes.c_float * hidden_cpu.numel()).from_address( - hidden_cpu.data_ptr() - ) - ).copy() - step_result["hidden_dtype"] = "float32" - step_result["hidden_f32"] = [float(x) for x in hidden_values] - else: - raise TypeError( - "expected BF16 or F32 hidden state, got %s" % hidden_cpu.dtype - ) - steps.append(step_result) - generated.append(token) - current = [token] - past = total - expected = expected[:case_new_tokens] - stable = generated == expected - if not stable and not args.allow_token_mismatch: - raise RuntimeError( - "%s trace changed: %s != %s" % (case_id, generated, expected) - ) - results.append({"case_id": case_id, "tokens": generated, "steps": steps}) - print( - "%-10s tokens=%d stable=%s" % (case_id, len(generated), stable), flush=True - ) - - os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) - with open(args.out, "w", encoding="utf-8") as f: - json.dump( - {"load_s": round(load_s, 4), "cases": results}, - f, - ensure_ascii=False, - indent=2, - ) - print("RESULT cases=%d load=%.3fs" % (len(results), load_s), flush=True) - - -if __name__ == "__main__": - main() diff --git a/scripts/gguf_routeb_llama_probe.py b/scripts/gguf_routeb_llama_probe.py deleted file mode 100644 index f467c52bb..000000000 --- a/scripts/gguf_routeb_llama_probe.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -"""Dump llama-server one-token responses at Route-B divergence prefixes.""" - -import argparse -import json -import urllib.request - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--inputs", required=True) - ap.add_argument("--compare", required=True) - ap.add_argument("--case-ids", nargs="+", required=True) - ap.add_argument("--server", default="http://127.0.0.1:18080") - ap.add_argument("--out", required=True) - args = ap.parse_args() - with open(args.inputs, encoding="utf-8") as f: - inputs = {x["id"]: x for x in json.load(f)["cases"]} - with open(args.compare, encoding="utf-8") as f: - cases = {x["id"]: x for x in json.load(f)["cases"]} - results = [] - for case_id in args.case_ids: - item = cases[case_id] - diff = item["first_difference"] - prefix = inputs[case_id]["input_ids"] + item["llama_tokens"][:diff] - body = { - "prompt": prefix, - "n_predict": 1, - "temperature": 0.0, - "top_k": 1, - "top_p": 1.0, - "min_p": 0.0, - "typical_p": 1.0, - "repeat_penalty": 1.0, - "repeat_last_n": 0, - "presence_penalty": 0.0, - "frequency_penalty": 0.0, - "seed": 1, - "ignore_eos": True, - "cache_prompt": False, - "return_tokens": True, - "n_probs": 100, - "stream": False, - "samplers": ["top_k", "temperature"], - } - req = urllib.request.Request( - args.server.rstrip("/") + "/completion", - data=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=180) as response: - raw = json.load(response) - results.append( - {"case_id": case_id, "diff": diff, "prefix": prefix, "response": raw} - ) - probs = raw.get("completion_probabilities", []) - print(case_id, "tokens=", raw.get("tokens"), "prob_entry=", probs[:1]) - with open(args.out, "w", encoding="utf-8") as f: - json.dump({"cases": results}, f, ensure_ascii=False, indent=2) - - -if __name__ == "__main__": - main() diff --git a/scripts/gguf_routeb_llama_ref.py b/scripts/gguf_routeb_llama_ref.py deleted file mode 100755 index e5df59714..000000000 --- a/scripts/gguf_routeb_llama_ref.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -"""Run deterministic raw-token completions through llama-server.""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -import time -import urllib.error -import urllib.request - - -def post_json(url: str, body: dict, timeout: int) -> dict: - request = urllib.request.Request( - url, - data=json.dumps(body).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=timeout) as response: - return json.load(response) - except urllib.error.HTTPError as exc: - detail = exc.read().decode("utf-8", "replace") - raise RuntimeError("HTTP %d: %s" % (exc.code, detail[:2000])) from exc - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--inputs", required=True) - ap.add_argument("--server", default="http://127.0.0.1:18080") - ap.add_argument("--new-tokens", type=int, default=8) - ap.add_argument("--repeats", type=int, default=2) - ap.add_argument("--n-probs", type=int, default=20) - ap.add_argument("--timeout", type=int, default=180) - ap.add_argument("--out", required=True) - args = ap.parse_args() - - with open(args.inputs, encoding="utf-8") as f: - source = json.load(f) - outputs = [] - all_ok = True - for case in source["cases"]: - runs = [] - for repeat in range(args.repeats): - body = { - "prompt": case["input_ids"], - "n_predict": args.new_tokens, - "temperature": 0.0, - "top_k": 1, - "top_p": 1.0, - "min_p": 0.0, - "typical_p": 1.0, - "repeat_penalty": 1.0, - "repeat_last_n": 0, - "presence_penalty": 0.0, - "frequency_penalty": 0.0, - "seed": 1, - "ignore_eos": True, - "cache_prompt": False, - "return_tokens": True, - "n_probs": args.n_probs, - "stream": False, - "samplers": ["top_k", "temperature"], - } - started = time.time() - response = post_json( - args.server.rstrip("/") + "/completion", body, args.timeout - ) - tokens = [int(x) for x in response.get("tokens", [])] - probabilities = response.get("completion_probabilities", []) - runs.append( - { - "repeat": repeat, - "tokens": tokens, - "content": response.get("content", ""), - "first_token_top_logprobs": ( - probabilities[0].get("top_logprobs", []) - if probabilities - else [] - ), - "elapsed_s": round(time.time() - started, 4), - "tokens_evaluated": response.get("tokens_evaluated"), - "tokens_predicted": response.get("tokens_predicted"), - } - ) - deterministic = all(x["tokens"] == runs[0]["tokens"] for x in runs[1:]) - exact_length = all(len(x["tokens"]) == args.new_tokens for x in runs) - ok = deterministic and exact_length - all_ok &= ok - outputs.append( - { - "id": case["id"], - "prompt": case["prompt"], - "input_ids": case["input_ids"], - "deterministic": deterministic, - "exact_length": exact_length, - "runs": runs, - } - ) - print( - "%-10s deterministic=%s length=%s tokens=%s" - % (case["id"], deterministic, exact_length, runs[0]["tokens"]) - ) - - result = { - "engine": "llama.cpp", - "server": args.server, - "new_tokens": args.new_tokens, - "repeats": args.repeats, - "cases": outputs, - "all_pass": all_ok, - } - os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) - with open(args.out, "w", encoding="utf-8") as f: - json.dump(result, f, ensure_ascii=False, indent=2) - print("RESULT cases=%d all_pass=%s" % (len(outputs), all_ok)) - return 0 if all_ok else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/gguf_routeb_llama_trace.py b/scripts/gguf_routeb_llama_trace.py deleted file mode 100644 index 15cca09af..000000000 --- a/scripts/gguf_routeb_llama_trace.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 -"""Capture all llama-server token probabilities for divergent Route-B cases.""" - -import argparse -import json -import os -import urllib.request - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--inputs", required=True) - ap.add_argument("--compare", required=True) - ap.add_argument("--server", default="http://127.0.0.1:18080") - ap.add_argument("--new-tokens", type=int, default=32) - ap.add_argument("--n-probs", type=int, default=100) - ap.add_argument("--out", required=True) - args = ap.parse_args() - with open(args.inputs, encoding="utf-8") as f: - inputs = {x["id"]: x for x in json.load(f)["cases"]} - with open(args.compare, encoding="utf-8") as f: - divergent = [ - x for x in json.load(f)["cases"] if x["first_difference"] is not None - ] - results = [] - for item in divergent: - case_id = item["id"] - body = { - "prompt": inputs[case_id]["input_ids"], - "n_predict": args.new_tokens, - "temperature": 0.0, - "top_k": 1, - "top_p": 1.0, - "min_p": 0.0, - "typical_p": 1.0, - "repeat_penalty": 1.0, - "repeat_last_n": 0, - "presence_penalty": 0.0, - "frequency_penalty": 0.0, - "seed": 1, - "ignore_eos": True, - "cache_prompt": False, - "return_tokens": True, - "n_probs": args.n_probs, - "stream": False, - "samplers": ["top_k", "temperature"], - } - req = urllib.request.Request( - args.server.rstrip("/") + "/completion", - data=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=300) as response: - raw = json.load(response) - tokens = [int(x) for x in raw.get("tokens", [])] - expected = [int(x) for x in item["llama_tokens"]] - if tokens != expected: - raise RuntimeError( - "%s rerun changed: %s != %s" % (case_id, tokens, expected) - ) - results.append( - { - "case_id": case_id, - "tokens": tokens, - "completion_probabilities": raw.get("completion_probabilities", []), - } - ) - print( - "%-10s tokens=%d probabilities=%d stable=%s" - % ( - case_id, - len(tokens), - len(results[-1]["completion_probabilities"]), - tokens == expected, - ), - flush=True, - ) - os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) - with open(args.out, "w", encoding="utf-8") as f: - json.dump({"cases": results}, f, ensure_ascii=False, indent=2) - print("RESULT cases=%d" % len(results), flush=True) - - -if __name__ == "__main__": - main() diff --git a/scripts/gguf_routeb_probe_params.py b/scripts/gguf_routeb_probe_params.py deleted file mode 100644 index 9a7890f72..000000000 --- a/scripts/gguf_routeb_probe_params.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -"""探针：用 mini qwen3_5 config 构造 InferEngine，导出 C++ 侧权威参数键与 shape。""" - -import json -import os -import sys -import tempfile - -import infinicore -from infinilm.cache import StaticKVCacheConfig -from infinilm.distributed import DistConfig -from infinilm.infer_engine import InferEngine - -CFG = { - "model_type": "qwen3_5", - "torch_dtype": "bfloat16", - "tie_word_embeddings": False, - "text_config": { - "model_type": "qwen3_5_text", - "hidden_size": 512, - "num_hidden_layers": 8, - "num_attention_heads": 2, - "num_key_value_heads": 1, - "head_dim": 256, - "intermediate_size": 1024, - "rms_norm_eps": 1e-6, - "max_position_embeddings": 262144, - "vocab_size": 1024, - "full_attention_interval": 4, - "linear_num_key_heads": 2, - "linear_num_value_heads": 6, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_conv_kernel_dim": 4, - "attention_bias": False, - "rope_parameters": { - "rope_type": "mrope", - "rope_theta": 10000000.0, - "partial_rotary_factor": 0.25, - "mrope_section": [11, 11, 10], - "mrope_interleaved": True, - }, - }, -} - - -def main(): - dev = sys.argv[1] if len(sys.argv) > 1 else "cpu" - d = infinicore.device(dev, 0) - tmp = tempfile.mkdtemp(prefix="mini_qwen35_") - with open(os.path.join(tmp, "config.json"), "w") as f: - json.dump(CFG, f, indent=2) - eng = InferEngine( - model_path=tmp, - device=d, - distributed_config=DistConfig(1), - cache_config=StaticKVCacheConfig(max_batch_size=1, max_cache_len=16), - ) - keys = list(eng.state_dict_keyname()) - sd = eng.state_dict()[0] - print("# device=%s 参数总数=%d" % (dev, len(keys))) - for k in sorted(keys): - t = sd.get(k) - shape = tuple(t.shape) if t is not None else "" - dt = getattr(t, "dtype", "") - print("%-58s %-22s %s" % (k, str(shape), dt)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/gguf_routeb_prompts.jsonl b/scripts/gguf_routeb_prompts.jsonl deleted file mode 100644 index d9cb51da0..000000000 --- a/scripts/gguf_routeb_prompts.jsonl +++ /dev/null @@ -1,32 +0,0 @@ -{"id":"en_01","category":"english","prompt":"The capital of France is","max_new_tokens":32} -{"id":"zh_01","category":"chinese","prompt":"中国的首都是","max_new_tokens":32} -{"id":"code_01","category":"code","prompt":"def fibonacci(n):\n ","max_new_tokens":32} -{"id":"en_02","category":"english","prompt":"Water freezes at a temperature of","max_new_tokens":32} -{"id":"en_03","category":"english","prompt":"The three primary colors of light are","max_new_tokens":32} -{"id":"en_04","category":"english","prompt":"In one concise sentence, photosynthesis is","max_new_tokens":32} -{"id":"en_05","category":"english","prompt":"A good unit test should verify that","max_new_tokens":32} -{"id":"en_06","category":"english","prompt":"The next number in the sequence 2, 4, 8, 16 is","max_new_tokens":32} -{"id":"en_07","category":"english","prompt":"To make a cup of tea, first","max_new_tokens":32} -{"id":"en_08","category":"english","prompt":"The opposite of expensive is","max_new_tokens":32} -{"id":"zh_02","category":"chinese","prompt":"请用一句话解释什么是重力：","max_new_tokens":32} -{"id":"zh_03","category":"chinese","prompt":"一年有四个季节，分别是","max_new_tokens":32} -{"id":"zh_04","category":"chinese","prompt":"计算机中的CPU主要负责","max_new_tokens":32} -{"id":"zh_05","category":"chinese","prompt":"如果今天是星期一，那么三天后是","max_new_tokens":32} -{"id":"zh_06","category":"chinese","prompt":"健康作息通常包括","max_new_tokens":32} -{"id":"zh_07","category":"chinese","prompt":"长城是中国著名的","max_new_tokens":32} -{"id":"zh_08","category":"chinese","prompt":"把下面这句话续写完整：人工智能可以帮助人们","max_new_tokens":32} -{"id":"code_02","category":"code","prompt":"def is_even(x):\n return","max_new_tokens":32} -{"id":"code_03","category":"code","prompt":"SELECT name FROM users WHERE","max_new_tokens":32} -{"id":"code_04","category":"code","prompt":"for i in range(5):\n print(","max_new_tokens":32} -{"id":"math_01","category":"math","prompt":"12 * 7 =","max_new_tokens":32} -{"id":"math_02","category":"math","prompt":"If x + 5 = 12, then x =","max_new_tokens":32} -{"id":"math_03","category":"math","prompt":"List the first three prime numbers:","max_new_tokens":32} -{"id":"math_04","category":"structured","prompt":"Return a JSON object with keys name and age:","max_new_tokens":32} -{"id":"ctx_01","category":"context","prompt":"Alice placed the red book on the kitchen table. Bob moved the blue cup to the shelf. Question: Where is the red book? Answer:","max_new_tokens":32} -{"id":"ctx_02","category":"context","prompt":"The meeting starts at 09:30 and lasts 45 minutes. Question: At what time does it end? Answer:","max_new_tokens":32} -{"id":"ctx_03","category":"context","prompt":"A shop has apples, pears, and oranges. Only the pears are on sale today. Question: Which fruit is on sale? Answer:","max_new_tokens":32} -{"id":"ctx_04","category":"context","prompt":"小明把钥匙放进了书包，然后把书包放在椅子上。问题：钥匙在哪里？回答：","max_new_tokens":32} -{"id":"mix_01","category":"mixed","prompt":"Translate into English: 机器学习","max_new_tokens":32} -{"id":"mix_02","category":"mixed","prompt":"解释 API endpoint 的含义：","max_new_tokens":32} -{"id":"mix_03","category":"mixed","prompt":"Symbols test: α + β =","max_new_tokens":32} -{"id":"mix_04","category":"mixed","prompt":"Complete the pair: 北京 -> China; Tokyo ->","max_new_tokens":32} diff --git a/scripts/gguf_routeb_shape_contract.py b/scripts/gguf_routeb_shape_contract.py deleted file mode 100644 index 84afefa73..000000000 --- a/scripts/gguf_routeb_shape_contract.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env python3 -""" -InfiniLM 路线 B —— 阶段 0.3 shape 契约回归（执行方案 §4.2 第 3 条） - -三方对账，任何一处对不上都在此暴露，而不是留到阶段 5 被 strict=False 静默丢权重： - - 1. 框架侧：CPU 构造 mini qwen3_5 InferEngine，导出 C++ 真实参数键 + shape， - 与 gguf_mapping.build_plan(MINI) 做双向 diff（缺键 / 多键 / shape 不符即 FAIL）。 - 2. GGUF 侧：build_plan(REAL) 的每条 gguf 名必须在真文件中存在，shape 必须与 - ne 反序一致（含 conv1d 的 squeeze），blob 条目的行字节必须能被块大小整除； - 共用同一源张量的条目（attn_qkv -> q|k|v）其 slices 必须无重叠地精确覆盖全行。 - 3. 反向无遗漏：真文件中未被丢弃、又未被任何条目消费的张量 = 0。 - 4. 阶段 3 作用域：统计 blob 实际用到的 GGML 类型集合，作为 kernel 必须覆盖的清单。 - -用法： - source scripts/gguf_routeb_env.sh - python3 scripts/gguf_routeb_shape_contract.py [--skip-min] [--engine-device cpu] -退出码 0 表示全部 PASS。 -""" - -from __future__ import annotations - -import argparse -import collections -import json -import os -import sys -from math import prod - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -sys.path.insert( - 0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py") -) - -import gguf_mapping as M # noqa: E402 -from gguf import GGUFReader # noqa: E402 -from gguf.constants import GGML_QUANT_SIZES # noqa: E402 -from gguf.constants import GGMLQuantizationType as Q - -DEFAULT_GGUF = "/home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf" -TYPE_NAME = {int(v.value): str(v.name) for v in Q} - -_PASS = 0 -_FAIL = 0 - - -def check(name, ok, detail=""): - global _PASS, _FAIL - if ok: - _PASS += 1 - print(" PASS %s" % name) - else: - _FAIL += 1 - print(" FAIL %s%s" % (name, ("\n %s" % detail) if detail else "")) - return ok - - -def dims_from_text_config(tc): - """config.json 的 text_config 段 -> Dims。打包器写出 config.json 后也用它自检。""" - return M.Dims( - hidden=tc["hidden_size"], - n_q_heads=tc["num_attention_heads"], - n_kv_heads=tc["num_key_value_heads"], - head_dim=tc["head_dim"], - ffn=tc["intermediate_size"], - lin_k_heads=tc["linear_num_key_heads"], - lin_v_heads=tc["linear_num_value_heads"], - lin_k_dim=tc["linear_key_head_dim"], - lin_v_dim=tc["linear_value_head_dim"], - conv_kernel=tc["linear_conv_kernel_dim"], - vocab=tc["vocab_size"], - n_layers=tc["num_hidden_layers"], - interval=tc["full_attention_interval"], - ) - - -def framework_side(engine_device): - print("\n== 1. 框架侧：mini InferEngine vs build_plan(MINI) ==") - import infinicore - from gguf_routeb_probe_params import CFG - from infinilm.cache import StaticKVCacheConfig - from infinilm.distributed import DistConfig - from infinilm.infer_engine import InferEngine - - check( - "探针 CFG 与 MINI 维度一致", - dims_from_text_config(CFG["text_config"]) == M.MINI, - "cfg=%s\n MINI=%s" % (dims_from_text_config(CFG["text_config"]), M.MINI), - ) - - # 不能用 /tmp：开发机上只读，写不进去。缓存在 HOME 下，无需清理权限。 - tmp = os.path.join( - os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache"), - "gguf_routeb_mini_cfg", - ) - os.makedirs(tmp, exist_ok=True) - with open(os.path.join(tmp, "config.json"), "w") as f: - json.dump(CFG, f) - eng = InferEngine( - model_path=tmp, - device=infinicore.device(engine_device, 0), - distributed_config=DistConfig(1), - cache_config=StaticKVCacheConfig(max_batch_size=1, max_cache_len=16), - ) - sd = eng.state_dict()[0] - actual = {k: tuple(int(x) for x in sd[k].shape) for k in eng.state_dict_keyname()} - print(" -> 引擎导出 %d 个参数（device=%s）" % (len(actual), engine_device)) - - plan = M.build_plan(M.MINI) - want = {} - for e in plan: - assert e.infinilm not in want, "映射表内重复键：%s" % e.infinilm - want[e.infinilm] = M.compress(e.shape) - check( - "映射表无重复键（%d 条）" % len(plan), - len(want) == len(plan), - "%d vs %d" % (len(want), len(plan)), - ) - - missing = sorted(set(actual) - set(want)) - extra = sorted(set(want) - set(actual)) - check( - "无缺键（框架要但映射表未提供 -> 会保持随机初始化）", - not missing, - str(missing[:12]), - ) - check( - "无多键（映射表提供但框架无此参数 -> strict=False 下静默丢）", - not extra, - str(extra[:12]), - ) - - bad = [ - (k, want[k], M.compress(actual[k])) - for k in sorted(set(actual) & set(want)) - if M.compress(actual[k]) != want[k] - ] - check("逐键 shape 全等（压缩长度为 1 的维后）", not bad, str(bad[:8])) - - -def dense_iq_bf16(plan, tensors, gguf_types, prod): - """v1 被稠密化的那 5 个 IQ4 张量若改回 blob，可省下的显存字节数。""" - return sum( - prod(e.shape) * 2 - int(tensors[e.gguf].n_bytes) - for e in plan - if not e.blob - and gguf_types.get(e.gguf) in M.V1_IQUANT_DENSE - and e.gguf in tensors - ) - - -def gguf_side(path): - print("\n== 2. GGUF 侧：build_plan(REAL) vs 真文件 ==") - reader = GGUFReader(path) - tensors = {t.name: t for t in reader.tensors} - gguf_types = { - n: TYPE_NAME.get(int(t.tensor_type), str(t.tensor_type)) - for n, t in tensors.items() - } - plan = M.build_plan(M.REAL) - n_exc = M.apply_v1_exceptions(plan, gguf_types) # v1 稠密化 IQ4（阶段 6 取消） - check("映射条目数 = %d" % len(plan), len(plan) == 947, str(len(plan))) - check("v1 稠密化例外命中 5 个 IQ4 张量", n_exc == 5, str(n_exc)) - - bad_name, bad_shape, bad_type, bad_rows = [], [], [], [] - ok_blob = 0 - type_hist = collections.defaultdict(collections.Counter) - for e in plan: - t = tensors.get(e.gguf) - if t is None: - bad_name.append(e.gguf) - continue - ne = tuple(int(x) for x in t.shape) # GGML ne 序 = [in, out] - hf = tuple(reversed(ne)) # HF/InfiniLM 序 = [out, in] - tn = TYPE_NAME.get(int(t.tensor_type), str(t.tensor_type)) - suffix = e.gguf.split(".", 2)[2] if e.gguf.startswith("blk.") else e.gguf - type_hist[suffix][tn] += 1 - - allowed = M.NATIVE_BLOB_TYPES if e.blob else M.DENSE_SRC_TYPES - if tn not in allowed: - bad_type.append("%s: %s 不在 %s" % (e.gguf, tn, allowed)) - # 共用源张量的条目只占一个行段，比对该段长度而非全量 - exp = M.compress(e.shape) - got = M.compress(hf) - if e.slices: - s, ep = e.slices[0] - got = (ep - s,) + got[1:] - if exp != got: - bad_shape.append( - "%s: 表 %s vs GGUF %s%s" - % ( - e.gguf, - exp, - got, - "" if not e.slices else "（按行段 %s）" % (e.slices[0],), - ) - ) - continue - if e.blob: - blk, ts = (int(x) for x in GGML_QUANT_SIZES[Q[tn]]) - n_in = hf[-1] - if n_in % blk: - bad_rows.append("%s: in=%d 不能被块大小 %d 整除" % (e.gguf, n_in, blk)) - else: - row_bytes = n_in // blk * ts - if row_bytes * hf[0] != int(t.n_bytes): - bad_rows.append( - "%s: %d 行 x %d B != n_bytes %d" - % (e.gguf, hf[0], row_bytes, t.n_bytes) - ) - else: - ok_blob += 1 - - check("每条目的 GGUF 源张量都存在", not bad_name, str(sorted(set(bad_name))[:10])) - check("源类型均在可实现集合内", not bad_type, str(bad_type[:6])) - check( - "shape 与 ne 反序全等（含 conv1d squeeze）", not bad_shape, str(bad_shape[:8]) - ) - check( - "blob 条目行字节可整除且与 n_bytes 相符（%d 条）" % ok_blob, - not bad_rows, - str(bad_rows[:6]), - ) - - print("\n== 3. 切片覆盖 + 反向无遗漏 ==") - shared = collections.defaultdict(list) - for e in plan: - shared[e.gguf].append(e) - cov = [] - for name, es in shared.items(): - if name not in tensors: - continue - n_out = int(tuple(reversed(tensors[name].shape))[0]) - segs = sorted((s, ep) for e in es for s, ep in e.slices) - if len(es) == 1 and not segs: - continue - if not segs: - cov.append("%s: %d 个条目共用但无 slices 声明" % (name, len(es))) - elif ( - segs[0][0] != 0 - or segs[-1][1] != n_out - or any(segs[i][1] != segs[i + 1][0] for i in range(len(segs) - 1)) - ): - cov.append("%s: 切片 %s 未无重叠覆盖 [0,%d)" % (name, segs, n_out)) - check("共用源张量的切片精确覆盖全行", not cov, str(cov[:6])) - - used = {e.gguf for e in plan} - dropped = {n for n in tensors if n.startswith(M.DROP_PREFIXES)} - orphan = sorted(set(tensors) - used - dropped) - check("无既未消费又未丢弃的张量", not orphan, str(orphan[:10])) - print( - " -> 消费 %d 个 / 丢弃 %d 个（MTP blk.%d.*）/ 文件共 %d 个" - % (len(set(tensors) & used), len(dropped), M.MTP_BLOCK, len(tensors)) - ) - - print("\n== 4. 阶段 3 kernel 作用域 ==") - all_types = collections.Counter() - for hist in type_hist.values(): - all_types.update(hist) - print( - " 按条目统计：" - + ", ".join("%s x%d" % (tn, c) for tn, c in all_types.most_common()) - ) - blob_types = { - e.gguf: TYPE_NAME.get(int(tensors[e.gguf].tensor_type)) - for e in plan - if e.blob and e.gguf in tensors - } - seen = collections.Counter(blob_types.values()) - print( - " blob 条目源类型：" - + ", ".join("%s x%d" % (tn, c) for tn, c in seen.most_common()) - ) - check( - "阶段 3 v1 需实现的类型集合 = %s" % sorted(seen), - set(seen) == set(M.NATIVE_BLOB_TYPES), - "缺 %s / 多 %s" - % (set(M.NATIVE_BLOB_TYPES) - set(seen), set(seen) - set(M.NATIVE_BLOB_TYPES)), - ) - check( - "IQ4_* 已被 v1 稠密化例外排除", - not ({"IQ4_NL", "IQ4_XS"} & set(seen)), - str(sorted(seen)), - ) - giB = 2**30 - total = sum(int(t.n_bytes) for t in reader.tensors) - blob_src = {e.gguf for e in plan if e.blob and e.gguf in tensors} - dense_src = {e.gguf for e in plan if not e.blob and e.gguf in tensors} - blob_src - b = sum(int(tensors[n].n_bytes) for n in blob_src) - d_src = sum(int(tensors[n].n_bytes) for n in dense_src) - drop = sum(int(t.n_bytes) for n, t in tensors.items() if n in dropped) - print( - " -> 文件 %.3f GiB = 逐字节 blob %.3f（%d 个） + 稠密化源 %.3f（%d 个）" - " + MTP 丢弃 %.3f" - % (total / giB, b / giB, len(blob_src), d_src / giB, len(dense_src), drop / giB) - ) - # 稠密化条目的显存 = InfiniLM 元素数 x 2B（按行段拆分的条目只算自己那段） - dense_bf16 = sum(prod(e.shape) * 2 for e in plan if not e.blob) - budget = (b + dense_bf16) / giB - print( - " -> v1 显存预算：blob %.3f + 稠密化 BF16 %.3f = %.3f GiB" - % (b / giB, dense_bf16 / giB, budget) - ) - check( - "v1 权重预算 <= 24.0 GiB（单卡 5090 32.6 GiB 留 KV 余量）", - budget <= 24.0, - "%.3f GiB" % budget, - ) - # 阶段 6 复利：IQ4 上原生 kernel 后再省；emb/lm_head 上 kernel 再省 2.51 GiB - st6 = budget - dense_iq_bf16(plan, tensors, gguf_types, prod) / giB - emb_out_blob = int(tensors["token_embd.weight"].n_bytes) + int( - tensors["output.weight"].n_bytes - ) - emb_out_bf16 = sum( - prod(e.shape) * 2 - for e in plan - if not e.blob and e.gguf in ("token_embd.weight", "output.weight") - ) - st6b = st6 - (emb_out_bf16 - emb_out_blob) / giB - print( - " -> 阶段 6：IQ4 原生 kernel %.3f GiB；再 emb/lm_head 原生 %.3f GiB" - % (st6, st6b) - ) - check( - "阶段 6 预算单调下降", - st6b < st6 < budget, - "%.3f / %.3f / %.3f" % (st6b, st6, budget), - ) - check( - "阶段 6 目标态 <= 20.5 GiB（相对路线 A 的 26.6 GiB 权重）", - st6b <= 20.5, - "%.3f GiB" % st6b, - ) - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--gguf", default=DEFAULT_GGUF) - ap.add_argument( - "--skip-min", action="store_true", help="跳过需要 infinilm 的框架侧检查" - ) - ap.add_argument("--engine-device", default="cpu") - a = ap.parse_args() - - if not a.skip_min: - framework_side(a.engine_device) - gguf_side(a.gguf) - - print("\n== 结果：%d PASS / %d FAIL ==" % (_PASS, _FAIL)) - return 0 if _FAIL == 0 else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/gguf_routeb_stage2_check.py b/scripts/gguf_routeb_stage2_check.py deleted file mode 100644 index eb99aaeb9..000000000 --- a/scripts/gguf_routeb_stage2_check.py +++ /dev/null @@ -1,360 +0,0 @@ -#!/usr/bin/env python3 -""" -InfiniLM 路线 B —— 阶段 2 验收（执行方案 §6.3 判据 1–3） - -拿 mini8 产物（8 层 / 121 条目 / blob 61 + 稠密 60）在**新写的 -GGUFBlockQuantization** 上走一遍「构造 -> 键对账 -> 加载 -> 首次 forward」。 -每条判据都能独立失败，不是「能加载」的同义反复： - - 1. 构造：C++ 侧每个 Linear 都用自己的 checkpoint stem 查类型表。stem 拼错 / - 融合组没登记 / 表外 ggml type -> resolve() 抛错，构造直接失败。 - 所以「构造通过」= 所有被查询的 stem 都恰好命中 1 个候选。 - 2. 键双向 diff：引擎 state_dict_keyname() 与产物 index 的张量名必须完全相等。 - 3. 逐键 shape 对账：blob 必须是 [out, row_bytes]，(block_size, type_size) 直接从 - gguf-py 的 GGML_QUANT_SIZES 取（**独立于 gguf.cpp 里那份常量表**）——两侧谁算 - 窄了/算宽了都会在下层的 load_no_sync 里炸，这里先炸出来，报错更好读。 - 4. 加载：load_model_state_dict_by_file 末尾的 check_parameters 对缺键/多键直接 - raise，等于框架替我们做 strict=False 的兜底审查（判据 1）。 - 5. 首次 forward：blob Linear 必须真的进了 linear_gguf 并返回（判据 3：没有静默 - 回落稠密 GEMM）。阶段 2 时这里期望的是抛「阶段 3 实现」占位，3.2 落地后 - 期望反过来：日志里出现带 M/N/K/ggml_type/row_bytes 的契约行，且 row_bytes - 用 gguf-py 的 (block_size, type_size) 独立重算相等。整模端到端（generate） - 由 scripts/gguf_routeb_stage3_check.py 覆盖：forward_raw 的 python 签名不暴 - 露 mamba_*_state_indices，GDN 模型走完 in_proj 后会在下游 conv1d 里因可选 - 入参为空抛 bad_optional_access —— 上游 API 缺口，与 GGUF 无关。 - -用法： - source /home/liuxd/InfiniLM/scripts/gguf_routeb_env.sh - /usr/bin/python3 scripts/gguf_routeb_stage2_check.py [--device cuda:0] [--no-forward] -退出码 0 = 全部 PASS。 -""" - -from __future__ import annotations - -import argparse -import collections -import ctypes -import json -import os -import re -import sys -import traceback - -_HERE = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, _HERE) -sys.path.insert( - 0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py") -) - -DEFAULT_MODEL = "/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8" -BLOB_SUFFIX = "weight_bytes" -# 与 csrc/layers/quantization/gguf.cpp 里那条诊断日志的格式对应 -BLOB_RE = re.compile( - r"linear_gguf: 首个 blob 前向 (\S+) — M=(\d+) N=(\d+) K=(\d+) " - r"ggml_type=(\d+) row_bytes=(\d+)" -) -MAX_DECODE_M = 8 # kMaxDecodeM：<=8 走 gemv，>8 走 prefill（阶段 3.3 起不再是上限） -PROMPT_TOKENS = 3 # 下面 forward_raw 喂的 token 数，用来核对契约行的 M - -_PASS = 0 -_FAIL = 0 - - -def check(name, ok, detail=""): - global _PASS, _FAIL - if ok: - _PASS += 1 - print(" PASS %s" % name) - else: - _FAIL += 1 - print(" FAIL %s%s" % (name, ("\n %s" % detail) if detail else "")) - return ok - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--model-path", default=DEFAULT_MODEL) - ap.add_argument("--device", default="cuda:0") - ap.add_argument("--no-forward", action="store_true", help="跳过首次 forward 判据") - a = ap.parse_args() - - import infinicore - from gguf.constants import GGML_QUANT_SIZES - from infinilm.cache import StaticKVCacheConfig - from infinilm.distributed import DistConfig - from infinilm.infer_engine import InferEngine - from infinilm.modeling_utils import load_model_state_dict_by_file - from safetensors import safe_open - - # ---------------------------------------------------------------- 0. config - print("\n== 0. 产物 config.json ==") - with open(os.path.join(a.model_path, "config.json")) as f: - cfg = json.load(f) - qc = cfg.get("quantization_config") or {} - check( - "quantization_config 在顶层且 quant_method=gguf", - qc.get("quant_method") == "gguf", - "qc keys=%s" % sorted(qc), - ) - table = qc.get("ggml_types") or {} - check("类型表非空（%d 条）" % len(table), bool(table)) - bs_ts = {int(t): (int(v[0]), int(v[1])) for t, v in GGML_QUANT_SIZES.items()} - ids = sorted({v for v in table.values() if isinstance(v, int)}) - check( - "表内 type id 都能从 gguf-py 查出 (block_size, type_size)：%s" % ids, - all(i in bs_ts for i in ids), - str([i for i in ids if i not in bs_ts]), - ) - - with open(os.path.join(a.model_path, "model.safetensors.index.json")) as f: - weight_map = json.load(f)["weight_map"] - n_blob = sum(1 for v in table.values() if isinstance(v, int)) - print( - " -> 类型表 %d 条：blob %d / 稠密 %d；产物 index %d 个张量；key_prefix='%s'" - % ( - len(table), - n_blob, - len(table) - n_blob, - len(weight_map), - qc.get("key_prefix"), - ) - ) - # 溯源：表键有两种历史形态。新规则（§6.0 纠正 2）= 张量名原文（与产物 index 同名）； - # 旧规则 = 去前缀的相对名且 blob 归一成 .weight（与 index 不同名）。两者 C++ 都能 - # 命中（裁前缀时 key_prefix 缺失就取 ""，探键时 weight_bytes / weight 都探）， - # 但必须知道眼下这份产物是哪一种，不然对不上时会查错方向。 - n_ident = len(set(table) & set(weight_map)) - print( - " -> 表键形态：%d/%d 条与产物张量名同名（新规则），其余 %d 条为相对名或前缀外键" - % (n_ident, len(table), len(table) - n_ident) - ) - - # ------------------------------------------------------------- 1. 构造引擎 - print("\n== 1. 用 GGUFBlockQuantization 构造引擎（device=%s）==" % a.device) - # infinicore.device("cuda:0", 0) 会报 “index should not be provided”，带冒号就不能再传 index - dev_spec = ( - infinicore.device(a.device) - if ":" in a.device - else infinicore.device(a.device, 0) - ) - try: - eng = InferEngine( - model_path=a.model_path, - device=dev_spec, - distributed_config=DistConfig(1), - cache_config=StaticKVCacheConfig(max_batch_size=1, max_cache_len=16), - ) - ok, err = True, "" - except Exception as e: # noqa: BLE001 - ok, err = False, "%s: %s" % (type(e).__name__, str(e)[:1200]) - check( - "构造通过（= 被查询的 stem 全部恰好命中 1 个候选，且无 TP/bias 违规）", ok, err - ) - if not ok: - print("\n构造都没过，后面全部跳过\n" + traceback.format_exc()) - return 1 - check( - "引擎确实走 GGUF 方案", - (eng.hf_config.get("quantization_config") or {}).get("quant_method") == "gguf", - ) - - # --------------------------------------------------------- 2. 键双向 diff - print("\n== 2. 引擎参数键 vs 产物张量名 ==") - keys = list(eng.state_dict_keyname()) - extra = sorted(set(keys) - set(weight_map)) - missing = sorted(set(weight_map) - set(keys)) - check( - "产物有、引擎不要（多键 -> strict=False 下静默丢权重）", - not extra, - str(extra[:12]), - ) - check("引擎要、产物没有（缺键 -> 保持随机初始化）", not missing, str(missing[:12])) - check( - "键数一致（引擎 %d / 产物 %d）" % (len(keys), len(weight_map)), - len(keys) == len(weight_map), - ) - - # ---------------------------------------------- 3. 逐键 dtype / shape 对账 - print("\n== 3. 逐键 shape 对账（blob 行字节独立重算）==") - sd_keys = set(keys) - meta = {} - for fn in sorted(set(weight_map.values())): - with safe_open(os.path.join(a.model_path, fn), framework="pt") as f: - for k in f.keys(): - if k in sd_keys: - meta[k] = ( - f.get_slice(k).get_dtype(), - list(f.get_slice(k).get_shape()), - ) - eng_sd = eng.state_dict()[0] - - # 照抄 C++ GGUFBlockQuantization::resolve() 的查表语义：表键 = 产物名裁掉 - # 已声明的 key_prefix（未声明则为 ""，即保留原样），探 stem+"weight_bytes" 与 - # stem+"weight" 两个候选。引擎侧的绝对键 = 模型参数路径，可能与表键不同形， - # 所以这里按候选集查而不是 table[k] 直查（命中数 != 1 算 FAIL，不让脚本 KeyError）。 - MOD_PREFIX = "model.language_model." - W_BLOB = "." + BLOB_SUFFIX - - def table_hits(k): - cands = {k, k[: -len(W_BLOB)] + ".weight" if k.endswith(W_BLOB) else k} - for base in list(cands): - if base.startswith(MOD_PREFIX): - cands.add(base[len(MOD_PREFIX) :]) - declared = qc.get("key_prefix") or "" - for base in list(cands): - if declared and base.startswith(declared): - cands.add(base[len(declared) :]) - return sorted(c for c in cands if c in table) - - bad_shape, bad_dtype, n_blob_eng, n_table_form = [], [], 0, collections.Counter() - for k in sorted(sd_keys & set(meta)): - e_shape = [int(x) for x in eng_sd[k].shape] - if e_shape != list(meta[k][1]): - bad_shape.append("%s: 引擎 %s vs 产物 %s" % (k, e_shape, meta[k][1])) - if k.endswith("." + BLOB_SUFFIX): - n_blob_eng += 1 - if "U8" not in str(eng_sd[k].dtype).upper() or meta[k][0] != "U8": - bad_dtype.append( - "%s: 引擎 %s / 产物 %s" % (k, eng_sd[k].dtype, meta[k][0]) - ) - hits = table_hits(k) - if len(hits) != 1: - bad_shape.append( - "%s: 类型表命中 %d 个候选 %s（C++ 会抛或静默走稠密）" - % (k, len(hits), hits[:4]) - ) - continue - n_table_form["与张量名同名" if hits[0] == k else "相对名/归一后缀"] += 1 - _bs, ts = bs_ts[int(table[hits[0]])] - if e_shape and ts and e_shape[-1] % ts: - bad_shape.append( - "%s: row_bytes=%d 不是 type_size %d 的整数倍" % (k, e_shape[-1], ts) - ) - check( - "%d 个 blob 键在引擎侧与产物侧都是 U8" % n_blob_eng, - not bad_dtype, - str(bad_dtype[:6]), - ) - check( - "全部 %d 键 shape 逐字相等（blob 为 [out, row_bytes]）" % len(sd_keys), - not bad_shape, - str(bad_shape[:8]), - ) - print( - " -> %d 个 blob 命中的表键形态：%s" - % ( - n_blob_eng, - ", ".join("%s x%d" % kv for kv in n_table_form.most_common()) or "无", - ) - ) - - # ----------------------------------------------------------------- 4. 加载 - print("\n== 4. 加载（末尾 check_parameters 会对缺/多键抛错 = 判据 1）==") - try: - load_model_state_dict_by_file(eng, a.model_path, dtype=eng.dtype) - ok, err = True, "" - except Exception as e: # noqa: BLE001 - ok, err = False, "%s: %s" % (type(e).__name__, str(e)[:1200]) - check("%d 个条目全部装载完毕" % len(weight_map), ok, err) - - # ------------------------------------------------------- 5. 首次 forward - if a.no_forward: - print("\n== 5. 跳过（--no-forward）==") - else: - print( - "\n== 5. 首个 blob Linear 必须进 linear_gguf 并返回（判据 3：不静默回落稠密）==" - ) - import torch - - def to_dev(t): - return infinicore.from_torch( - t.cuda(0) if a.device.startswith("cuda") else t - ) - - ids = to_dev(torch.tensor([[114, 5, 7]], dtype=torch.int32)) - # qwen3_5 是 mrope（position_id_axes=3），position_ids 的轴序在 C++ 侧 - # 只要求最后一维是 seq，这里按 [axes, seq] / [seq] 两种形状各试一次， - # 目的是越过入参校验走到第一个 Linear —— 判据只看那里抛的是什么。 - cands = [ - to_dev(torch.tensor([[0, 1, 2], [0, 1, 2], [0, 1, 2]], dtype=torch.int32)), - to_dev(torch.tensor([[0, 1, 2]], dtype=torch.int32)), - ] - - # RankWorker 会把工作线程里的异常换个文案再抛一次（python 侧只看到 - # “RankWorker is closing”），真实抛出只落在 spdlog 里。实测 spdlog 走的是 - # **stdout**（把 2 单独分流到文件后 “linear_gguf” 那条 [error] 仍留在 - # stdout），所以 fd 1、2 都得用 memfd 接住（沙箱里 /tmp 只读）。 - def open_cap(): - try: - return os.memfd_create("stage2_log") - except AttributeError: - return os.open( - os.path.join(_HERE, ".stage2_log.tmp"), - os.O_RDWR | os.O_CREAT | os.O_TRUNC, - 0o600, - ) - - libc = ctypes.CDLL(None) - caps = {fd: open_cap() for fd in (1, 2)} - saved = {fd: os.dup(fd) for fd in caps} - msgs = [] - try: - # 先把手头的正常输出推完再换管道：否则 step 4 的 PASS 还躺在 python - # 的块缓冲里，换完才被 flush，会打到 memfd 里而不在日志文件中。 - sys.stdout.flush() - sys.stderr.flush() - for fd, mem in caps.items(): - os.dup2(mem, fd) - for pos in cands: - try: - eng.forward_raw(input_ids=ids, position_ids=pos) - msgs.append("<没抛异常：blob 被当成稠密权重跑了！>") - break - except Exception as e: # noqa: BLE001 - msgs.append( - "%s: %s" - % (type(e).__name__, str(e).strip().splitlines()[0][:200]) - ) - finally: - libc.fflush(None) # C++ 侧重定向到文件时是块缓冲，不冲读不到 - sys.stdout.flush() - sys.stderr.flush() - for fd, mem in caps.items(): - os.fsync(mem) - os.dup2(saved[fd], fd) - os.close(saved[fd]) - captured = "" - for mem in caps.values(): - os.lseek(mem, 0, os.SEEK_SET) - captured += os.read(mem, 1 << 20).decode("utf-8", "replace") - os.close(mem) - line = next((ln for ln in captured.splitlines() if "linear_gguf" in ln), "") - m = BLOB_RE.search(line) - if not m: - check( - "首个 blob Linear 进入 linear_gguf 并返回（未回落稠密）", - False, - "python: %s\n 日志尾部: %s" - % (" | ".join(msgs), captured[-600:]), - ) - else: - M, N, K, tid, row_bytes = [int(m.group(i)) for i in range(2, 7)] - check("首个 blob Linear 进入 linear_gguf 并返回（未回落稠密）", True) - # 只留 linear_gguf 之后的部分：spdlog 前缀占掉大半行，按整行截断会把张量名切掉 - print(" %s" % line[line.find("linear_gguf") :].strip()) - bs, ts = bs_ts[tid] - # 阶段 3.3 前这里评的是“M <= 8”（当时的 decode 护栏）；现在 M 的唯一 - # 契约是“等于本次喂进去的 token 数”，大了小了都算错。 - check( - "契约行自洽：M=%d 等于 prompt token 数 %d 且 row_bytes=%d == (K/%d)*%d" - % (M, PROMPT_TOKENS, row_bytes, bs, ts), - M == PROMPT_TOKENS and row_bytes == (K // bs) * ts, - "type=%d (block_size, type_size)=(%d,%d)" % (tid, bs, ts), - ) - - print("\n== 结果：%d PASS / %d FAIL ==" % (_PASS, _FAIL)) - return 0 if _FAIL == 0 else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/gguf_routeb_stage3_check.py b/scripts/gguf_routeb_stage3_check.py deleted file mode 100644 index c5b10d7c1..000000000 --- a/scripts/gguf_routeb_stage3_check.py +++ /dev/null @@ -1,361 +0,0 @@ -#!/usr/bin/env python3 -""" -InfiniLM 路线 B —— 阶段 3 端到端验收（执行方案 §7.1 判据 4/5） - -拿 mini8 产物在**量化形态**下真跑一遍 generate，逐条判据独立可失败： - - 1. PagedKVCacheConfig + attention_backend="paged-attn" 的引擎能构造并加载 121 条目。 - （必须 paged：Qwen3NextCausalConv1D::forward 取 mamba_metadata 的三个 - optional.value()，而 forward_raw 的 python 签名不暴露 - mamba_*_state_indices —— 上游 API 缺口，与 GGUF 无关，见 §7.2 备注。） - 2. **prefill 正例**：prompt 长度 12（> kMaxDecodeM=8）的 generate 必须跑完。阶段 3.3 - 之前这里是必抛「超过 decode kernel 的上限」，现在反过来：抛就算 FAIL。 - 3. 日志里出现「首个 blob 前向 …」契约行，且 **M 等于 prompt 长度**（证明整个 - 批量一次进了 kernel、没被拆开也没回落），row_bytes 用 gguf-py 的 - (block_size, type_size) 独立重算相等。 - 4. 贪心（top_k=1 / temperature=0）两次同 prompt 结果逐字相同 —— 说明 kernel - 没有 NaN/不确定行为（数值对不对是阶段 4 的比对，这里不比数值）。 - 5. token id 落在词表内。 - 6. **decode 回归**：prompt 长度 4（<= kMaxDecodeM）仍走 gemv、仍跑完 —— 撤护栏不 - 许把已经能用的短 prompt 路径弄坏。 - 7. （--count-blob-calls）把自己在 gdb 下重跑一遍，用断点命中次数证明 - 「每一步、每个 blob 模块」都进了 kernel：命中数 == 步数 × blob 条目数。 - 这条与路径无关（gemv/prefill 都过同一个 infiniopLinearGguf），少了就是有 - blob 静默回落稠密，多了就是有别的稠密 Linear 被误开。 - -用法： - source /home/liuxd/InfiniLM/scripts/gguf_routeb_env.sh - /usr/bin/python3 scripts/gguf_routeb_stage3_check.py [--new-tokens 8] [--count-blob-calls] -退出码 0 = 全部 PASS。 -""" - -from __future__ import annotations - -import argparse -import ctypes -import os -import re -import subprocess -import sys -import tempfile - -_HERE = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert( - 0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py") -) - -DEFAULT_MODEL = "/home/liuxd/models/Qwen3.8-27B-GGUF-native-mini8" -BLOB_RE = re.compile( - r"linear_gguf: 首个 blob 前向 (\S+) — M=(\d+) N=(\d+) K=(\d+) " - r"ggml_type=(\d+) row_bytes=(\d+)" -) -MAX_DECODE_M = 8 # kMaxDecodeM：<=8 走 gemv，>8 走 prefill（两条路径同一个谓词） -PREFILL_M = 12 # > MAX_DECODE_M：阶段 3.3 的 prefill 正例（旧行为是必抛） -DECODE_M = 4 # <= MAX_DECODE_M：decode 回归用例 - -_PASS = 0 -_FAIL = 0 - - -def check(name, ok, detail=""): - global _PASS, _FAIL - if ok: - _PASS += 1 - print(" PASS %s" % name) - else: - _FAIL += 1 - print(" FAIL %s%s" % (name, ("\n %s" % detail) if detail else "")) - return ok - - -# --------------------------------------------------------------- C++ 日志捕获 -def _open_cap(): - try: - return os.memfd_create("stage3_log") - except AttributeError: - path = os.path.join(tempfile.gettempdir(), ".stage3_log.tmp") - try: - return os.open(path, os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600) - except OSError: - return os.open( - os.path.join(_HERE, ".stage3_log.tmp"), - os.O_RDWR | os.O_CREAT | os.O_TRUNC, - 0o600, - ) - - -class capture: - """把 fd 1/2 换到内存文件，用于读 spdlog 的输出。 - - RankWorker 会把工作线程里的异常换个文案再抛一次（python 侧只看到 - “RankWorker …”），真实抛出点只落在 spdlog 里；实测 spdlog 走 stdout， - 所以 1、2 两个 fd 都得接。 - """ - - def __enter__(self): - sys.stdout.flush() - sys.stderr.flush() - libc = ctypes.CDLL(None) - self._libc = libc - self.captured = "" - self._caps = {fd: _open_cap() for fd in (1, 2)} - self._saved = {fd: os.dup(fd) for fd in self._caps} - for fd, mem in self._caps.items(): - os.dup2(mem, fd) - return self - - def __exit__(self, *exc): - self._libc.fflush(None) # C++ 侧块缓冲，不冲就读不到 - sys.stdout.flush() - sys.stderr.flush() - for fd, mem in self._caps.items(): - try: - os.fsync(mem) - except OSError: - pass - os.lseek(mem, 0, os.SEEK_SET) - self.captured += os.read(mem, 1 << 22).decode("utf-8", "replace") - os.dup2(self._saved[fd], fd) - os.close(self._saved[fd]) - os.close(mem) - return False - - -# ------------------------------------------------------------------ gdb 计数 -def count_blob_calls(inner_argv): - """在 gdb 下重跑本脚本（inner_argv 已带 --route-b-inner），读断点命中次数。""" - script = os.path.join(tempfile.gettempdir(), "stage3_count.gdb") - try: - with open(script, "w") as f: - f.write( - "set pagination off\nset confirm off\n" - "set breakpoint pending on\n" - "break infiniopLinearGguf\ncommands\nsilent\ncontinue\nend\n" - 'run\nprintf "\\n===BPSTAT===\\n"\ninfo breakpoints\n' - ) - except OSError: - script = os.path.join(_HERE, ".stage3_count.gdb") - with open(script, "w") as f: - f.write( - "set pagination off\nset confirm off\n" - "set breakpoint pending on\n" - "break infiniopLinearGguf\ncommands\nsilent\ncontinue\nend\n" - 'run\nprintf "\\n===BPSTAT===\\n"\ninfo breakpoints\n' - ) - cmd = ["gdb", "-q", "-batch", "-x", script, "--args", sys.executable] + inner_argv - print(" -> %s" % " ".join(cmd[:8])) - p = subprocess.run(cmd, capture_output=True, text=True) - tail = p.stdout + p.stderr - m = re.search(r"breakpoint already hit (\d+) times", tail) - return int(m.group(1)) if m else None, tail - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--model-path", default=DEFAULT_MODEL) - ap.add_argument("--new-tokens", type=int, default=8) - ap.add_argument("--num-blocks", type=int, default=16) - ap.add_argument("--block-size", type=int, default=256) - ap.add_argument("--count-blob-calls", action="store_true") - ap.add_argument( - "--route-b-inner", - action="store_true", - help="内部用：gdb 子进程模式，只做前 6 条判据", - ) - a, _unknown = ap.parse_known_args() - - import json - - import infinicore - from gguf.constants import GGML_QUANT_SIZES - from infinilm.cache import PagedKVCacheConfig - from infinilm.distributed import DistConfig - from infinilm.infer_engine import GenerationConfig, InferEngine - from infinilm.modeling_utils import load_model_state_dict_by_file - - with open(os.path.join(a.model_path, "config.json")) as f: - cfg = json.load(f) - table = (cfg.get("quantization_config") or {}).get("ggml_types") or {} - n_blob = sum(1 for v in table.values() if isinstance(v, int)) - text_cfg = ( - cfg.get("text_config") if isinstance(cfg.get("text_config"), dict) else cfg - ) - vocab = int(text_cfg.get("vocab_size") or 0) - - def build(): - return InferEngine( - model_path=a.model_path, - device=infinicore.device("cuda:0"), - distributed_config=DistConfig(1), - cache_config=PagedKVCacheConfig( - a.num_blocks, a.block_size, max_batch_size=1 - ), - attention_backend="paged-attn", - ) - - # ------------------------------------------------------------- 1. 构造加载 - print( - "\n== 1. paged 引擎构造 + 加载（blob %d / 稠密 %d）==" - % (n_blob, len(table) - n_blob) - ) - try: - eng = build() - ok, err = True, "" - except Exception as e: # noqa: BLE001 - ok, err = False, "%s: %s" % (type(e).__name__, str(e)[:1200]) - check("构造通过（PagedKVCacheConfig + paged-attn）", ok, err) - if not ok: - return 1 - check( - "has_mamba_cache 且 enable_paged_attn（GDN 模型只能走这条路）", - eng.has_mamba_cache and eng.enable_paged_attn, - ) - try: - load_model_state_dict_by_file(eng, a.model_path, dtype=eng.dtype) - ok, err = True, "" - except Exception as e: # noqa: BLE001 - ok, err = False, "%s: %s" % (type(e).__name__, str(e)[:1200]) - check("权重装载完毕", ok, err) - if not ok: - return 1 - - def do_generate(tokens): - ids = infinicore.from_list([tokens], dtype=infinicore.int64) - out = eng.generate( - ids, - GenerationConfig( - max_new_tokens=a.new_tokens, - temperature=0.0, - top_k=1, - top_p=1.0, - eos_token_id=None, - stop_on_eos=False, - ), - ) - return [int(x.to_numpy().reshape(-1)[0]) for x in out] - - # 成功的 generate 次数；每完成一次 = 1 次 prefill + (new_tokens-1) 次 decode - # = new_tokens 个前向步，每步每个 blob 各进 kernel 一次（判据 7 的期望值）。 - done_generates = 0 - - def one_generate(tokens): - nonlocal done_generates - toks = do_generate(tokens) - done_generates += 1 - return toks - - # ------------------------------------- 2/3/4/5. prefill 正例（prompt > decode 上限） - print( - "\n== 2-5. prefill：prompt=%d token（> kMaxDecodeM=%d）==" - % (PREFILL_M, MAX_DECODE_M) - ) - pre_prompt = list(range(100, 100 + PREFILL_M)) - with capture() as cap: - try: - toks1 = one_generate(pre_prompt) - perr = "" - except BaseException as e: # noqa: BLE001 - toks1, perr = ( - None, - "%s: %s" % (type(e).__name__, str(e).strip().splitlines()[:1]), - ) - log1 = cap.captured - check( - "prefill generate 走完 %d 步（M=%d 不再抛）" % (a.new_tokens, PREFILL_M), - toks1 is not None, - perr + "\n 日志尾部: " + log1[-500:], - ) - if toks1 is None: - print("\n== 结果：%d PASS / %d FAIL ==" % (_PASS, _FAIL)) - return 1 - print(" tokens=%s" % toks1) - check( - "token id 落在词表 [0,%d) 内" % vocab, - not vocab or all(0 <= t < vocab for t in toks1), - ) - - toks2 = one_generate(pre_prompt) - check("贪心两次结果逐字相同", toks1 == toks2, "%s vs %s" % (toks1, toks2)) - - m = BLOB_RE.search(log1) - check( - "日志出现 blob 前向契约行（= blob 没被当稠密权重跑）", - bool(m), - "捕获 %d 字节，未见 linear_gguf 行" % len(log1), - ) - if m: - key, M, N, K, tid, row_bytes = ( - m.group(1), - *[int(m.group(i)) for i in range(2, 7)], - ) - print( - " %s — M=%d N=%d K=%d ggml_type=%d row_bytes=%d" - % (key, M, N, K, tid, row_bytes) - ) - bs, ts = GGML_QUANT_SIZES[tid] - # 契约行是进 kernel 的第一个 blob，而第一个 blob 就在 prompt 的 prefill 里。 - # M 必须等于 prompt 长度：小了就是上层把 prompt 拆碎了/没走 prefill。 - check( - "契约行 M=%d 等于 prompt 长度 %d（整批进 kernel）" % (M, PREFILL_M), - M == PREFILL_M, - "M=%d" % M, - ) - check( - "该批只能由 prefill 路径处理（M=%d > kMaxDecodeM=%d）" % (M, MAX_DECODE_M), - M > MAX_DECODE_M, - "M=%d" % M, - ) - check( - "契约行 row_bytes == (K/%d)*%d 自洽" % (bs, ts), - row_bytes == (K // int(bs)) * int(ts), - "row_bytes=%d 期望=%d" % (row_bytes, (K // int(bs)) * int(ts)), - ) - - # ------------------------------------------- 6. decode 回归（短 prompt 仍可用） - print( - "\n== 6. decode 回归：prompt=%d token（<= %d，仍走 gemv）==" - % (DECODE_M, MAX_DECODE_M) - ) - dec_prompt = list(range(300, 300 + DECODE_M)) - try: - toks3 = one_generate(dec_prompt) - err3 = "" - except BaseException as e: # noqa: BLE001 - toks3, err3 = None, "%s: %s" % (type(e).__name__, str(e).strip()[:200]) - check( - "短 prompt 用例走完 %d 步（撤护栏未弄坏 gemv 路径）" % a.new_tokens, - toks3 is not None, - err3, - ) - if toks3 is not None: - print(" tokens=%s" % toks3) - - # --------------------------------------------------- 7. 断点命中数（可选） - if a.count_blob_calls and not a.route_b_inner: - print("\n== 7. gdb 断点计数：每步 × 每个 blob ==") - inner = ( - [os.path.abspath(sys.argv[0])] - + [x for x in sys.argv[1:] if x != "--count-blob-calls"] - + ["--route-b-inner"] - ) - n, tail = count_blob_calls(inner) - steps = re.search(r"INNER_STEPS=(\d+)", tail) - steps = int(steps.group(1)) if steps else None - expect = steps * n_blob if steps else None - check( - "infiniopLinearGguf 命中 %s 次 == 步数 %s × blob %d = %s" - % (n, steps, n_blob, expect), - n is not None and n == expect, - "实际 %s / 期望 %s\n 子进程输出尾部: %s" % (n, expect, tail[-600:]), - ) - elif a.route_b_inner: - # 子进程里：把实际完成的前向步数报给外层。每次 generate = 1 次 prefill + - # (max_new_tokens-1) 次 decode = max_new_tokens 步；本脚本一共跑 3 次。 - print("INNER_STEPS=%d" % (done_generates * a.new_tokens)) - - print("\n== 结果：%d PASS / %d FAIL ==" % (_PASS, _FAIL)) - return 0 if _FAIL == 0 else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/gguf_routeb_tokenizer_check.py b/scripts/gguf_routeb_tokenizer_check.py deleted file mode 100755 index 29eca2cdf..000000000 --- a/scripts/gguf_routeb_tokenizer_check.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python3 -"""Build canonical input IDs with llama.cpp and compare the packaged tokenizer.""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -import urllib.error -import urllib.request - - -def load_cases(path: str, selected: set[str]) -> list[dict]: - cases = [] - with open(path, encoding="utf-8") as f: - for line in f: - if line.strip(): - item = json.loads(line) - if not selected or item["id"] in selected: - cases.append(item) - missing = selected - {x["id"] for x in cases} - if missing: - raise ValueError("unknown case ids: %s" % sorted(missing)) - return cases - - -def post_json(url: str, body: dict, timeout: int = 30) -> dict: - request = urllib.request.Request( - url, - data=json.dumps(body, ensure_ascii=False).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=timeout) as response: - return json.load(response) - except urllib.error.HTTPError as exc: - detail = exc.read().decode("utf-8", "replace") - raise RuntimeError("HTTP %d: %s" % (exc.code, detail[:1000])) from exc - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--model-path", required=True) - ap.add_argument("--prompts", required=True) - ap.add_argument("--server", default="http://127.0.0.1:18080") - ap.add_argument("--case-ids", default="") - ap.add_argument("--out", required=True) - args = ap.parse_args() - - selected = {x for x in args.case_ids.split(",") if x} - cases = load_cases(args.prompts, selected) - - from transformers import AutoTokenizer - - common = {"local_files_only": True, "trust_remote_code": False} - tok_default = AutoTokenizer.from_pretrained(args.model_path, **common) - try: - tok_fixed = AutoTokenizer.from_pretrained( - args.model_path, fix_mistral_regex=True, **common - ) - fixed_error = None - except Exception as exc: # compatibility with older transformers - tok_fixed = None - fixed_error = "%s: %s" % (type(exc).__name__, exc) - - results = [] - default_ok = fixed_ok = True - for case in cases: - llama = post_json( - args.server.rstrip("/") + "/tokenize", - { - "content": case["prompt"], - "add_special": False, - "parse_special": True, - "with_pieces": False, - }, - )["tokens"] - llama = [int(x) for x in llama] - local_default = [ - int(x) for x in tok_default.encode(case["prompt"], add_special_tokens=False) - ] - local_fixed = ( - None - if tok_fixed is None - else [ - int(x) - for x in tok_fixed.encode(case["prompt"], add_special_tokens=False) - ] - ) - match_default = llama == local_default - match_fixed = local_fixed is not None and llama == local_fixed - default_ok &= match_default - fixed_ok &= match_fixed - results.append( - { - **case, - "input_ids": llama, - "local_default_ids": local_default, - "local_fixed_ids": local_fixed, - "default_match": match_default, - "fixed_match": match_fixed, - } - ) - print( - "%-10s llama=%3d default=%s fixed=%s" - % ( - case["id"], - len(llama), - match_default, - "NA" if local_fixed is None else str(match_fixed), - ) - ) - - if default_ok: - selected_variant = "default" - elif fixed_ok: - selected_variant = "fix_mistral_regex=True" - else: - selected_variant = None - - output = { - "model_path": os.path.abspath(args.model_path), - "server": args.server, - "add_special": False, - "parse_special": True, - "selected_local_variant": selected_variant, - "default_all_match": default_ok, - "fixed_all_match": fixed_ok, - "fixed_load_error": fixed_error, - "cases": results, - } - os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) - with open(args.out, "w", encoding="utf-8") as f: - json.dump(output, f, ensure_ascii=False, indent=2) - print( - "RESULT default_all=%s fixed_all=%s selected=%s cases=%d" - % (default_ok, fixed_ok, selected_variant, len(results)) - ) - return 0 if selected_variant else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/gguf_routeb_typecensus.py b/scripts/gguf_routeb_typecensus.py deleted file mode 100644 index f676fe82d..000000000 --- a/scripts/gguf_routeb_typecensus.py +++ /dev/null @@ -1,85 +0,0 @@ -import collections -import sys - -sys.path.insert(0, "/home/liuxd/llama.cpp/gguf-py") -from gguf import GGUFReader # noqa: E402 -from gguf.constants import GGMLQuantizationType as QType # noqa: E402 - -GGUF = "/home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf" -r = GGUFReader(GGUF) -T = {t.name: t for t in r.tensors} - - -def tn(name): - return QType(int(T[name].tensor_type)).name - - -per = collections.defaultdict(collections.Counter) -for i in range(64): - full = (i + 1) % 4 == 0 - names = ( - [ - f"blk.{i}.attn_q.weight", - f"blk.{i}.attn_k.weight", - f"blk.{i}.attn_v.weight", - f"blk.{i}.attn_output.weight", - ] - if full - else [ - f"blk.{i}.attn_qkv.weight", - f"blk.{i}.attn_gate.weight", - f"blk.{i}.ssm_out.weight", - ] - ) - names += [ - f"blk.{i}.ffn_gate.weight", - f"blk.{i}.ffn_up.weight", - f"blk.{i}.ffn_down.weight", - ] - for n in names: - per[n.split(".")[2]][tn(n)] += 1 - -print("=== 按张量角色的类型分布（64 层）===") -for k, v in sorted(per.items()): - print(f" {k:14s}", dict(v)) - -print("=== full-attn 层内 q/k/v 类型是否一致（决定融合 blob 能否共用一块 buffer）===") -bad = [ - (i, [tn(f"blk.{i}.attn_{x}.weight") for x in ("q", "k", "v")]) - for i in range(3, 64, 4) -] -bad = [b for b in bad if len(set(b[1])) != 1] -print(f" 不一致层数 = {len(bad)} 样例 = {bad[:6]}") - -print("=== ffn gate/up 类型是否一致（决定 GateUp 融合 blob 能否共用一块 buffer）===") -bad2 = [(i, [tn(f"blk.{i}.ffn_{x}.weight") for x in ("gate", "up")]) for i in range(64)] -bad2 = [b for b in bad2 if len(set(b[1])) != 1] -print(f" 不一致层数 = {len(bad2)} 样例 = {bad2[:6]}") - -print("=== GDN 层 attn_qkv / attn_gate / ssm_out 抽样类型 ===") -for i in (0, 1, 2, 4, 62): - print( - " ", - i, - { - s: tn(f"blk.{i}.{s}.weight") - for s in ("attn_qkv", "attn_gate", "ssm_out", "ffn_gate", "ffn_down") - }, - ) - -print("=== 每个 Linear 的 (角色 -> 类型) 逐层矩阵，看同一角色跨层是否稳定 ===") -for role in ( - "attn_q", - "attn_k", - "attn_v", - "attn_output", - "ffn_gate", - "ffn_up", - "ffn_down", -): - c = collections.Counter() - for i in range(64): - n = f"blk.{i}.{role}.weight" - if n in T: - c[tn(n)] += 1 - print(f" {role:12s}", dict(c)) diff --git a/scripts/gguf_to_infinilm.py b/scripts/gguf_to_infinilm.py index 6ffa7e3fc..cb0a3d9af 100644 --- a/scripts/gguf_to_infinilm.py +++ b/scripts/gguf_to_infinilm.py @@ -1,21 +1,13 @@ #!/usr/bin/env python3 -""" -InfiniLM 路线 B —— 阶段 1 打包器：GGUF -> InfiniLM 原生量化产物（执行方案 §5） - - Qwen3.8-27B-UD-Q6_K.gguf -> models/Qwen3.8-27B-GGUF-native/ - config.json + model-0000N-of-0000M.safetensors + index - -铁律（阶段 0 的教训写在这里，别再用第二套定义）： - * 键名 / shape / 哪些走 blob / 哪些稠密化，全部来自 `gguf_mapping.build_plan(REAL)` - + `apply_v1_exceptions()`。本文件**不得**出现第二张表。 - * 反量化一律调 `gguf.quants.dequantize`，禁止自己实现解码。 - * 置换只沿 dim0 整行/整元素搬，块内字节绝不动（§2.7 已证明字节级可行）。 - * 取向：gguf-py 的 `tensor.data` 已经是 [out, in]（量化张量是 [out, row_bytes]）， - 与 InfiniLM 参数同序 ⇒ 全程不转置数据。 - -用法： - source /home/liuxd/InfiniLM/scripts/gguf_routeb_env.sh - python3 scripts/gguf_to_infinilm.py [--dry-run] [--layers 4] [--verify all] +"""Convert a GGUF model into an InfiniLM packed-weight checkpoint. + +All names, shapes, packed/dense choices, and transformations come from +``gguf_mapping``. Dequantization uses ``gguf.quants.dequantize``. Packed data +may only be moved as complete rows; bytes within quantization blocks remain +unchanged. GGUF tensors already match InfiniLM's [out, in] orientation. + +Example: + python3 scripts/gguf_to_infinilm.py --gguf MODEL.gguf --out OUT_DIR """ from __future__ import annotations @@ -30,9 +22,8 @@ _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -sys.path.insert( - 0, os.path.join(os.environ.get("LLAMA_CPP_DIR", "/home/liuxd/llama.cpp"), "gguf-py") -) +if os.environ.get("LLAMA_CPP_DIR"): + sys.path.insert(0, os.path.join(os.environ["LLAMA_CPP_DIR"], "gguf-py")) import gguf_mapping as M # noqa: E402 import gguf_transforms as X # noqa: E402 @@ -42,15 +33,11 @@ from gguf.constants import GGMLQuantizationType as Q # noqa: E402 from gguf.quants import dequantize # noqa: E402 -DEFAULT_GGUF = "/home/liuxd/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q6_K.gguf" -DEFAULT_OUT = "/home/liuxd/models/Qwen3.8-27B-GGUF-native" -DEFAULT_TOKENIZER = "/home/liuxd/models/Qwen3.8-27B-BF16" - TYPE_NAME = {int(v.value): str(v.name) for v in Q} TYPE_ID = {str(v.name): int(v.value) for v in Q} UNQUANTIZED = ("F32", "F16", "BF16") -# 分词器配置文件：词表本身从 GGUF 导出，这些附属文件优先从 --tokenizer-dir 复制。 +# Tokenizer vocabulary comes from GGUF; copy auxiliary files when available. TOKENIZER_FILES = ( "tokenizer_config.json", "chat_template.jinja", @@ -75,8 +62,7 @@ def blk_sizes(type_name: str) -> tuple[int, int]: return int(bs), int(ts) -# safetensors 报的是 GGML 风 dtype 名（BF16/U8），torch 报的是 bfloat16/uint8， -# 不归一就会把 947 个键全判成不符（实测踩过）。 +# Normalize safetensors and torch dtype names before comparison. _DTYPE_ALIAS = {"BF16": "bfloat16", "F16": "float16", "F32": "float32", "U8": "uint8"} @@ -86,16 +72,18 @@ def norm_dtype(s) -> str: # --------------------------------------------------------------------------- -# 源 -> 目标：单一实现 +# Source-to-target conversion # --------------------------------------------------------------------------- def dense_float32(src: np.ndarray, type_name: str, chunk_rows: int) -> np.ndarray: - """源张量的若干行 -> float32 [rows, in]。未量化类型只是换 dtype。""" + """Convert source rows to float32 [rows, in].""" if type_name in UNQUANTIZED: return np.asarray(src, dtype=np.float32) if src.ndim != 2: - raise ValueError("量化源张量应是 [out, row_bytes]，实测 %s" % (src.shape,)) + raise ValueError( + "quantized source must have shape [out, row_bytes], got %s" % (src.shape,) + ) rows = src.shape[0] if rows == 0: return np.zeros((0,), dtype=np.float32) @@ -103,7 +91,7 @@ def dense_float32(src: np.ndarray, type_name: str, chunk_rows: int) -> np.ndarra first = np.asarray(dequantize(src[:chunk_rows], q), dtype=np.float32) if rows <= chunk_rows: return first - # 预分配而不是 parts+concatenate：lm_head（248320×5120）峰值从 ~10 GB 降到 ~5 GB + # Preallocate to bound peak memory for large tensors such as lm_head. out = np.empty((rows,) + first.shape[1:], dtype=np.float32) out[:chunk_rows] = first for i in range(chunk_rows, rows, chunk_rows): @@ -114,13 +102,14 @@ def dense_float32(src: np.ndarray, type_name: str, chunk_rows: int) -> np.ndarra def make_blob(e, t, dims, opt): - """逐字节路径：U8 [out, row_bytes]，只在需要时做整行置换。""" + """Build a U8 [out, row_bytes] tensor, permuting only complete rows.""" bs, ts = blk_sizes(opt.types[t.name]) n_out, n_in = int(e.shape[0]), int(e.shape[1]) rb = M.row_bytes(n_in, bs, ts) if int(t.data.shape[-1]) != rb: raise ValueError( - "%s: 源行字节 %d != 映射表期望 %d" % (e.gguf, int(t.data.shape[-1]), rb) + "%s: source row bytes %d != expected %d" + % (e.gguf, int(t.data.shape[-1]), rb) ) arr = t.data if e.slices: @@ -128,7 +117,7 @@ def make_blob(e, t, dims, opt): arr = arr[s:ep] if int(arr.shape[0]) != n_out: raise ValueError( - "%s: 取段后 %d 行 != 映射表 %d" % (e.gguf, arr.shape[0], n_out) + "%s: sliced rows %d != mapping rows %d" % (e.gguf, arr.shape[0], n_out) ) if M.needs_vperm(e): arr = X.apply_vperm(arr, e, dims, opt.vperm) @@ -136,7 +125,7 @@ def make_blob(e, t, dims, opt): def entry_float32(e, t, dims, opt) -> np.ndarray: - """稠密化条目的 float32 值。单一实现：写盘（make_dense）与自检（比 BF16 位）共用。""" + """Return float32 values shared by dense output and verification.""" tn = opt.types[t.name] src = t.data if e.slices: @@ -154,19 +143,20 @@ def entry_float32(e, t, dims, opt) -> np.ndarray: elif tr in (M.T_DENSE, M.T_NONE): continue else: - raise ValueError("%s: 未知 transform %r" % (e.infinilm, tr)) + raise ValueError("%s: unknown transform %r" % (e.infinilm, tr)) want = tuple(int(x) for x in e.shape) if tuple(arr.shape) != want: if arr.size != prod(want): raise ValueError( - "%s: 变换后 shape %s != 映射表 %s" % (e.infinilm, arr.shape, want) + "%s: transformed shape %s != mapped shape %s" + % (e.infinilm, arr.shape, want) ) - arr = arr.reshape(want) # §2.11 第 5 条：conv1d 补中间维 + arr = arr.reshape(want) # Restore a squeezed singleton convolution dimension. return arr def make_dense(e, t, dims, opt): - """稠密化路径：反量化 / 换 dtype -> float32 -> BF16（cast 交给 torch，不自实现）。""" + """Dequantize or cast through float32, then let torch produce BF16.""" return torch_from(entry_float32(e, t, dims, opt), "bf16") @@ -178,42 +168,35 @@ def torch_from(arr: np.ndarray, dtype): def _is_baked_plus1_norm(name: str) -> bool: - """llama.cpp 转换时已对 norm.weight baked +1 的那些参数（conversion/qwen.py:394， - linear_attn.norm 除外）。与 modeling_utils 的 `_remap_qwen3_5` 加载期 +1 集合一一对应： - input/post_attention_layernorm、self_attn.q/k_norm、最终 model.norm 都以 'norm.weight' 结尾。""" + """Return whether llama.cpp baked a +1 normalization offset into GGUF.""" return name.endswith("norm.weight") and not name.endswith("linear_attn.norm.weight") def build(e, t, dims, opt, dense_all: bool): - """一条映射条目 -> 一个待写盘的张量 + 名称。dense_all 用于 --emit-dense-ref。""" + """Build one output tensor and name; dense_all creates the dense reference.""" e2 = e if dense_all and e.blob: e2 = _as_dense(e) tens = make_blob(e2, t, dims, opt) if e2.blob else make_dense(e2, t, dims, opt) - # dense-ref 的 ssm_out 列序必须从 GGUF 的 tiled 换成 grouped，否则与 blob 路径（运行时 gather）语义不同， - # §8.3 的逐层 cos_sim 对拍就失去意义。稠密 BF16 可以随便换列（不像 blob 跨块要重量化），所以这里直接 permute。 - # vperm=none 时 blob 路径不做运行时 gather，denseref 也必须保持 GGUF 原生列序，二者才同构。 + # Dense-reference output can permute columns directly. Match the packed path, + # which performs the equivalent activation permutation at runtime. if dense_all and e.act_vperm and opt.vperm != "none": n_k, r, hd = dims.lin_k_heads, dims.v_per_k, dims.lin_v_dim out_dim, in_dim = int(e.shape[0]), int(e.shape[1]) if in_dim != n_k * r * hd: raise ValueError( - "%s: in_dim %d != num_k_heads*num_v_per_k*head_dim = %d，无法按头分块置换列" + "%s: input dimension %d != num_k_heads*num_v_per_k*head_dim %d; cannot permute complete heads" % (e.infinilm, in_dim, n_k * r * hd) ) - # [out, in] 解释为 [out, r, n_k, hd]（tiled 序）-> 对调 1,2 轴 -> [out, n_k, r, hd]（grouped 序）-> flatten + # [out, r, n_k, hd] tiled -> [out, n_k, r, hd] grouped -> flatten. tens = ( tens.view(out_dim, r, n_k, hd) .transpose(1, 2) .contiguous() .view(out_dim, in_dim) ) - # dense-ref 版删掉了 quantization_config（见主写盘处），框架按普通 HF 模型加载； - # python 侧 `_remap_qwen3_5`（modeling_utils L808）对**非 gguf** 模型会把 norm 权重 +1 - # （HF 存 delta、C++ 用完整权重的约定）。而 dense-ref 的 norm 值是从 GGUF 原样搬来的 - # **已 baked +1 的完整权重**，再 +1 就变成 2+w（实测使块输入翻倍、级联污染 §8.3）。 - # 故 dense-ref 预存 (w-1)，让加载期 +1 恰好还原成 w，与 blob 路径（gguf=True 不 +1）同构。 - # 集合与 modeling_utils:799 `norm_weight_suffixes` 一致：linear_attn.norm 除外。 + # The dense reference loads through the non-GGUF remap, which adds +1 to + # selected norm weights. Store w-1 so loading reconstructs the baked GGUF w. if dense_all and _is_baked_plus1_norm(e.infinilm): tens = tens - 1 name = M.ckpt_name(e2) @@ -224,7 +207,7 @@ def build(e, t, dims, opt, dense_all: bool): def _as_dense(e): - """blob 条目的“同样内容但稠密化”视图（只给 dense-ref 用，不改原表）。""" + """Return a dense-reference view of a packed entry without changing the plan.""" key = (e.infinilm, e.vperm) v = _DENSE_CACHE.get(key) if v is None: @@ -245,18 +228,17 @@ def _as_dense(e): # --------------------------------------------------------------------------- -# 维度：从 GGUF 元数据推导，并与映射表的 REAL 对账 +# Derive dimensions from GGUF metadata and validate the target profile. # --------------------------------------------------------------------------- def _dec(x) -> float: - """float32 元数据归回十进制字面量（1e-6 而非 9.999999974752427e-07）， - 让 config.json 与 HF 原始 config 逐字符一致。7 位有效数字对 float32 无损。""" + """Render float32 metadata with a stable seven-significant-digit decimal.""" return float("%.7g" % float(x)) def dims_from_gguf(reader) -> M.Dims: - """元数据键名沿用 llama.cpp 标准写法，与审计脚本 E 节实测同一批键。""" + """Derive dimensions from standard llama.cpp GGUF metadata keys.""" g = lambda suffix, idx=0: X.gguf_meta(reader, suffix)[idx] # noqa: E731 n_layers = int(g("block_count")) - int(g("nextn_predict_layers")) inner = int(g("ssm.inner_size")) @@ -279,7 +261,7 @@ def dims_from_gguf(reader) -> M.Dims: vocab=vocab, n_layers=n_layers, interval=int(g("full_attention_interval")), - mrope_section=tuple(sec[:3]), # 丢掉尾 0：§2.11 第 4 条 + mrope_section=tuple(sec[:3]), # InfiniLM consumes three MRoPE sections. rope_theta=_dec(g("rope.freq_base")), partial_rotary_factor=_dec(dim_cnt / head_dim), rms_norm_eps=_dec(g("attention.layer_norm_rms_epsilon")), @@ -288,11 +270,7 @@ def dims_from_gguf(reader) -> M.Dims: def check_dims(d: M.Dims) -> None: - """元数据推导必须与映射表钉死的 REAL 一致，否则说明换了模型还硬套表。 - - float 字段用相对容差：GGUF 存的是 float32，1e-6 读回来是 - 9.999999974e-07，按 == 比会误报（实测本文件的 rms_norm_eps 就撞在这上面）。 - """ + """Reject inputs whose metadata does not match the target model profile.""" diff = [] for f in _DIM_FIELDS: got, want = getattr(d, f.name), getattr(M.REAL, f.name) @@ -303,11 +281,11 @@ def check_dims(d: M.Dims) -> None: diff.append("%s: %r != %r" % (f.name, got, want)) if diff: raise SystemExit( - "GGUF 元数据推导出的维度与 gguf_mapping.REAL 不符：%s\n" - "=> 先按新模型实测重做阶段 0，不要改打包器来迁就。" % diff + "GGUF metadata does not match gguf_mapping.REAL: %s\n" + "Create and validate a model-specific mapping before conversion." % diff ) log( - " rms_norm_eps：GGUF float32 %r -> config 写 HF 十进制 %r" + " rms_norm_eps: GGUF float32 %r -> config decimal %r" % (float(d.rms_norm_eps), M.REAL.rms_norm_eps) ) @@ -318,7 +296,7 @@ def check_dims(d: M.Dims) -> None: # --------------------------------------------------------------------------- -# 分片写出 +# Sharded output # --------------------------------------------------------------------------- @@ -351,7 +329,7 @@ def flush(self) -> None: for k in self.buf: self.weight_map[k] = fname log( - " 写出 %s（%.2f GiB，%d 个张量）" + " wrote %s (%.2f GiB, %d tensors)" % (fname, self.buf_bytes / _GiB, len(self.buf)) ) self.shards[-1] = fname @@ -374,19 +352,19 @@ def finish(self) -> None: indent=1, sort_keys=True, ) - log(" 分片 %d 个，合计 %.3f GiB" % (n, self.total / _GiB)) + log(" %d shards, %.3f GiB total" % (n, self.total / _GiB)) # --------------------------------------------------------------------------- -# 自检 +# Verification # --------------------------------------------------------------------------- def rows_hash(a) -> str: - """把 [rows, cols] 字节阵的**行多重集**压成一个摘要（排序后逐行喂 hash）。 + """Hash the multiset of rows in a [rows, cols] byte array. - 用途：置换过的 blob 不能直接与源逐字节比（那等于拿置换代码自证）， - 但可以无条件断言“产物行集 == 源行集”（置换只是整行搬，不允许改字节）。 + A row permutation cannot be compared positionally, but it must preserve the + complete multiset of packed rows. """ import hashlib @@ -399,11 +377,7 @@ def rows_hash(a) -> str: def dense_bits_check(e, t, dims, opt, prod_t) -> bool: - """BF16 条目的位级校验：逐行块算期望值并与产物对应行块比，峰值内存有界。 - - 上一版直接 `bf16_bits(整块 float32)`，在 lm_head（12.7 亿元素）上把进程 OOM kill 掉了。 - V 头置换 / A_log 是跨行或逐元素语义，不能切块，但这类条目都很小，走全量路径。 - """ + """Verify dense BF16 entries bitwise in bounded row chunks.""" import torch if M.needs_vperm(e): @@ -438,15 +412,15 @@ def dense_bits_check(e, t, dims, opt, prod_t) -> bool: return True -_BIG_ELEMS = 64 * 1024 * 1024 # 切块阈值：一次最多算 64M 元素（float32 峰值 256 MB） +_BIG_ELEMS = 64 * 1024 * 1024 # At most 64M float32 elements per verification chunk. def verify(out_dir: str, plan, tensors, dims, opt, sample) -> int: - """重读产物：全量比键/shape/dtype，分类抽样比字节。返回 FAIL 数。""" + """Reload output, validate metadata, and sample bytes. Return failure count.""" import torch from safetensors import safe_open - log("\n== 自检：重读产物 ==") + log("\n== Verification: reload output ==") bs_files = sorted( f for f in os.listdir(out_dir) @@ -474,76 +448,75 @@ def verify(out_dir: str, plan, tensors, dims, opt, sample) -> int: want[name] = (shape, dt, e) missing = sorted(set(want) - set(got)) extra = sorted(set(got) - set(want)) - for label, keys in (("缺键", missing), ("多键", extra)): + for label, keys in (("missing keys", missing), ("extra keys", extra)): if keys: fails += 1 - log(" FAIL %s %d 个：%s" % (label, len(keys), keys[:6])) + log(" FAIL %s (%d): %s" % (label, len(keys), keys[:6])) else: - log(" PASS 无%s" % label) + log(" PASS no %s" % label) bad = [k for k in set(want) & set(got) if want[k][:2] != got[k]] if bad: fails += 1 log( - " FAIL shape/dtype 不符 %d 个：%s" + " FAIL shape/dtype mismatch (%d): %s" % (len(bad), [(k, want[k][:2], got[k]) for k in sorted(bad)[:4]]) ) else: - log(" PASS 全部 %d 个键的 shape+dtype 与映射表一致" % len(want)) + log(" PASS shape and dtype for all %d keys" % len(want)) - # config.json 的类型表必须与产物张量名**双向逐字相等**：阶段 2 的 C++ 就是拿 - # 这些名字查表决定 blob / 稠密（方案 §6.0 纠正 2），两边对不上会在运行期变成 - # “查不到 key”，那比 shape 错更难查。 + # Type-table keys must exactly match output tensor names in both directions. with open(os.path.join(out_dir, "config.json")) as fp: cfg = json.load(fp) qcfg = cfg.get("quantization_config") or {} table = qcfg.get("ggml_types") or {} if qcfg.get("quant_method") != "gguf": fails += 1 - log( - " FAIL config.json 顶层 quantization_config.quant_method != 'gguf'（或在 text_config 里）" - ) + log(" FAIL top-level quantization_config.quant_method is not 'gguf'") elif qcfg.get("key_prefix") != M.PREFIX: fails += 1 - log(" FAIL config.json 缺 key_prefix=%r（阶段 2 C++ 用它裁表 key）" % M.PREFIX) + log(" FAIL config.json is missing key_prefix=%r" % M.PREFIX) else: - log(" PASS quantization_config 在顶层，key_prefix=%r" % M.PREFIX) + log(" PASS top-level quantization_config with key_prefix=%r" % M.PREFIX) for label, keys in ( - ("类型表缺键", sorted(set(got) - set(table))), - ("类型表多键", sorted(set(table) - set(got))), + ("type-table missing keys", sorted(set(got) - set(table))), + ("type-table extra keys", sorted(set(table) - set(got))), ): if keys: fails += 1 - log(" FAIL %s %d 个：%s" % (label, len(keys), keys[:6])) + log(" FAIL %s (%d): %s" % (label, len(keys), keys[:6])) else: - log(" PASS 无%s（%d 个 key 与张量名逐字相等）" % (label, len(table))) + log(" PASS no %s (%d exact tensor-name matches)" % (label, len(table))) - # 分类抽样（按名排序取首个，可复现）：三种字节路径必须有各自的代表， - # 纯随机抽 3 个会全部落在“未置换 memcpy”上，那样根本测不到置换与切片。 + # Deterministically sample every conversion category rather than relying on + # random samples that may miss permutations and slices. def sel(pred): return sorted(k for k, (_, _, e) in want.items() if pred(e)) cats = [ ( - "blob 未置换", + "packed unchanged", sel(lambda e: e.blob and not e.slices and not M.needs_vperm(e)), ), - ("blob V 置换", sel(lambda e: e.blob and not e.slices and M.needs_vperm(e))), - ("blob 融合切片", sel(lambda e: e.blob and e.slices)), ( - "bf16 反量化", + "packed V permutation", + sel(lambda e: e.blob and not e.slices and M.needs_vperm(e)), + ), + ("packed fused slice", sel(lambda e: e.blob and e.slices)), + ( + "BF16 dequantization", sel(lambda e: not e.blob and not e.slices and not M.needs_vperm(e)), ), - ("bf16 置换+alog", sel(lambda e: not e.blob and M.needs_vperm(e))), - ("bf16 融合切片", sel(lambda e: not e.blob and e.slices)), + ("BF16 permutation and A_log", sel(lambda e: not e.blob and M.needs_vperm(e))), + ("BF16 fused slice", sel(lambda e: not e.blob and e.slices)), ] picks = [c[1][0] for c in cats if c[1]] if sample == "all": picks = sorted(k for k, (_, _, e) in want.items() if e.blob) log( - " 抽样 %d 个：%s" + " sampled %d entries: %s" % ( len(picks), - "全部 blob" + "all packed entries" if sample == "all" else " ".join("%s=%s" % (c, len(v)) for c, v in cats), ) @@ -561,16 +534,18 @@ def sel(pred): checks.append(("shape", False)) elif e.blob: p = prod_t.numpy() - checks.append(("与重建一致", bool(torch.equal(prod_t, ref)))) + checks.append(("matches reconstruction", bool(torch.equal(prod_t, ref)))) if M.needs_vperm(e): - checks.append(("行集与源相同", rows_hash(p) == rows_hash(src))) + checks.append( + ("same source-row multiset", rows_hash(p) == rows_hash(src)) + ) else: - checks.append(("与 GGUF 源逐字节", np.array_equal(p, src))) + checks.append(("byte-identical to GGUF source", np.array_equal(p, src))) else: - # BF16：拿 numpy 的 RNE 位模式比，相当于独立验一次 torch 的 cast + 读写往返 + # Independently compare BF16 bits against NumPy RNE conversion. checks.append( ( - "BF16 位与 numpy RNE 一致", + "BF16 bits match NumPy RNE", dense_bits_check(e, tensors[e.gguf], dims, opt, prod_t), ) ) @@ -582,7 +557,7 @@ def sel(pred): "PASS" if ok else "FAIL", k, str(tuple(int(x) for x in prod_t.shape)), - "，".join("%s=%s" % (n, "Y" if v else "N") for n, v in checks), + ", ".join("%s=%s" % (n, "Y" if v else "N") for n, v in checks), ) ) return fails @@ -597,84 +572,85 @@ def main() -> int: ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - ap.add_argument("--gguf", default=DEFAULT_GGUF) - ap.add_argument("--out", default=DEFAULT_OUT) - ap.add_argument("--tokenizer-dir", default=DEFAULT_TOKENIZER) + ap.add_argument("--gguf", required=True, help="source GGUF file") + ap.add_argument("--out", required=True, help="output checkpoint directory") + ap.add_argument( + "--tokenizer-dir", + default="", + help="optional directory with auxiliary tokenizer files", + ) ap.add_argument( "--dense-iq", action=argparse.BooleanOptionalAction, default=True, - help="v1 把 5 个 IQ4_NL/IQ4_XS 稠密化（阶段 6 上了码本 kernel 后 --no-dense-iq）", + help="convert IQ4_NL/IQ4_XS tensors to dense BF16", ) ap.add_argument( "--dense-embed", action=argparse.BooleanOptionalAction, default=True, - help="v1 恒为 True；--no-dense-embed 需要阶段 6 的 embedding kernel", + help="convert embedding and output head to dense BF16", ) ap.add_argument( "--vperm", choices=("inv", "fwd", "none"), default="inv", - help="V 头 tiled->grouped 方向；阶段 4 A/B 用（§2.7）", + help="value-head permutation direction", ) ap.add_argument( "--emit-dense-ref", metavar="PATH", default=None, - help="额外产出一份全反量化 BF16 版（阶段 4 自洽基准，不部署）", + help="also emit a fully dequantized BF16 reference checkpoint", ) ap.add_argument("--max-shard-gib", type=float, default=4.0) ap.add_argument( - "--chunk-rows", type=int, default=8192, help="反量化分块行数，限制峰值内存" + "--chunk-rows", type=int, default=8192, help="rows per dequantization chunk" ) ap.add_argument( "--layers", type=int, default=None, - help="只打前 N 层，并同步把 config 的 num_hidden_layers 改成 N" - "（产物可直接被框架构造 + 加载，阶段 2/3 用小模型验收用）", + help="convert only the first N layers and update num_hidden_layers", ) ap.add_argument("--verify", choices=("off", "sample", "all"), default="sample") ap.add_argument( "--skip-pack", action="store_true", - help="不重写 23 GiB 权重，只做分词器导出 + 自检（迭代自检逻辑用）", + help="reuse existing weights while refreshing config, tokenizer, and verification", ) ap.add_argument( "--dry-run", action="store_true", - help="全量校验取向/shape/字节数，不写盘（稠密化条目也只算 shape）", + help="validate orientation, shapes, and packed byte sizes without writing output", ) a = ap.parse_args() if not a.dense_embed: raise SystemExit( - "--no-dense-embed 需要阶段 6 的 embedding / lm_head 原生 kernel，" - "v1 没有它们就只能稠密化（§2.4）" + "--no-dense-embed requires native packed embedding and lm_head kernels" ) t0 = time.time() - log("读取 GGUF 元数据：%s" % a.gguf) + log("Reading GGUF metadata: %s" % a.gguf) reader = GGUFReader(a.gguf) tensors = {t.name: t for t in reader.tensors} dims = dims_from_gguf(reader) check_dims(dims) log( - " 维度与映射表 REAL 一致：%d 层，hidden=%d，vocab=%d" + " dimensions match mapping profile: %d layers, hidden=%d, vocab=%d" % (dims.n_layers, dims.hidden, dims.vocab) ) - # --layers 必须在 check_dims **之后**覆盖：维度照旧逐项校 REAL（防止换模型后硬套本表）， - # 但 config 的 num_hidden_layers / layer_types 要跟着改，否则截断产物与 config - # 不自洽，框架构造 64 层却只拿到 N 层权重（旧版本里这条表现为“不可加载”）。 + # Validate the full model profile before applying the optional layer limit, + # then keep the emitted config consistent with the truncated checkpoint. if a.layers is not None: if not 0 < a.layers < dims.n_layers: raise SystemExit( - "--layers 必须在 (0, %d) 之间，实际 %d" % (dims.n_layers, a.layers) + "--layers must be in (0, %d), got %d" % (dims.n_layers, a.layers) ) log( - " --layers %d：num_hidden_layers %d -> %d，产物可加载" + " --layers %d: num_hidden_layers %d -> %d" % (a.layers, dims.n_layers, a.layers) ) dims.n_layers = a.layers @@ -684,15 +660,15 @@ def main() -> int: opt.types = {n: TYPE_NAME[int(t.tensor_type)] for n, t in tensors.items()} plan = M.build_plan(dims) n_exc = M.apply_v1_exceptions(plan, opt.types, enabled=a.dense_iq) - log(" 映射条目 %d，v1 稠密化例外命中 %d 个 IQ4" % (len(plan), n_exc)) + log(" mapping entries %d, dense IQ4 fallbacks %d" % (len(plan), n_exc)) blob = [e for e in plan if e.blob] log( - " blob %d 个 / 稠密化 %d 个 / 丢弃 MTP 前缀 %s" + " packed %d / dense %d / excluded MTP prefixes %s" % (len(blob), len(plan) - len(blob), M.DROP_PREFIXES) ) if a.dry_run: - log("\n== dry-run：逐条目校验取向与字节数（不写盘、不反量化）==") + log("\n== Dry run: validate orientation and byte sizes ==") blob_bytes = dense_bytes = 0 for e in plan: t = tensors[e.gguf] @@ -702,25 +678,26 @@ def main() -> int: rb = M.row_bytes(int(e.shape[1]), bs, ts) if int(t.data.shape[-1]) != rb: raise ValueError( - "%s: 源行字节 %d != 期望 %d" + "%s: source row bytes %d != expected %d" % (e.gguf, int(t.data.shape[-1]), rb) ) if int(t.data.shape[0]) < n_out: raise ValueError( - "%s: 源 %d 行 < 条目需 %d 行" % (e.gguf, t.data.shape[0], n_out) + "%s: source has %d rows, entry requires %d" + % (e.gguf, t.data.shape[0], n_out) ) blob_bytes += n_out * rb else: n = prod(tuple(int(x) for x in e.shape)) if not e.slices and prod(int(x) for x in t.shape) != n: raise ValueError( - "%s: 源元素数 %s != 条目 shape %s" + "%s: source element count %s != entry shape %s" % (e.gguf, t.shape, tuple(e.shape)) ) dense_bytes += n * 2 log( - " PASS %d 个条目取向/字节数自洽：blob %.3f GiB + 稠密化 BF16 %.3f GiB" - " = 产物应占 %.3f GiB" + " PASS %d entries: packed %.3f GiB + dense BF16 %.3f GiB" + " = expected output %.3f GiB" % ( len(plan), blob_bytes / _GiB, @@ -740,30 +717,31 @@ def main() -> int: if a.skip_pack: with open(os.path.join(a.out, "model.safetensors.index.json")) as fp: w.total = json.load(fp)["metadata"]["total_size"] - log("\n== --skip-pack：沿用已有权重，仅重写 config.json / 分词器与自检 ==") + log("\n== --skip-pack: refresh config, tokenizer, and verification ==") else: - log("\n== 写出 %s ==" % a.out) + log("\n== Writing %s ==" % a.out) for i, e in enumerate(plan): t = tensors[e.gguf] name, tens = build(e, t, dims, opt, False) w.add(name, tens) if (i + 1) % 100 == 0: - log(" ... %d/%d 条目（%.1f s）" % (i + 1, len(plan), time.time() - t0)) + log( + " ... %d/%d entries (%.1f s)" + % (i + 1, len(plan), time.time() - t0) + ) w.finish() - # config.json 两条路都要写：activation_vperm 这类语义元数据只能在这里刷新， - # 留在 else 里会让 --skip-pack 沿用旧 config（为了几个键重打包 7.2 GiB 不值）。 - # 规则由映射表派生（M.activation_vperm_rules），C++ 照单执行，不在两边各抄一份。 + # Refresh semantic metadata even with --skip-pack. Derive rules from the + # mapping so Python and C++ do not maintain duplicate definitions. rules = M.activation_vperm_rules(dims, plan) if a.vperm == "none": - # --vperm none = 全链路不做任何 V 头置换：in_proj 不重排、out_proj 不 gather、 - # denseref 不列置换。config 必须同步清空规则，否则 C++ 照旧 gather。 + # Keep conversion, runtime, and dense-reference paths aligned. rules = [] cfg = M.make_root_config(dims, ggml_types, rules) with open(os.path.join(a.out, "config.json"), "w") as fp: json.dump(cfg, fp, indent=1, sort_keys=True) log( - " config.json：%d 个 ggml_types 键（quantization_config 在顶层）+ 激活 V 头置换规则 %d 条：%s" + " config.json: %d ggml_types keys and %d activation V-head rules: %s" % ( len(ggml_types), len(rules), @@ -772,7 +750,7 @@ def main() -> int: % (r["suffix"], r["num_k_heads"], r["num_v_per_k"], r["head_dim"]) for r in rules ) - or "无", + or "none", ) ) @@ -783,8 +761,7 @@ def main() -> int: json.dump( { "gguf": os.path.abspath(a.gguf), - # 张量 data 区之和 != 文件大小（后者含元数据与对齐填充）， - # 两者都记下来，免得日后拿这个数去对 stat 产生误会 + # Tensor payload excludes file metadata and alignment padding. "gguf_file_bytes": os.path.getsize(os.path.abspath(a.gguf)), "gguf_tensor_data_bytes": sum( int(t.n_bytes) for t in reader.tensors @@ -809,7 +786,7 @@ def main() -> int: fails += verify(a.out, plan, tensors, dims, opt, a.verify) if a.emit_dense_ref: - log("\n== 额外产出稠密基准版 %s ==" % a.emit_dense_ref) + log("\n== Writing dense reference %s ==" % a.emit_dense_ref) os.makedirs(a.emit_dense_ref, exist_ok=True) wr = ShardWriter(a.emit_dense_ref, int(a.max_shard_gib * _GiB)) for e in plan: @@ -819,42 +796,33 @@ def main() -> int: ref_cfg = M.make_root_config( dims, {M.type_table_key(k): "dense_bf16" for k in ggml_types}, rules ) - # 稠密基准版不写 quantization_config：框架默认 NoneQuantization，C++ 里没有人 - # 执行置换。它的 ssm_out 列序在打包期已置换为 grouped（见 build() 里的 act_vperm 分支）， - # 与 blob 路径（运行时 gather）语义相同，可做逐层 cos_sim 对拍（§8.3）。 + # The dense reference omits quantization_config and stores grouped + # columns directly, matching the packed path's runtime semantics. del ref_cfg["quantization_config"] with open(os.path.join(a.emit_dense_ref, "config.json"), "w") as fp: json.dump(ref_cfg, fp, indent=1, sort_keys=True) export_tokenizer(reader, a.emit_dense_ref, a.tokenizer_dir, dims) log( - "\n===== 完成：%.1f s，产物 %.3f GiB，自检 FAIL %d 处 =====" + "\n===== Complete: %.1f s, output %.3f GiB, verification failures %d =====" % (time.time() - t0, w.total / _GiB, fails) ) return 1 if fails else 0 def export_tokenizer(reader, out_dir: str, tokenizer_dir: str, dims) -> int: - """产物自带完整分词器。为什么不是简单 copy： - - 实测 `--tokenizer-dir`（models/Qwen3.8-27B-BF16）只有 vocab.json，**没有** - merges.txt / tokenizer.json，`AutoTokenizer.from_pretrained` 直接报 - "`vocab` and `merges` must be both be from memory or both filenames"。 - GGUF 内嵌完整 byte-level BPE（实测 248320 tokens / 247587 merges， - tokenizer.ggml.model=gpt2, pre=qwen35），词表与 embedding 行数同源，故以 GGUF 为准 - 写 vocab.json + merges.txt，其余配置文件从 tokenizer_dir 复制。 - """ + """Export GGUF BPE vocabulary and copy optional tokenizer configuration.""" tokens = [str(t) for t in X.gguf_meta(reader, "tokenizer.ggml.tokens")] merges = [str(m) for m in X.gguf_meta(reader, "tokenizer.ggml.merges")] model = str(X.gguf_meta(reader, "tokenizer.ggml.model")[0]) if len(tokens) != dims.vocab: raise SystemExit( - "GGUF 词表 %d != config vocab_size %d，词表与 embedding 不同源" + "GGUF vocabulary size %d != config vocab_size %d" % (len(tokens), dims.vocab) ) if model != "gpt2": log( - " 警告：tokenizer.ggml.model=%r 非 gpt2，vocab.json/merges.txt 写法需复核" + " WARNING tokenizer.ggml.model=%r is not gpt2; verify vocab/merges export" % model ) with open(os.path.join(out_dir, "vocab.json"), "w", encoding="utf-8") as fp: @@ -862,7 +830,7 @@ def export_tokenizer(reader, out_dir: str, tokenizer_dir: str, dims) -> int: with open(os.path.join(out_dir, "merges.txt"), "w", encoding="utf-8") as fp: fp.write("#version: 0.2\n" + "\n".join(merges) + "\n") log( - " 词表来自 GGUF：vocab %d / merges %d（model=%s）" + " tokenizer from GGUF: vocab %d / merges %d (model=%s)" % (len(tokens), len(merges), model) ) @@ -872,42 +840,45 @@ def export_tokenizer(reader, out_dir: str, tokenizer_dir: str, dims) -> int: else [] ) if not have: - log(" 警告：分词器配置目录不存在：%s（只写了词表）" % tokenizer_dir) + log(" WARNING tokenizer config directory not found: %s" % tokenizer_dir) copied = [] for f in TOKENIZER_FILES: dst = os.path.join(out_dir, f) if f in have and not os.path.exists(dst): shutil.copy2(os.path.join(tokenizer_dir, f), dst) copied.append(f) - log(" 附属配置复制 %d 个：%s" % (len(copied), " ".join(sorted(copied)))) + log( + " copied %d auxiliary tokenizer files: %s" + % (len(copied), " ".join(sorted(copied))) + ) if "tokenizer_config.json" not in copied + have: raise SystemExit( - "产物缺 tokenizer_config.json：既没从 %s 复制到，也没导出兜底" + "output is missing tokenizer_config.json; it was not available in %s" % tokenizer_dir ) return check_tokenizer(out_dir, dims) def check_tokenizer(out_dir: str, dims) -> int: - """真装一次 AutoTokenizer 并做编解码往返（阶段 5 的前置条件，现在就能测）。""" + """Load AutoTokenizer and verify an encode/decode round trip.""" try: from transformers import AutoTokenizer except ImportError: - log(" SKIP 分词器自检：本环境无 transformers") + log(" SKIP tokenizer verification: transformers is unavailable") return 0 try: tk = AutoTokenizer.from_pretrained(out_dir) n, cls = len(tk), type(tk).__name__ - s = "你好，世界 hello world 27B" + s = "Hello, world 27B" ids = tk.encode(s) ok = n == dims.vocab and tk.decode(ids) == s log( - " %s 分词器 %s vocab=%d 往返=%s" + " %s tokenizer %s vocab=%d round_trip=%s" % ("PASS" if ok else "FAIL", cls, n, tk.decode(ids) == s) ) return 0 if ok else 1 except Exception as exc: # noqa: BLE001 - log(" FAIL 分词器加载：%s: %s" % (type(exc).__name__, str(exc)[:200])) + log(" FAIL tokenizer load: %s: %s" % (type(exc).__name__, str(exc)[:200])) return 1 diff --git a/scripts/gguf_transforms.py b/scripts/gguf_transforms.py index 7cba7132e..7be63e141 100644 --- a/scripts/gguf_transforms.py +++ b/scripts/gguf_transforms.py @@ -1,18 +1,12 @@ #!/usr/bin/env python3 -""" -InfiniLM 路线 B —— 打包期变换（纯 numpy，不依赖 gguf-py / torch / InfiniCore）。 - -为什么单独一个文件：审计脚本 `gguf_routeb_audit.py` C 节要**证明**这些置换自等/可行， -打包器 `gguf_to_infinilm.py` 要**执行**同一份置换。两处各写一遍正是阶段 0 踩过坑 -（同一事实两份定义 -> 两套互相矛盾的预算数字），故这里只有一份实现。 - -置换方向约定（依据 llama.cpp conversion/qwen.py:571-605 与 §2.7）： - HF / InfiniLM 序 = grouped，索引 [k][v] - GGUF 序 = tiled ，索引 [v][k] （dst[v*n_k + k] = src[k*n_v_per_k + v]） - ⇒ llama.cpp 写入 = grouped -> tiled = reorder_v - ⇒ 本方案打包 = tiled -> grouped = reorder_v_inverse -方向本身仍属阶段 4 的 A/B 项（作用域已钉死，方向未闭环），故打包器暴露 -`--vperm {inv,fwd,none}` 三个取值，默认 inv。 +"""Pure NumPy transforms shared by the GGUF-to-InfiniLM converter. + +Value-head ordering follows llama.cpp's Qwen conversion: + HF / InfiniLM = grouped [key][value] + GGUF = tiled [value][key] + +Therefore ``reorder_v`` converts grouped to tiled order and +``reorder_v_inverse`` converts tiled to grouped order. """ from __future__ import annotations @@ -20,14 +14,15 @@ import numpy as np # --------------------------------------------------------------------------- -# V 头置换 +# Value-head permutation # --------------------------------------------------------------------------- def reorder_v(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: - """grouped -> tiled，与 llama.cpp `_reorder_v_heads` 同语义（沿 dim0 的整头/整元素置换）。 + """Convert grouped to tiled order along dimension 0. - 支持任意尾部维度：1-D（A_log/dt_bias，hd=1）、2-D（权重行）、3-D（conv1d [C,1,K]）。 + Trailing dimensions are preserved, including 1-D scalars per head, + 2-D weight rows, and 3-D convolution weights. """ rest = t.shape[1:] return ( @@ -38,7 +33,7 @@ def reorder_v(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: def reorder_v_inverse(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.ndarray: - """逆变换 = 两个轴参数对调后再调用一次。""" + """Convert tiled to grouped order along dimension 0.""" rest = t.shape[1:] return ( t.reshape((n_v_per_k, n_k, hd) + rest) @@ -51,23 +46,23 @@ def reorder_v_inverse(t: np.ndarray, n_k: int, n_v_per_k: int, hd: int) -> np.nd def vperm_head_dim(e, dims) -> int: - """一条映射条目里每个 value 头占多少元素。 + """Return the number of elements per value head for a mapping entry. - `in_proj_a/b`、`A_log`、`dt_bias` 是 head_dim=1 的退化形式（每头一个标量）， - 其余（in_proj_v / in_proj_z / conv1d 的 V 段）是 lin_v_dim 个。判据用 shape 而不是 - 键名匹配，避免打包器里再写一张名字表。 + Derive the value from shape rather than tensor names so the converter does + not need a second name table. """ n_heads = dims.lin_v_heads rows = int(e.shape[0]) if e.vperm == "all" else dims.value_dim if rows % n_heads: raise ValueError( - "%s：作用域行数 %d 不能被 value 头数 %d 整除" % (e.infinilm, rows, n_heads) + "%s: scope rows %d are not divisible by %d value heads" + % (e.infinilm, rows, n_heads) ) return rows // n_heads def apply_vperm(arr: np.ndarray, e, dims, direction: str = "inv") -> np.ndarray: - """按条目的作用域（all / v_tail）对 dim0 做 V 头置换。""" + """Permute value heads along dimension 0 within an all or v_tail scope.""" fn = _VPERM[direction] if fn is None: return arr @@ -75,13 +70,15 @@ def apply_vperm(arr: np.ndarray, e, dims, direction: str = "inv") -> np.ndarray: v_per_k = dims.lin_v_heads // n_k if int(dims.lin_v_heads) % n_k: raise ValueError( - "lin_v_heads %d 不能被 lin_k_heads %d 整除" % (dims.lin_v_heads, n_k) + "lin_v_heads %d is not divisible by lin_k_heads %d" + % (dims.lin_v_heads, n_k) ) if e.vperm == "v_tail": n_v = n_k * v_per_k * hd if arr.shape[0] < n_v: raise ValueError( - "%s：dim0=%d 小于 value 段长度 %d" % (e.infinilm, arr.shape[0], n_v) + "%s: dimension 0 size %d is smaller than value segment %d" + % (e.infinilm, arr.shape[0], n_v) ) out = np.asarray(arr, dtype=arr.dtype) return np.concatenate([out[:-n_v], fn(out[-n_v:], n_k, v_per_k, hd)], axis=0) @@ -89,42 +86,37 @@ def apply_vperm(arr: np.ndarray, e, dims, direction: str = "inv") -> np.ndarray: # --------------------------------------------------------------------------- -# 其它变换 +# Other transforms # --------------------------------------------------------------------------- def alog_from_ssm_a(a: np.ndarray) -> np.ndarray: - """A_log = log(-ssm_a)。 - - GGUF 存的是 `-exp(A_log)`（conversion/qwen.py:388），而 InfiniCore - fused_gated_delta_net_gating 自己算 -expf(A_log) ⇒ 它要 HF 约定。 - 实测本文件 48 个值全为负；出现非负值说明源不是这个约定，必须炸出来而不是静默 NaN。 - """ + """Recover the HF ``A_log`` convention from GGUF ``-exp(A_log)`` values.""" a = np.asarray(a, dtype=np.float32) if not np.all(a < 0): raise ValueError( - "ssm_a 存在非负值（min=%g），无法取 log(-x)；" - "请核对 conversion/qwen.py 的 A_log 约定" % float(a.min()) + "ssm_a contains a non-negative value (min=%g); cannot compute log(-x). " + "Check the A_log convention in conversion/qwen.py." % float(a.min()) ) return np.log(-a) def gguf_meta(reader, suffix: str): - """元数据键带架构前缀（qwen35.*），允许传短名；contents() 对单元素返回标量，统一成列表。""" + """Read metadata with architecture/general prefixes and return a list.""" for key in ("qwen35.%s" % suffix, "general.%s" % suffix, suffix): if key in reader.fields: v = reader.fields[key].contents() return v if isinstance(v, (list, tuple, np.ndarray)) else [v] - raise KeyError("GGUF 元数据缺少：%s（qwen35./general. 前缀均未命中）" % suffix) + raise KeyError( + "missing GGUF metadata %s (no qwen35/general/unprefixed match)" % suffix + ) def bf16_bits(x: np.ndarray) -> np.ndarray: - """float32 -> bfloat16 的位模式（uint16）。 + """Return round-to-nearest-even bfloat16 bit patterns as uint16. - 只做 round-to-nearest-even 的截断，与 torch 的 `.to(torch.bfloat16)` 等价； - 打包器实际写盘用 torch 做 cast，这里留给校验路径把 BF16 张量按位比回来。 - 全程 uint32 而不升 uint64：进位只丢失 bit32，不影响要取的 bit16..31， - 内存却减半（lm_head 这类亿级张量上 uint64 会直接 OOM）。 + Keep arithmetic in uint32 to avoid doubling memory for large tensors. An + overflow beyond bit 31 cannot affect the retained bfloat16 bits. """ u = np.ascontiguousarray(x, dtype=np.float32).view(np.uint32) bias = ((u >> np.uint32(16)) & np.uint32(1)) + np.uint32(0x7FFF) diff --git a/test/scripts/test_gguf_routeb.py b/test/scripts/test_gguf_routeb.py new file mode 100644 index 000000000..eacb95b80 --- /dev/null +++ b/test/scripts/test_gguf_routeb.py @@ -0,0 +1,69 @@ +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace + +import numpy as np + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts")) + +import gguf_mapping as mapping # noqa: E402 +import gguf_transforms as transforms # noqa: E402 + + +class GGUFTransformsTest(unittest.TestCase): + def test_value_head_permutation_round_trip(self): + source = np.arange(2 * 3 * 4 * 5, dtype=np.uint8).reshape(24, 5) + tiled = transforms.reorder_v(source, n_k=2, n_v_per_k=3, hd=4) + restored = transforms.reorder_v_inverse(tiled, n_k=2, n_v_per_k=3, hd=4) + + np.testing.assert_array_equal(restored, source) + self.assertEqual(sorted(map(bytes, tiled)), sorted(map(bytes, source))) + + def test_tail_permutation_preserves_prefix_and_rows(self): + dims = SimpleNamespace(lin_k_heads=2, lin_v_heads=6, value_dim=12) + entry = SimpleNamespace(shape=(20, 3), vperm="v_tail", infinilm="conv") + source = np.arange(60, dtype=np.uint8).reshape(20, 3) + + tiled = transforms.apply_vperm(source, entry, dims, direction="fwd") + restored = transforms.apply_vperm(tiled, entry, dims, direction="inv") + + np.testing.assert_array_equal(tiled[:8], source[:8]) + np.testing.assert_array_equal(restored, source) + self.assertEqual(sorted(map(bytes, tiled[8:])), sorted(map(bytes, source[8:]))) + + def test_bf16_bits_for_exact_values(self): + values = np.array([0.0, 1.0, -2.0, np.inf], dtype=np.float32) + expected = np.array([0x0000, 0x3F80, 0xC000, 0x7F80], dtype=np.uint16) + np.testing.assert_array_equal(transforms.bf16_bits(values), expected) + + +class GGUFMappingTest(unittest.TestCase): + def test_generated_config_keeps_quantization_at_root(self): + table = {"model.language_model.layers.0.mlp.down_proj.weight_bytes": mapping.Q6_K} + rules = mapping.activation_vperm_rules(mapping.REAL, mapping.build_plan(mapping.REAL)) + config = mapping.make_root_config(mapping.REAL, table, rules) + + self.assertNotIn("quantization_config", config["text_config"]) + quant = config["quantization_config"] + self.assertEqual(quant["quant_method"], "gguf") + self.assertEqual(quant["key_prefix"], mapping.PREFIX) + self.assertEqual(quant["ggml_types"], table) + self.assertEqual(quant["activation_vperm"], rules) + self.assertTrue(rules) + self.assertEqual(len({rule["suffix"] for rule in rules}), len(rules)) + + def test_packed_checkpoint_name_and_row_size(self): + entry = SimpleNamespace(blob=True, infinilm="model.language_model.layers.0.mlp.down_proj.weight") + self.assertEqual( + mapping.ckpt_name(entry), + "model.language_model.layers.0.mlp.down_proj.weight_bytes", + ) + self.assertEqual(mapping.row_bytes(5120, block_size=256, type_size=210), 4200) + with self.assertRaises(ValueError): + mapping.row_bytes(5119, block_size=256, type_size=210) + + +if __name__ == "__main__": + unittest.main() From c643e7af9ebde0a00ba296fcc8bb0047eec6f55e Mon Sep 17 00:00:00 2001 From: xindongliu594 Date: Fri, 4 Sep 2026 14:39:03 +0800 Subject: [PATCH 4/5] fix: reduce GGUF quantization log noise --- csrc/layers/quantization/gguf.cpp | 35 +++++++++++++++++-------------- test/scripts/test_gguf_routeb.py | 12 ++++++++--- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/csrc/layers/quantization/gguf.cpp b/csrc/layers/quantization/gguf.cpp index c15bc2aaa..9c52e2990 100644 --- a/csrc/layers/quantization/gguf.cpp +++ b/csrc/layers/quantization/gguf.cpp @@ -172,28 +172,31 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) } } - spdlog::info( - "GGUF block quantization: {} entries (blob {} / dense {} / outside prefix {}), key_prefix='{}'{}", - types_.size(), n_blob, n_dense, n_outside, key_prefix_, - key_prefix_.empty() - ? " (not set; table keys are relative safetensors names)" - : " (for example, root-level lm_head entries)"); - - std::string vs; - for (const auto &r : vperm_) { - if (!vs.empty()) { - vs += ", "; + static std::atomic config_logged{false}; + if (!config_logged.exchange(true, std::memory_order_relaxed)) { + spdlog::info( + "GGUF block quantization: {} entries (blob {} / dense {} / outside prefix {}), key_prefix='{}'{}", + types_.size(), n_blob, n_dense, n_outside, key_prefix_, + key_prefix_.empty() + ? " (not set; table keys are relative safetensors names)" + : " (for example, root-level lm_head entries)"); + + std::string vs; + for (const auto &r : vperm_) { + if (!vs.empty()) { + vs += ", "; + } + vs += r.suffix + "=" + std::to_string(r.n_k) + "x" + std::to_string(r.r) + "x" + std::to_string(r.hd); } - vs += r.suffix + "=" + std::to_string(r.n_k) + "x" + std::to_string(r.r) + "x" + std::to_string(r.hd); + spdlog::info("GGUF block quantization: {} activation V-head permutation rules (grouped->tiled): {}", + vperm_.size(), vs.empty() ? "none" : vs); } - spdlog::info("GGUF block quantization: {} activation V-head permutation rules (grouped->tiled): {}", - vperm_.size(), vs.empty() ? "none" : vs); } GGUFBlockQuantization::~GGUFBlockQuantization() { if (n_blob_ + n_dense_ + n_group_ > 0) { - spdlog::info("GGUF block quantization: layout matches blob {} / dense {} / fused group {}", - n_blob_, n_dense_, n_group_); + spdlog::debug("GGUF block quantization: layout matches blob {} / dense {} / fused group {}", + n_blob_, n_dense_, n_group_); } } diff --git a/test/scripts/test_gguf_routeb.py b/test/scripts/test_gguf_routeb.py index eacb95b80..fa0add8f3 100644 --- a/test/scripts/test_gguf_routeb.py +++ b/test/scripts/test_gguf_routeb.py @@ -41,8 +41,12 @@ def test_bf16_bits_for_exact_values(self): class GGUFMappingTest(unittest.TestCase): def test_generated_config_keeps_quantization_at_root(self): - table = {"model.language_model.layers.0.mlp.down_proj.weight_bytes": mapping.Q6_K} - rules = mapping.activation_vperm_rules(mapping.REAL, mapping.build_plan(mapping.REAL)) + table = { + "model.language_model.layers.0.mlp.down_proj.weight_bytes": mapping.Q6_K + } + rules = mapping.activation_vperm_rules( + mapping.REAL, mapping.build_plan(mapping.REAL) + ) config = mapping.make_root_config(mapping.REAL, table, rules) self.assertNotIn("quantization_config", config["text_config"]) @@ -55,7 +59,9 @@ def test_generated_config_keeps_quantization_at_root(self): self.assertEqual(len({rule["suffix"] for rule in rules}), len(rules)) def test_packed_checkpoint_name_and_row_size(self): - entry = SimpleNamespace(blob=True, infinilm="model.language_model.layers.0.mlp.down_proj.weight") + entry = SimpleNamespace( + blob=True, infinilm="model.language_model.layers.0.mlp.down_proj.weight" + ) self.assertEqual( mapping.ckpt_name(entry), "model.language_model.layers.0.mlp.down_proj.weight_bytes", From 27609c37a3403467317d578e1a113a8885c256c7 Mon Sep 17 00:00:00 2001 From: xindongliu594 Date: Fri, 11 Sep 2026 15:05:59 +0800 Subject: [PATCH 5/5] chore: remove GGUF research diagnostics --- .../layers/causal_lm_templates/text_model.hpp | 44 +----- csrc/layers/quantization/gguf.cpp | 82 +---------- csrc/layers/quantization/gguf.hpp | 4 - csrc/models/qwen3_5/qwen3_5_attention.cpp | 55 +------- csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp | 129 +----------------- csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp | 12 -- .../qwen3_next/qwen3_next_gated_deltanet.cpp | 40 ------ 7 files changed, 11 insertions(+), 355 deletions(-) diff --git a/csrc/layers/causal_lm_templates/text_model.hpp b/csrc/layers/causal_lm_templates/text_model.hpp index a979b913c..49b60d7f4 100644 --- a/csrc/layers/causal_lm_templates/text_model.hpp +++ b/csrc/layers/causal_lm_templates/text_model.hpp @@ -6,15 +6,11 @@ #include "infinicore/nn/embedding.hpp" #include "infinicore/nn/rmsnorm.hpp" #include "infinicore/ops.hpp" -#include "infinicore/ops/add_rms_norm.hpp" -#include "infinicore/ops/cast.hpp" #include "infinicore/ops/distributed/allgather.hpp" #include "infinicore/ops/distributed/send_recv.hpp" #include "infinicore/tensor.hpp" -#include #include #include -#include namespace infinilm::layers::causal_lm_templates { @@ -83,8 +79,7 @@ class TextModel : public infinicore::nn::Module { return hidden_states; } - dump_pre_final_norm_if_requested(hidden_states, residual); - final_norm_inplace(hidden_states, residual); + norm_->forward_inplace(hidden_states, residual); return hidden_states; } @@ -124,8 +119,7 @@ class TextModel : public infinicore::nn::Module { return hidden_states; } - dump_pre_final_norm_if_requested(hidden_states, residual); - final_norm_inplace(hidden_states, residual); + norm_->forward_inplace(hidden_states, residual); return hidden_states; } @@ -142,22 +136,6 @@ class TextModel : public infinicore::nn::Module { INFINICORE_NN_MODULE(infinicore::nn::RMSNorm, norm); private: - void final_norm_inplace(infinicore::Tensor &hidden_states, - infinicore::Tensor &residual) const { - const char *env = std::getenv("INFINILM_FINAL_NORM_FP32_FUSED"); - const bool enabled = env != nullptr && env[0] != '\0' && std::string(env) != "0"; - if (!enabled) { - norm_->forward_inplace(hidden_states, residual); - return; - } - auto y32 = infinicore::Tensor::empty(hidden_states->shape(), infinicore::DataType::F32, hidden_states->device()); - auto sum32 = infinicore::Tensor::empty(residual->shape(), infinicore::DataType::F32, residual->device()); - infinicore::op::add_rms_norm_(y32, sum32, hidden_states, residual, norm_->weight(), - static_cast(norm_->eps())); - hidden_states = y32; - residual = sum32; - } - bool is_first_pp_stage() const { return pp_stage_ == 0; } bool is_last_pp_stage() const { return pp_stage_ + 1 == pp_size_; } @@ -228,24 +206,6 @@ class TextModel : public infinicore::nn::Module { return infinicore::op::add(residual, hidden_states); } - void dump_pre_final_norm_if_requested( - infinicore::Tensor &hidden_states, - infinicore::Tensor &residual) const { - const char *dump_dir = std::getenv("INFINILM_FINAL_PRENORM_DUMP_DIR"); - if (dump_dir == nullptr || dump_dir[0] == '\0') { - return; - } - const char *dump_numel = std::getenv("INFINILM_FINAL_PRENORM_DUMP_NUMEL"); - if (dump_numel != nullptr && dump_numel[0] != '\0' - && hidden_states->numel() - != std::strtoull(dump_numel, nullptr, 10)) { - return; - } - auto pre_norm = materialize_hidden_states(hidden_states, residual); - pre_norm->debug( - std::string(dump_dir) + "/infini_pre_final_norm.bin"); - } - infinicore::DataType dtype_{infinicore::DataType::F32}; size_t hidden_size_{0}; size_t pp_size_{1}; diff --git a/csrc/layers/quantization/gguf.cpp b/csrc/layers/quantization/gguf.cpp index 9c52e2990..73517ef51 100644 --- a/csrc/layers/quantization/gguf.cpp +++ b/csrc/layers/quantization/gguf.cpp @@ -3,11 +3,8 @@ #include #include #include -#include #include -#include -#include #include #include @@ -54,20 +51,6 @@ std::string supported_types() { constexpr const char *DENSE_MARK = "dense_bf16"; -bool env_enabled(const char *name) { - const char *value = std::getenv(name); - return value != nullptr && value[0] != '\0' && std::strcmp(value, "0") != 0; -} - -bool use_f32_decode_output(const std::string &table_key, size_t m_count) { - if (!env_enabled("INFINI_GGUF_F32_DECODE_OUT") || m_count > 16) { - return false; - } - const char *match = std::getenv("INFINI_GGUF_F32_DECODE_OUT_MATCH"); - return match == nullptr || match[0] == '\0' - || table_key.find(match) != std::string::npos; -} - } // namespace GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) @@ -83,9 +66,6 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) throw std::runtime_error("GGUFBlockQuantization: ggml_types is empty"); } - size_t n_blob = 0; - size_t n_dense = 0; - size_t n_outside = 0; for (const auto &kv : table.items()) { const std::string &name = kv.key(); // Keep keys outside key_prefix unchanged. This includes root-level @@ -93,8 +73,6 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) std::string key = name; if (!key_prefix_.empty() && name.compare(0, key_prefix_.size(), key_prefix_) == 0) { key = name.substr(key_prefix_.size()); - } else { - ++n_outside; } int64_t id = DENSE_BF16; @@ -105,7 +83,6 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) "GGUFBlockQuantization: value '" + v + "' for '" + name + "' is neither an integer type id nor \"" + DENSE_MARK + "\""); } - ++n_dense; } else { if (!kv.value().is_number_integer()) { throw std::runtime_error( @@ -122,7 +99,6 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) + std::to_string(id) + " (supported: " + supported_types() + "); unsupported types must be converted to dense BF16"); } - ++n_blob; } if (!types_.emplace(std::move(key), TypeEntry{id, name}).second) { @@ -172,33 +148,9 @@ GGUFBlockQuantization::GGUFBlockQuantization(const nlohmann::json &quant_config) } } - static std::atomic config_logged{false}; - if (!config_logged.exchange(true, std::memory_order_relaxed)) { - spdlog::info( - "GGUF block quantization: {} entries (blob {} / dense {} / outside prefix {}), key_prefix='{}'{}", - types_.size(), n_blob, n_dense, n_outside, key_prefix_, - key_prefix_.empty() - ? " (not set; table keys are relative safetensors names)" - : " (for example, root-level lm_head entries)"); - - std::string vs; - for (const auto &r : vperm_) { - if (!vs.empty()) { - vs += ", "; - } - vs += r.suffix + "=" + std::to_string(r.n_k) + "x" + std::to_string(r.r) + "x" + std::to_string(r.hd); - } - spdlog::info("GGUF block quantization: {} activation V-head permutation rules (grouped->tiled): {}", - vperm_.size(), vs.empty() ? "none" : vs); - } } -GGUFBlockQuantization::~GGUFBlockQuantization() { - if (n_blob_ + n_dense_ + n_group_ > 0) { - spdlog::debug("GGUF block quantization: layout matches blob {} / dense {} / fused group {}", - n_blob_, n_dense_, n_group_); - } -} +GGUFBlockQuantization::~GGUFBlockQuantization() = default; bool GGUFBlockQuantization::is_known_type(int64_t type_id) { return ggml_block(type_id) != nullptr; @@ -336,18 +288,15 @@ std::vector GGUFBlockQuantization::get_param_layout( "GGUFBlockQuantization: fused-group stem '" + stem + "' has no '" + stem + "..*' entry in the type table"); } - ++n_group_; return {}; } const int64_t id = resolve(stem); if (id == DENSE_BF16) { - ++n_dense_; // The converter stored this tensor as dense BF16; use regular GEMM. return {{"weight", {out_features, in_features}, dtype, split_dim, tp_rank, tp_size}}; } - ++n_blob_; const size_t rb = row_bytes(in_features, id); return {{{BLOB_SUFFIX}, {out_features, rb}, infinicore::DataType::U8, split_dim, tp_rank, tp_size}}; } @@ -401,26 +350,8 @@ infinicore::Tensor GGUFBlockQuantization::forward_shard( auto flat = x->view({M, K}); flat = flat->is_contiguous() ? flat : flat->contiguous(); - const bool f32_decode_out = use_f32_decode_output(table_key, M); - const auto out_dtype = f32_decode_out - ? infinicore::DataType::F32 - : input->dtype(); - auto out = infinicore::Tensor::empty({M, N}, out_dtype, input->device()); - // Log the first packed invocation as a lightweight wiring diagnostic. - static std::atomic blob_calls{0}; - if (blob_calls.fetch_add(1) == 0) { - spdlog::info( - "linear_gguf: first packed forward {} -- M={} N={} K={} ggml_type={} row_bytes={}", - table_key, M, N, K, type_id, w->size(1)); - } - if (f32_decode_out) { - static std::atomic f32_calls{0}; - if (f32_calls.fetch_add(1) == 0) { - spdlog::warn( - "linear_gguf: experimental F32 decode output enabled; first match {} -- M={} N={} K={}", - table_key, M, N, K); - } - } + auto out = infinicore::Tensor::empty( + {M, N}, input->dtype(), input->device()); infinicore::op::linear_gguf_(out, flat, w, type_id); std::vector out_shape(x_shape.begin(), x_shape.end() - 1); @@ -469,13 +400,6 @@ infinicore::Tensor GGUFBlockQuantization::forward( } } else if (rule) { x = gather_grouped_to_tiled(*rule, input, describe(stem)); - // Log the first permutation as a lightweight wiring diagnostic. - static std::atomic vperm_applied{0}; - if (vperm_applied.fetch_add(1) == 0) { - spdlog::info( - "linear_gguf: first activation V-head permutation {} -- grouped->tiled {}x{}x{}", - describe(stem), rule->n_k, rule->r, rule->hd); - } } // A non-fused layer owns exactly one weight or weight_bytes parameter. diff --git a/csrc/layers/quantization/gguf.hpp b/csrc/layers/quantization/gguf.hpp index bd55cf485..e03f58a36 100644 --- a/csrc/layers/quantization/gguf.hpp +++ b/csrc/layers/quantization/gguf.hpp @@ -138,10 +138,6 @@ class GGUFBlockQuantization : public BaseQuantization { std::unordered_map types_; std::string key_prefix_; std::vector vperm_; // Empty when the converted model needs no permutation. - // Mutable because layout queries are logically const. - mutable size_t n_blob_ = 0; - mutable size_t n_dense_ = 0; - mutable size_t n_group_ = 0; }; } // namespace infinilm::quantization diff --git a/csrc/models/qwen3_5/qwen3_5_attention.cpp b/csrc/models/qwen3_5/qwen3_5_attention.cpp index 6371f6f65..a7ff902ed 100644 --- a/csrc/models/qwen3_5/qwen3_5_attention.cpp +++ b/csrc/models/qwen3_5/qwen3_5_attention.cpp @@ -6,7 +6,6 @@ #include "../../utils.hpp" #include #include -#include #include #include #include @@ -14,25 +13,6 @@ #include namespace infinilm::models::qwen3_5 { -namespace { - -bool should_dump_attention(size_t layer_idx) { - const char *dump_dir = std::getenv("INFINILM_ATTENTION_DUMP_DIR"); - const char *target = std::getenv("INFINILM_ATTENTION_DUMP_LAYER"); - return dump_dir != nullptr && dump_dir[0] != '\0' - && target != nullptr && target[0] != '\0' - && layer_idx == std::strtoull(target, nullptr, 10); -} - -void dump_attention_tensor(const infinicore::Tensor &tensor, - const char *name, - size_t layer_idx) { - const char *dump_dir = std::getenv("INFINILM_ATTENTION_DUMP_DIR"); - tensor->debug(std::string(dump_dir) + "/infini_attention_" + name + "_" - + std::to_string(layer_idx) + ".bin"); -} - -} // namespace Qwen35Attention::Qwen35Attention(std::shared_ptr model_config, size_t layer_idx, @@ -161,50 +141,23 @@ infinicore::Tensor Qwen35Attention::forward_paged_(const infinicore::Tensor &pos ASSERT_EQ(batch_size, 1); auto [q, gate, k, v] = qkv_proj_->forward_split(hidden_states_mutable); - const bool dump_attention = should_dump_attention(layer_idx_); - if (dump_attention) { - dump_attention_tensor(q, "q_raw", layer_idx_); - dump_attention_tensor(gate, "gate_raw", layer_idx_); - dump_attention_tensor(k, "k_raw", layer_idx_); - dump_attention_tensor(v, "v_raw", layer_idx_); - } auto q_reshaped = q->view({seq_len, num_attention_heads_, head_dim_}); auto k_reshaped = k->view({seq_len, num_key_value_heads_, head_dim_}); auto v_reshaped = v->view({seq_len, num_key_value_heads_, head_dim_}); q_reshaped = q_norm_->forward(q_reshaped); k_reshaped = k_norm_->forward(k_reshaped); - if (dump_attention) { - dump_attention_tensor(q_reshaped, "q_norm", layer_idx_); - dump_attention_tensor(k_reshaped, "k_norm", layer_idx_); - } auto pos_shape = position_ids->shape(); if (pos_shape.size() != 2 && pos_shape.size() != 1) { throw std::runtime_error("Unexpected position_ids shape"); } std::tie(q_reshaped, k_reshaped) = mrope_->forward(q_reshaped, k_reshaped, position_ids); - if (dump_attention) { - dump_attention_tensor(q_reshaped, "q_rope", layer_idx_); - dump_attention_tensor(k_reshaped, "k_rope", layer_idx_); - } auto attn_output = attn_->forward(q_reshaped, k_reshaped, v_reshaped); - if (dump_attention) { - dump_attention_tensor(attn_output, "core_output", layer_idx_); - } - auto gate_sigmoid = infinicore::op::sigmoid(gate)->view(attn_output->shape()); - if (dump_attention) { - dump_attention_tensor(gate_sigmoid, "gate_sigmoid", layer_idx_); - } - attn_output = infinicore::op::mul(attn_output, gate_sigmoid); - if (dump_attention) { - dump_attention_tensor(attn_output, "gated_output", layer_idx_); - } - auto projected = o_proj_->forward(attn_output); - if (dump_attention) { - dump_attention_tensor(projected, "projected_output", layer_idx_); - } - return projected; + attn_output = infinicore::op::mul( + attn_output, + infinicore::op::sigmoid(gate)->view(attn_output->shape())); + return o_proj_->forward(attn_output); } } // namespace infinilm::models::qwen3_5 diff --git a/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp b/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp index 810678f8f..b6704e2e4 100644 --- a/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp +++ b/csrc/models/qwen3_5/qwen3_5_decoderLayer.cpp @@ -1,45 +1,10 @@ #include "qwen3_5_decoderLayer.hpp" #include "infinicore/ops.hpp" -#include "infinicore/ops/add_rms_norm.hpp" -#include "infinicore/ops/cast.hpp" -#include #include #include #include namespace infinilm::models::qwen3_5 { -namespace { - -void dump_prefill_tensor(const infinicore::Tensor &tensor, - const std::string &filename) { - const char *dump_dir = std::getenv("INFINILM_LAYER_DUMP_DIR"); - if (dump_dir == nullptr || dump_dir[0] == '\0' || !tensor) { - return; - } - const char *dump_numel = std::getenv("INFINILM_LAYER_DUMP_NUMEL"); - if (dump_numel == nullptr || dump_numel[0] == '\0' - || tensor->numel() != std::strtoull(dump_numel, nullptr, 10)) { - return; - } - tensor->debug(std::string(dump_dir) + "/" + filename); -} - -bool should_dump_layer(size_t layer_idx) { - const char *first_n = std::getenv("INFINILM_LAYER_DUMP_FIRST_N"); - if (first_n != nullptr && first_n[0] != '\0' - && layer_idx < std::strtoull(first_n, nullptr, 10)) { - return true; - } - return (layer_idx + 1) % 8 == 0; -} - -bool should_dump_operators(size_t layer_idx) { - const char *target = std::getenv("INFINILM_OPERATOR_DUMP_LAYER"); - return target != nullptr && target[0] != '\0' - && layer_idx == std::strtoull(target, nullptr, 10); -} - -} // namespace Qwen35DecoderLayer::Qwen35DecoderLayer(std::shared_ptr model_config, size_t layer_idx, @@ -70,105 +35,15 @@ Qwen35DecoderLayer::Qwen35DecoderLayer(std::shared_ptr Qwen35DecoderLayer::forward(const infinicore::Tensor &positions, infinicore::Tensor &hidden_states, infinicore::Tensor &residual) { - if (layer_idx_ == 0) { - dump_prefill_tensor(hidden_states, "infini_embed.bin"); - } - if (residual - && hidden_states->dtype() == infinicore::DataType::F32 - && residual->dtype() == infinicore::DataType::BF16) { - auto y = infinicore::Tensor::empty( - hidden_states->shape(), infinicore::DataType::BF16, hidden_states->device()); - auto residual_out = infinicore::Tensor::empty( - residual->shape(), infinicore::DataType::BF16, residual->device()); - infinicore::op::add_rms_norm_( - y, residual_out, hidden_states, residual, - input_layernorm_->weight(), - static_cast(input_layernorm_->eps())); - hidden_states = y; - residual = residual_out; - } else { - input_layernorm_->forward_inplace(hidden_states, residual); - } + input_layernorm_->forward_inplace(hidden_states, residual); if ("linear_attention" == layer_type_) { hidden_states = linear_attn_->forward(hidden_states); } else if ("full_attention" == layer_type_) { hidden_states = self_attn_->forward(positions, hidden_states); } - const char *fp32_fused_env = std::getenv("INFINILM_POST_NORM_FP32_FUSED"); - const bool fp32_fused = fp32_fused_env != nullptr && fp32_fused_env[0] != '\0' - && std::string(fp32_fused_env) != "0"; - const bool mixed_gguf_f32 = residual - && hidden_states->dtype() == infinicore::DataType::F32 - && residual->dtype() == infinicore::DataType::BF16; - if (mixed_gguf_f32) { - auto y = infinicore::Tensor::empty( - hidden_states->shape(), infinicore::DataType::BF16, hidden_states->device()); - auto residual_out = infinicore::Tensor::empty( - residual->shape(), infinicore::DataType::BF16, residual->device()); - infinicore::op::add_rms_norm_( - y, residual_out, hidden_states, residual, - post_attention_layernorm_->weight(), - static_cast(post_attention_layernorm_->eps())); - hidden_states = y; - residual = residual_out; - } else if (fp32_fused) { - auto a32 = infinicore::Tensor::empty(hidden_states->shape(), infinicore::DataType::F32, hidden_states->device()); - auto b32 = infinicore::Tensor::empty(residual->shape(), infinicore::DataType::F32, residual->device()); - infinicore::op::cast_(a32, hidden_states); - infinicore::op::cast_(b32, residual); - auto y32 = infinicore::Tensor::empty(hidden_states->shape(), infinicore::DataType::F32, hidden_states->device()); - auto r32 = infinicore::Tensor::empty(residual->shape(), infinicore::DataType::F32, residual->device()); - infinicore::op::add_rms_norm_(y32, r32, a32, b32, - post_attention_layernorm_->weight(), - static_cast(post_attention_layernorm_->eps())); - hidden_states = y32; - residual = r32; - } else { - post_attention_layernorm_->forward_inplace(hidden_states, residual); - } - if (should_dump_operators(layer_idx_)) { - dump_prefill_tensor(residual, - "infini_attn_residual_" + std::to_string(layer_idx_) + ".bin"); - dump_prefill_tensor(hidden_states, - "infini_attn_post_norm_" + std::to_string(layer_idx_) + ".bin"); - } - const char *fp32_mlp_env = std::getenv("INFINILM_POST_NORM_FP32_MLP"); - const bool fp32_mlp = fp32_mlp_env != nullptr && fp32_mlp_env[0] != '\0' - && std::string(fp32_mlp_env) != "0"; - if (fp32_mlp && !fp32_fused) { - auto fp32_hidden = infinicore::Tensor::empty( - hidden_states->shape(), infinicore::DataType::F32, hidden_states->device()); - infinicore::op::cast_(fp32_hidden, hidden_states); - hidden_states = fp32_hidden; - } + post_attention_layernorm_->forward_inplace(hidden_states, residual); hidden_states = mlp_->forward(hidden_states); - if (should_dump_operators(layer_idx_)) { - dump_prefill_tensor(hidden_states, - "infini_ffn_out_" + std::to_string(layer_idx_) + ".bin"); - } - if (fp32_mlp && !fp32_fused) { - auto bf16_hidden = infinicore::Tensor::empty( - hidden_states->shape(), infinicore::DataType::BF16, hidden_states->device()); - infinicore::op::cast_(bf16_hidden, hidden_states); - hidden_states = bf16_hidden; - } - if (should_dump_layer(layer_idx_)) { - auto materialized = residual ? infinicore::op::add(residual, hidden_states) - : hidden_states; - dump_prefill_tensor(materialized, - "infini_layer_" + std::to_string(layer_idx_) + "_post_ffn.bin"); - } - if (fp32_fused) { - auto bf16_hidden = infinicore::Tensor::empty( - hidden_states->shape(), infinicore::DataType::BF16, hidden_states->device()); - infinicore::op::cast_(bf16_hidden, hidden_states); - hidden_states = bf16_hidden; - auto bf16_residual = infinicore::Tensor::empty( - residual->shape(), infinicore::DataType::BF16, residual->device()); - infinicore::op::cast_(bf16_residual, residual); - residual = bf16_residual; - } return std::make_tuple(hidden_states, residual); } diff --git a/csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp b/csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp index 1105b2d24..c3d476442 100644 --- a/csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp +++ b/csrc/models/qwen3_5/qwen3_5_for_causal_lm.cpp @@ -2,15 +2,12 @@ #include "../models_registry.hpp" #include "infinicore/ops/gemm.hpp" -#include #include #include #include namespace infinilm::models::qwen3_5 { -// TextModel diagnostic hooks are compiled into this Qwen3.5 translation unit. - Qwen35ForCausalLM::Qwen35ForCausalLM( std::shared_ptr model_config, const infinicore::Device &device) { @@ -30,15 +27,6 @@ Qwen35ForCausalLM::Qwen35ForCausalLM( InfinilmModel::Output Qwen35ForCausalLM::forward( const InfinilmModel::Input &input) const { auto hidden_states = model_->forward(input); - const char *dump_dir = std::getenv("INFINILM_LAYER_DUMP_DIR"); - const char *dump_numel = std::getenv("INFINILM_LAYER_DUMP_NUMEL"); - if (dump_dir != nullptr && dump_dir[0] != '\0' - && dump_numel != nullptr && dump_numel[0] != '\0' - && hidden_states->numel() - == std::strtoull(dump_numel, nullptr, 10)) { - hidden_states->debug( - std::string(dump_dir) + "/infini_result_norm.bin"); - } infinicore::Tensor logits; if (fp32_lm_head_output_) { auto hidden = hidden_states->is_contiguous() diff --git a/csrc/models/qwen3_next/qwen3_next_gated_deltanet.cpp b/csrc/models/qwen3_next/qwen3_next_gated_deltanet.cpp index eac3ae58f..3fe55612b 100644 --- a/csrc/models/qwen3_next/qwen3_next_gated_deltanet.cpp +++ b/csrc/models/qwen3_next/qwen3_next_gated_deltanet.cpp @@ -10,38 +10,12 @@ #include #include -#include #include #include #include #include namespace infinilm::models::qwen3_next { -namespace { - -bool should_dump_gdn(size_t layer_idx, size_t seq_len) { - const char *target_layer = std::getenv("INFINILM_GDN_DUMP_LAYER"); - const char *target_seq_len = std::getenv("INFINILM_GDN_DUMP_SEQ_LEN"); - return target_layer != nullptr && target_layer[0] != '\0' - && target_seq_len != nullptr && target_seq_len[0] != '\0' - && layer_idx == std::strtoull(target_layer, nullptr, 10) - && seq_len == std::strtoull(target_seq_len, nullptr, 10); -} - -void dump_gdn_tensor(const infinicore::Tensor &tensor, - const std::string &name, - size_t layer_idx, - size_t seq_len) { - const char *dump_dir = std::getenv("INFINILM_LAYER_DUMP_DIR"); - if (dump_dir == nullptr || dump_dir[0] == '\0' || !tensor - || !should_dump_gdn(layer_idx, seq_len)) { - return; - } - tensor->debug(std::string(dump_dir) + "/infini_gdn_" + name + "_" - + std::to_string(layer_idx) + ".bin"); -} - -} // namespace Qwen3NextCausalConv1D::Qwen3NextCausalConv1D(std::shared_ptr model_config, size_t layer_idx, @@ -185,16 +159,11 @@ infinicore::Tensor Qwen3NextGatedDeltaNet::forward(const infinicore::Tensor &hid auto z = in_proj_z_->forward(hidden_states_mutable); auto a = in_proj_a_->forward(hidden_states_mutable); auto b = in_proj_b_->forward(hidden_states_mutable); - dump_gdn_tensor(qkv, "qkv_mixed", layer_idx_, seq_len); - dump_gdn_tensor(z, "z", layer_idx_, seq_len); - dump_gdn_tensor(a, "alpha", layer_idx_, seq_len); - dump_gdn_tensor(b, "beta", layer_idx_, seq_len); auto &forward_context = infinilm::global_state::get_forward_context(); auto &mamba_metadata = forward_context.mamba_metadata; auto conv_qkv = this->conv1d_->forward(qkv); - dump_gdn_tensor(conv_qkv, "conv_output_silu", layer_idx_, seq_len); auto q = conv_qkv->narrow({{2, 0, local_key_dim_}}); auto k = conv_qkv->narrow({{2, local_key_dim_, local_key_dim_}}); @@ -220,8 +189,6 @@ infinicore::Tensor Qwen3NextGatedDeltaNet::forward(const infinicore::Tensor &hid {seq_len, 1, local_num_value_heads_}, {b->stride(1), b->stride(0), 1}); auto [g, beta] = infinicore::op::fused_gated_delta_net_gating(A_log_, a_heads, b_heads, dt_bias_); - dump_gdn_tensor(g, "gate", layer_idx_, seq_len); - dump_gdn_tensor(beta, "beta_sigmoid", layer_idx_, seq_len); delta_out = infinicore::op::recurrent_gated_delta_rule_indexed( q_delta, @@ -255,8 +222,6 @@ infinicore::Tensor Qwen3NextGatedDeltaNet::forward(const infinicore::Tensor &hid {1, seq_len, local_num_value_heads_}, {b->stride(0), b->stride(1), 1}); auto [g, beta] = infinicore::op::fused_gated_delta_net_gating(A_log_, a_heads, b_heads, dt_bias_); - dump_gdn_tensor(g, "gate", layer_idx_, seq_len); - dump_gdn_tensor(beta, "beta_sigmoid", layer_idx_, seq_len); delta_out = infinicore::op::chunk_gated_delta_rule( q_delta, @@ -277,17 +242,12 @@ infinicore::Tensor Qwen3NextGatedDeltaNet::forward(const infinicore::Tensor &hid auto delta_out_2d = delta_out->as_strided( {batch_size * seq_len * local_num_value_heads_, value_head_dim_}, {static_cast(value_head_dim_), 1}); - dump_gdn_tensor(delta_out, "delta_out", layer_idx_, seq_len); auto v_norm_2d = norm_->forward(delta_out_2d); auto v_norm = v_norm_2d->as_strided( {batch_size, seq_len, local_value_dim_}, {static_cast(seq_len * local_value_dim_), static_cast(local_value_dim_), 1}); - dump_gdn_tensor(v_norm, "v_norm", layer_idx_, seq_len); auto gated = infinicore::op::mul(v_norm, infinicore::op::silu(z)); - dump_gdn_tensor(gated, "gated", layer_idx_, seq_len); - dump_gdn_tensor(gated, "final_output", layer_idx_, seq_len); auto output = out_proj_->forward(gated); - dump_gdn_tensor(output, "linear_attn_out", layer_idx_, seq_len); return output; }