diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index c295aed46..1636fff02 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -429,6 +429,7 @@ add_library(dflash_common STATIC src/qwen35moe/qwen35moe_daemon.cpp src/qwen35moe/qwen35moe_pipelined_decode.cpp # ── Common MoE hybrid infrastructure ── + src/common/heterogeneous_stage_planner.cpp src/common/moe_hybrid_placement.cpp src/common/moe_hybrid_routing_stats.cpp src/common/moe_hybrid_storage.cpp diff --git a/server/config/ds4/r9700-strix/decode-routing.csv b/server/config/ds4/r9700-strix/decode-routing.csv new file mode 100644 index 000000000..27b0c283c --- /dev/null +++ b/server/config/ds4/r9700-strix/decode-routing.csv @@ -0,0 +1,45 @@ +# hotness table: n_layer=43 n_expert=256 n_expert_used=6 +# format: one row per layer, columns are expert activation counts (expert 0..N-1) +3,1,2,1,0,2,2,1,0,0,2,0,0,0,3,0,0,4,2,0,1,0,2,2,3,65,0,1,0,0,1,0,1,0,1,0,0,1,1,1,0,0,0,2,0,1,1,0,1,0,1,4,2,0,1,1,1,1,65,0,2,0,1,3,1,1,2,0,0,1,1,1,1,1,2,2,2,1,1,3,0,0,0,2,4,0,1,0,0,2,1,3,3,0,0,0,1,3,1,0,1,0,3,4,2,0,1,0,1,1,1,2,0,1,0,2,3,0,0,3,1,1,1,2,1,3,0,3,2,1,0,0,1,0,0,5,0,1,1,2,0,2,1,1,2,1,0,2,3,0,0,0,2,1,1,2,3,1,2,1,2,1,1,2,0,0,1,0,1,1,3,0,1,1,68,0,1,0,1,2,1,0,2,1,0,3,1,1,1,1,2,1,0,1,7,2,2,0,2,3,0,0,1,1,1,0,0,2,1,0,1,0,0,1,0,65,1,2,5,1,0,1,0,0,2,0,1,2,1,1,0,2,2,0,1,1,0,2,2,2,1,1,3,1,0,1,0,1,3,2,2,1,0,2,1,0 +0,1,1,1,2,2,1,1,3,3,2,2,2,0,0,2,1,0,0,2,1,1,1,1,1,3,0,65,1,1,0,0,3,3,2,68,0,2,1,0,1,4,0,0,0,0,4,0,2,0,0,2,1,0,2,0,2,2,1,1,2,3,0,1,0,1,0,1,3,2,3,1,0,0,1,0,1,0,0,0,3,0,1,3,2,1,1,0,1,1,1,0,0,0,1,1,0,3,0,0,2,2,1,1,0,1,4,0,1,1,1,0,1,1,2,2,0,1,1,1,3,1,0,1,4,2,2,1,1,0,2,1,0,1,2,1,1,1,0,0,1,0,0,0,0,0,1,0,2,2,1,0,0,1,0,3,1,1,4,0,0,0,1,1,3,1,0,1,3,1,1,1,3,0,0,0,0,3,4,0,1,0,0,3,1,4,0,0,0,2,0,3,1,1,1,2,1,1,0,0,0,0,1,3,1,4,0,0,66,1,2,0,1,2,4,0,3,2,0,0,2,1,2,0,0,2,2,1,66,0,1,0,2,3,3,2,0,2,1,0,1,1,0,3,1,0,1,1,0,0,0,3,1,2,1,1 +0,1,1,1,0,3,3,5,65,0,2,0,4,0,4,0,2,2,1,0,1,0,1,2,2,1,1,1,2,5,0,2,1,1,2,2,1,1,1,2,1,0,1,1,3,0,0,0,1,2,3,0,2,2,2,2,1,1,1,2,1,1,0,3,1,1,1,2,2,1,0,0,1,0,1,67,1,0,2,1,1,1,1,3,3,0,1,1,0,2,0,1,1,1,0,1,1,0,2,0,1,0,0,4,2,3,2,3,2,2,0,2,0,1,1,2,0,0,0,0,1,2,1,0,0,1,2,1,1,0,1,2,2,0,1,0,1,1,0,2,2,1,4,1,1,1,0,2,0,0,0,0,0,1,1,0,1,1,1,0,0,1,0,0,1,0,2,0,1,1,2,2,0,0,0,1,0,0,1,0,1,0,3,1,1,1,0,0,65,1,2,1,3,3,0,67,1,1,2,1,1,2,0,2,1,1,0,1,2,3,0,0,2,1,1,0,0,1,0,2,2,2,1,2,0,0,1,0,0,2,1,0,1,1,3,2,0,3,0,2,1,0,4,3,0,0,2,3,0,0,1,1,0,3,1,1 +0,0,0,0,0,0,1,27,0,0,0,0,0,0,0,48,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,27,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,13,1,6,0,0,0,65,0,2,0,0,4,0,0,0,0,1,0,0,0,0,0,0,0,3,0,3,0,0,1,0,0,0,0,44,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,33,8,0,0,0,0,0,0,0,0,0,0,32,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,2,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,30,0,0,0,68,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,15,0,2,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,17,0,0,0,0,11,0,0,0,0,0,0,0,0,0,0 +0,0,0,6,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0,0,71,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,15,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,65,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,122,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,0,0,4,0,0,0,0,0,0,16,0,1,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,5,0,0,0,0,9,0,0,0,0,0,1,0,0,0,0,12,0,1,0,0,0,8,0,0,0,0,0,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,35,0,0,0,0,0,1,0,0,0,0,0,1,0,0,2,0,59,0,0,0,0,0,0,0,0,3,0,2,0,0,0,1,0,0,0,0,0,9,0,53,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1,0,0,0,0,0,0,0,0,0,5,0,0,6,0,1,0,6,0,0,0,0,0,9,1,0,0,0,0,0,0,0,0,0,116,0,0,0,1,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,67,0,0,18,0,0,1,0,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,32,10,0,0,0,0,0,5,14,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,2,0,3,0,0,8,0,7,8,0,0,0,0,0,1,65,0,7,0,1,0,1,0,0,0,0,0,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,2,0,3,2,0,33,0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,19,0,2,0,0,0,0,0,10,0,0,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,15,2,0,0,0,0,0,0,0,0,0,0,8,0,0,0 +0,133,0,0,1,0,0,0,0,7,0,56,0,4,0,0,22,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,0,0,2,0,0,0,0,0,0,20,0,0,0,0,0,1,9,1,0,0,0,9,0,15,1,0,0,0,0,0,0,0,0,53,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,12,0,2,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,37,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,9,0,0,2,1,0,0,0,4,0,0,0,0,0,0,0,0,0,0,7,0,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,1,0,0,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,2,0,0,0,0,5,8,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0,0,0,1,0,0,0,0,0,0,0,0,94,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +0,2,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,4,0,0,0,0,0,0,0,0,0,0,0,13,0,0,0,0,0,0,25,0,0,0,0,0,0,0,0,0,0,1,0,0,38,0,0,0,1,0,0,0,0,0,0,0,3,51,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,15,26,0,1,0,0,2,0,11,0,0,0,0,43,4,1,0,0,2,61,0,0,0,0,0,0,9,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,1,4,0,0,4,0,0,56,0,0,0,0,0,0,5,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,2,0,0,0,0,29,0,0,0,70,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,42,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0 +0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,0,0,0,0,0,0,0,0,1,1,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,73,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,45,0,0,0,0,1,0,2,0,3,0,1,0,1,0,0,1,0,0,0,1,0,0,0,2,2,0,0,0,0,7,0,0,0,0,0,5,0,44,0,0,0,0,3,0,0,0,0,2,0,0,0,6,1,0,3,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,9,0,0,5,0,0,97,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,34,0,0,0,0,0,0,0,6,0,0,0,0,27,0,0,0,0,0,0,67,0,3,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0 +1,0,0,0,0,0,0,1,0,0,2,0,0,0,0,3,0,1,0,0,0,0,0,12,0,0,0,0,0,0,0,10,0,0,0,0,0,0,0,0,1,0,1,3,0,0,2,0,3,0,0,0,0,0,0,3,0,12,0,0,0,0,76,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,4,6,16,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,59,1,2,0,0,0,0,0,0,0,0,0,34,0,1,8,0,0,0,0,0,0,0,0,2,0,0,0,0,65,3,0,0,0,0,0,0,1,0,0,0,0,0,0,1,1,0,83,0,0,0,0,0,0,1,0,2,0,0,0,1,0,1,1,0,14,0,0,2,0,0,0,1,0,0,24,0,0,0,0,0,0,0,0,0,0,0,0,4,0,6,0,0,0,0,0,3,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,31,0,0,0,0,0,0,18,0,0,0,0,0,0,0,0,0,0,0,0,0 +0,0,1,0,2,0,0,0,0,0,0,68,0,3,0,0,30,0,2,0,0,0,0,3,0,7,0,0,0,0,0,0,7,0,0,0,0,0,19,0,59,0,6,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,99,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,4,1,0,0,0,0,0,0,0,0,3,0,54,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,5,0,1,1,0,0,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,26,2,2,0,0,0,8,3,0,0,0,0,1,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,54,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,2,2,0,0,0,0,0,0,2,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0,25 +0,0,0,0,0,0,1,0,18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,4,6,0,3,0,23,11,0,0,4,0,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0,2,3,0,0,0,0,0,1,0,0,0,11,0,0,6,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,21,4,0,0,5,0,0,0,0,6,2,0,1,0,0,0,0,0,0,0,0,0,0,0,110,0,3,6,0,0,0,0,0,0,0,0,1,0,0,27,0,0,1,0,0,1,0,0,1,0,1,0,0,0,12,0,0,0,5,0,11,0,0,0,1,2,18,1,0,0,0,0,0,0,0,0,5,2,0,0,1,0,0,0,0,0,0,0,1,0,0,0,3,8,0,0,0,0,0,0,0,10,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,9,0,10,1,1,2,0,0,3,0,1,3,0,0,0,7,0,0,0,0,0,0,5,0,0,13,0,1,0,0,2,0,3,1,0,0,0,2,0,0,0,0,0,17,0,0,0,0,58 +0,0,10,0,0,0,0,0,0,0,0,0,0,0,34,0,0,0,0,0,1,0,0,0,0,0,1,0,4,0,0,0,0,8,0,0,0,0,2,0,0,0,0,5,0,0,0,0,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0,0,1,12,6,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,3,0,0,13,0,0,0,0,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,70,0,0,0,0,0,0,2,0,1,1,3,0,0,2,0,0,1,44,1,0,0,0,0,3,0,0,0,0,0,6,0,0,3,0,0,0,0,0,0,9,0,0,0,0,3,0,0,2,0,0,6,2,0,0,0,0,0,0,4,0,16,0,0,0,8,0,0,2,0,1,0,0,0,1,0,0,0,0,0,0,0,69,0,0,0,14,0,0,0,0,0,4,1,4,0,2,3,0,0,0,0,0,0,81,2,5,0,0,4,0,0,0,0,0,0,0,0,0,0,4,0,0,1,0,0,1,0,0,0,0,0 +0,0,0,0,28,0,0,0,0,3,0,0,0,30,40,0,0,0,0,22,0,0,6,0,0,0,0,0,0,0,3,0,3,0,0,0,1,0,3,0,0,0,0,0,9,0,0,0,1,0,0,0,0,32,0,0,0,0,3,15,0,0,1,1,3,0,0,0,0,0,0,0,2,0,0,0,0,4,0,0,3,0,25,0,0,0,1,3,0,0,0,15,0,0,0,0,0,0,1,0,1,0,0,1,2,13,0,0,3,0,0,0,0,0,0,0,1,0,0,3,4,0,0,0,0,3,0,0,0,3,0,2,0,0,0,0,5,0,0,0,0,1,0,0,0,0,0,0,15,0,0,0,0,2,0,2,48,1,0,4,2,0,0,0,0,0,0,3,0,1,1,0,0,0,0,1,0,0,0,3,0,0,3,18,4,0,0,0,0,0,0,0,0,0,3,1,0,0,0,1,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,19,2,0,1,0,0,0,0,9,0,1,0,31,0,0,0,0,2,0,2,6 +0,29,0,0,0,0,1,5,13,0,0,1,0,0,0,0,0,0,67,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,3,6,0,0,0,0,0,14,0,0,0,45,0,0,0,1,0,0,0,0,0,1,2,0,0,0,1,3,0,0,0,0,0,0,0,7,0,0,0,0,1,0,0,0,0,34,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,7,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,88,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,11,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,5,0,3,0,0,0,0,0,0,0,0,4,12,0,7,1,0,0,0,0,1,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,10,0,49,0,1,2,0,0,0,13,40,0,0,0,0,0,0,0,0,30,0,0,0,9,0,0,0,0,0,0,0 +0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,3,0,0,0,2,0,0,0,0,6,0,0,6,0,0,0,0,0,1,0,0,2,1,0,0,0,0,3,0,0,0,25,0,66,1,0,0,0,0,7,0,2,0,0,0,1,0,0,0,0,0,40,0,0,0,0,0,0,0,0,0,0,0,12,1,0,0,0,5,0,1,4,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,10,0,0,0,0,0,3,0,0,0,0,2,0,0,0,0,0,4,0,0,17,0,0,32,2,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,125,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,6,0,0,0,0,0,1,9,3,9,0,0,0,0,0,1,0,0,1,0,1,22,0,0,0,0,0,0,0,0,2,11,3,1,0,0,0,0,0,0,0,16,0,0,0,4,2,0,0,0,2,0,0,0,2,0,0,0,0,0,0,4,41,0 +0,0,0,0,0,0,34,0,0,0,8,0,2,0,0,3,0,0,0,0,0,0,0,0,0,73,0,1,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,1,0,0,4,11,0,0,0,0,12,0,0,0,39,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,5,0,0,1,0,0,0,0,1,1,0,8,0,0,0,0,0,0,2,0,17,0,0,0,4,0,1,0,0,0,0,5,20,0,0,0,0,0,0,0,0,0,0,1,7,0,0,0,2,1,0,21,62,0,2,4,0,1,0,0,45,0,0,0,5,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,19,20,0,0,0,0,3,0,0,0,0,7,0,0,0,0,16,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,3,0,0,0,0,2,8,6,0,0,0,0,2,1,0,3,0,30 +0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,0,0,10,6,1,3,4,0,0,0,0,2,0,21,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,91,0,0,0,0,1,2,0,67,0,0,0,34,0,0,0,6,1,0,4,0,0,2,0,8,5,0,0,2,0,0,0,0,0,3,0,0,0,0,20,0,0,0,0,0,0,0,0,0,0,46,11,0,0,3,19,0,0,0,0,1,0,0,1,0,0,0,0,9,0,0,0,0,0,0,10,0,0,0,0,0,2,10,1,0,0,0,0,0,0,5,2,3,5,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,23,1,11,0,1,0,0,0,0,0,1,0,0,0,0,13,1,5,0,0,0,0,1,0,1,0,5,0,3,15,0,0,0,0,0,0,0,1,0,0,0,0,5,0,2,0,0,0,0,0,12,0,2,0,0,2,0,2,0,0,0,0 +0,0,0,0,0,0,0,0,23,0,0,3,63,0,0,13,8,0,0,0,0,0,0,1,0,4,0,0,0,0,0,4,0,4,3,0,0,1,0,0,0,7,0,0,0,0,0,0,2,0,0,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,1,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,3,8,1,0,0,0,0,9,0,0,22,0,0,5,1,0,0,0,0,0,0,0,50,0,0,58,0,67,0,0,0,0,0,0,0,0,0,0,0,0,2,3,1,0,0,0,0,0,10,0,12,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,15,0,0,2,0,0,0,4,0,0,0,4,0,1,16,0,1,34,0,0,3,0,0,0,0,1,0,0,0,0,0,0,13,2,0,0,0,0,0,0,0,0,10,0,0,6,0,0,0,0,0,0,0,0,5,0,0,0,0,9,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,8,0,0,0,0,3,0,0,0,0 +0,0,1,0,3,0,4,1,0,0,18,5,0,0,0,0,0,1,1,0,6,0,0,20,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,4,0,0,4,25,0,0,0,0,0,0,13,1,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,5,0,0,0,5,0,0,0,0,0,0,0,42,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,23,0,0,0,1,0,2,0,0,0,0,2,0,13,0,0,0,0,0,0,0,0,50,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,2,0,0,1,0,0,0,2,0,0,9,0,3,0,1,2,0,0,13,1,2,0,1,2,1,0,0,3,12,0,3,2,60,2,0,0,0,0,0,6,0,0,1,0,0,0,0,1,0,0,0,0,0,2,1,0,0,0,0,0,0,0,0,0,1,0,0,0,7,1,0,0,0,0,2,0,3,0,67,54,0,4,0,0,0,0,0,0,0,5,0,0,0,3,0,0,0,0,0,1,0,0,0 +0,0,0,0,0,0,0,3,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,10,0,34,0,1,0,0,0,3,0,0,2,1,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,5,0,0,0,0,41,0,4,0,0,2,0,0,0,0,0,0,0,0,0,9,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,1,1,0,0,0,0,0,12,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,34,14,0,0,0,0,0,0,1,0,1,2,52,0,0,0,1,1,0,9,0,0,0,0,1,0,0,0,0,3,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,13,1,0,0,1,0,0,0,0,0,0,3,1,0,0,2,0,0,0,0,0,2,0,4,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,109,16,0,0,0,0,7,70,3,0,0,0,0,0,0,0,0,0,0,0,0,17,1,4,0,0,0,0,0,0,0,4,16,0,0,5,0,0 +0,0,0,0,0,1,0,34,1,0,0,1,0,1,0,0,5,0,4,0,1,0,0,0,3,0,0,0,2,1,0,0,2,0,1,1,0,4,0,0,1,0,0,12,0,0,0,0,0,43,0,0,1,0,8,4,0,0,0,0,0,0,21,0,0,1,0,0,1,0,4,0,0,0,0,0,1,0,0,0,0,6,0,0,3,0,26,0,14,14,0,0,0,0,0,0,4,0,0,0,1,2,0,0,0,1,2,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,2,65,4,0,0,0,4,0,0,51,0,0,0,0,0,0,7,0,0,0,0,0,0,0,0,2,3,0,0,0,0,1,0,11,0,0,1,0,0,0,4,0,17,0,1,1,0,1,2,0,2,0,0,25,0,0,0,0,5,0,0,0,0,0,2,31,0,0,0,0,0,0,8,0,0,18,0,0,0,0,1,0,0,0,1,0,0,0,0,0,26,0,0,0,0,3,2,0,0,0,0,0,0,2,1,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0 +27,0,0,0,0,0,6,0,0,0,0,5,2,0,3,0,0,0,0,5,0,5,1,0,0,0,0,0,0,0,0,6,0,0,0,6,0,0,0,0,0,0,0,0,2,0,0,3,0,0,0,1,0,0,2,0,0,0,0,58,2,0,0,0,0,1,0,0,0,0,1,0,26,0,10,0,0,1,0,0,0,5,0,0,0,0,0,0,0,11,0,1,0,0,0,0,3,2,0,0,1,0,0,2,0,0,2,0,0,0,2,1,3,0,0,0,0,0,1,15,0,0,1,0,0,0,6,0,0,0,0,0,7,0,0,4,0,0,0,0,0,0,0,0,0,0,2,0,0,83,0,0,0,0,0,0,0,0,1,1,0,0,5,0,0,0,10,0,0,0,0,0,1,26,4,0,0,1,0,0,0,0,0,0,0,0,0,1,0,4,0,1,1,0,1,0,0,1,1,0,0,0,1,0,0,7,0,0,0,0,0,1,1,0,0,0,14,0,0,0,0,0,0,4,2,0,0,0,0,0,0,0,0,0,0,2,4,0,16,23,0,0,0,0,0,0,68,1,1,4,0,2,0,1,0,8 +4,0,0,5,0,0,0,0,2,0,2,0,0,0,0,14,0,1,0,0,0,0,0,0,4,14,0,0,0,0,1,1,3,0,0,0,2,2,5,0,0,0,0,4,0,0,0,0,3,2,0,10,4,0,0,0,0,0,0,0,5,46,0,0,1,1,0,0,10,0,5,3,1,0,0,0,16,0,0,0,1,3,0,0,2,0,0,0,3,0,5,0,0,1,31,4,0,0,2,0,0,4,0,0,1,2,0,6,0,0,0,0,0,0,0,1,0,0,23,0,53,0,0,0,0,0,0,0,1,0,0,6,0,0,0,0,0,0,1,1,0,0,2,0,0,1,0,0,0,0,2,0,0,0,0,7,0,0,0,0,0,0,0,0,6,8,0,0,3,0,2,0,0,0,5,1,0,0,0,3,0,0,0,2,0,0,0,3,0,0,0,0,0,9,0,1,0,0,0,0,0,0,0,0,5,0,0,6,0,0,10,6,2,0,1,0,0,1,2,8,0,0,0,0,0,6,0,14,4,1,25,1,2,0,0,1,3,0,0,0,0,20,1,12,0,8,2,1,0,0,9,11,1,0,0,4 +1,0,0,7,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,1,13,0,0,14,2,0,0,0,0,1,1,0,0,0,1,0,29,0,5,0,2,1,1,0,1,0,0,23,0,0,0,0,0,0,3,0,0,5,1,0,0,2,0,9,3,11,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,5,10,0,2,0,0,0,0,2,0,0,0,1,0,0,0,4,1,0,0,0,0,0,0,0,0,0,0,3,13,0,0,0,0,0,24,0,108,1,0,2,0,0,0,14,0,0,3,0,1,0,0,0,21,0,0,3,1,0,0,19,0,0,0,0,1,3,0,7,2,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,2,3,0,0,0,8,0,0,0,0,0,19,2,0,2,0,7,0,0,11,0,0,2,0,3,0,0,0,0,0,0,6,0,0,0,0,3,40,0,0,0,0,2,0,0,0,0,0,0,0,1,2,0,0,3,0,1,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,10,0,0,0,9,4,0,0,0,2,0,3 +0,0,0,0,1,0,4,2,0,3,0,11,0,2,0,7,0,0,0,0,18,0,0,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,1,0,12,0,3,0,0,0,0,0,0,0,10,1,0,0,0,0,4,0,0,0,3,0,0,0,0,1,2,3,0,1,0,5,2,2,0,0,0,0,0,0,0,0,0,9,0,2,0,4,0,3,5,2,4,1,0,0,0,0,0,2,1,6,0,0,0,0,0,0,0,0,6,2,91,1,0,0,2,0,0,2,0,0,0,0,0,1,0,5,2,0,0,0,6,0,3,2,0,0,0,0,4,0,0,0,0,0,0,17,6,7,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,32,0,0,0,0,0,1,30,0,0,1,0,9,0,4,0,0,7,3,0,0,0,0,0,0,1,0,0,0,4,0,0,0,9,7,4,0,0,0,0,0,0,0,0,2,0,0,0,7,0,0,0,0,76,0,0,0,0,0,0,2,0,0,0,4,0,0,2,0,3,0,1,0,8,0,0,3,0,1,0,0,10,0,0,0,1,0,0,0,1 +0,0,0,0,0,0,0,0,0,0,0,28,0,0,0,0,0,0,14,0,0,0,6,0,0,0,4,0,0,0,0,0,1,0,0,0,2,0,3,0,0,0,0,0,1,2,0,0,1,0,0,0,2,3,0,0,5,0,2,0,0,4,0,0,0,0,3,0,5,0,0,0,0,1,0,14,0,1,7,0,0,0,0,5,0,2,14,28,0,0,0,1,0,89,0,0,0,0,0,0,0,0,19,0,0,0,0,0,0,0,0,0,0,16,0,0,1,0,0,0,2,0,0,0,0,1,0,0,0,3,0,0,1,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,1,0,1,0,0,0,3,2,0,2,0,0,19,0,0,0,0,0,8,0,0,0,0,0,9,0,0,0,1,0,1,75,0,0,0,0,0,0,0,0,0,11,0,0,0,1,2,0,0,50,0,0,0,7,0,7,1,0,0,4,1,0,1,1,0,0,0,0,0,1,0,0,0,13,1,2,1,0,0,0,0,0,1,3,0,0,0,0,0,0,3,10,0,4,0,0 +0,1,57,11,0,0,0,0,0,0,0,0,4,0,0,0,5,0,0,0,0,2,0,0,0,27,6,0,21,0,0,0,0,0,6,0,0,0,1,15,0,0,0,0,0,0,0,0,0,0,0,0,0,1,5,0,0,0,0,0,0,2,0,0,0,4,6,5,0,19,0,0,7,11,0,1,0,0,0,0,0,0,9,1,3,0,0,0,7,0,0,0,1,0,0,65,0,0,0,0,0,0,0,0,0,5,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,3,0,0,0,0,3,0,0,2,0,0,0,0,0,4,33,7,1,0,0,0,0,0,0,4,0,1,1,0,0,0,7,7,0,0,0,0,0,6,0,0,1,0,0,0,22,0,53,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,11,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,2,3,0,0,1,0,0,1,0,4,15,4,0,1,0,0,1,0,0,0,0,0,1,0,0,0,15,0,1,0,0,0,0,3,0,0,1,0 +0,0,0,0,4,0,5,10,0,0,0,2,1,0,0,0,0,0,0,78,2,2,0,0,5,0,0,0,2,0,0,1,2,0,0,4,2,0,0,0,0,0,0,0,0,15,0,7,8,0,0,0,0,3,0,0,2,0,2,0,0,2,0,0,0,0,0,13,2,10,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,2,0,1,0,5,0,1,0,1,0,1,1,0,5,0,5,4,0,0,0,9,3,0,20,0,1,0,0,0,0,0,0,0,0,0,0,2,0,1,0,0,0,0,0,8,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0,0,0,0,0,0,0,0,0,4,0,0,9,0,5,73,0,0,0,5,0,0,0,0,0,0,17,0,0,0,0,1,0,0,0,5,0,0,4,4,0,0,1,1,0,0,0,16,2,1,0,0,0,0,0,0,0,0,1,0,0,0,4,1,0,0,0,0,0,0,0,0,0,84,0,0,0,0,0,0,2,0,2,2,1,0,0,0,4,1,7,0,0,2,1,7,0,0,0,1,6,0,3,0,0 +0,7,7,1,0,0,0,1,0,1,0,1,0,5,0,0,0,0,9,4,0,0,0,2,0,0,0,0,0,2,7,0,0,0,0,0,2,0,0,0,0,0,0,0,1,0,0,1,3,5,1,2,1,0,9,1,0,0,0,0,0,5,0,0,2,0,0,0,34,0,0,0,0,0,0,0,0,9,0,0,0,5,0,0,0,0,0,0,0,0,0,6,0,3,0,8,2,0,0,0,0,0,0,0,1,2,0,14,0,0,0,0,2,0,0,0,2,22,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,3,0,8,0,5,4,43,0,0,0,0,2,0,0,5,0,1,0,6,1,0,7,15,0,0,0,0,1,0,0,0,0,38,0,0,0,0,6,0,4,3,0,0,0,5,0,0,3,0,0,0,7,0,0,0,0,3,3,2,8,0,0,0,0,0,0,0,0,0,0,0,0,0,6,0,0,0,6,1,0,2,0,1,0,0,0,6,0,0,0,0,0,4,0,0,0,76,0,5,0,0,0,8,0,1,0,0,2,0,1,29,0,0,0,0,0,0,6,5,10,0,0 +0,2,0,0,0,1,0,1,0,1,0,3,0,0,3,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,2,0,15,0,3,3,0,0,3,0,0,0,0,0,1,0,8,2,0,4,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,6,0,0,0,0,0,0,2,0,0,57,0,7,0,2,0,8,0,3,0,0,7,0,0,0,0,5,3,1,1,10,0,0,0,0,0,1,0,0,0,0,0,1,18,5,5,0,3,0,0,0,7,0,0,0,0,0,0,1,8,0,0,0,0,1,0,0,0,1,0,5,0,2,9,1,3,1,0,0,0,1,2,0,0,0,10,0,0,0,0,0,0,0,0,14,0,0,0,3,0,0,0,0,0,0,0,0,23,3,0,1,0,0,0,0,0,0,2,16,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,11,0,0,8,0,0,2,0,1,0,0,0,5,0,0,0,0,0,1,1,0,0,0,1,0,6,4,29,9,6,0,0,0,0,0,0,0,0,11,0,0,0,0,1,86,0,0,38 +0,0,3,1,0,0,0,0,0,0,0,0,0,2,4,0,0,0,6,0,2,2,9,0,1,6,0,0,5,0,0,0,1,0,0,0,14,0,0,2,4,6,0,0,0,4,0,3,1,0,16,1,0,0,0,0,1,1,0,3,2,0,0,0,13,0,0,0,32,0,0,1,0,0,0,2,0,2,20,0,0,0,0,6,0,0,0,1,0,0,0,0,2,12,10,0,1,0,0,0,0,3,0,0,3,4,8,1,0,0,0,1,0,2,4,2,0,0,0,0,0,3,24,3,0,0,7,0,0,0,8,0,0,0,7,1,0,0,0,0,0,0,0,23,0,0,0,9,0,0,0,2,0,0,0,0,1,0,6,0,6,0,14,0,0,0,0,0,13,0,6,1,0,0,0,1,4,20,0,13,0,0,0,2,1,4,0,0,0,3,5,4,0,0,1,0,22,0,0,0,0,0,0,0,0,0,0,2,4,1,1,0,0,0,0,1,0,0,1,0,13,0,23,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0,1,0,6,2,1,0,12,21,0,1,0,0,0,1,5 +2,0,0,0,1,0,1,0,0,0,1,3,2,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,2,9,0,0,3,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,3,0,24,0,1,0,36,2,0,0,0,0,1,0,0,6,0,0,0,22,0,0,0,0,0,0,0,0,0,0,0,3,9,11,4,1,0,0,0,3,0,0,0,0,0,4,2,0,0,0,0,6,24,0,0,0,0,2,0,2,0,0,4,4,0,0,0,0,0,0,0,0,2,17,8,2,0,0,13,0,2,0,0,0,0,0,0,0,1,1,2,0,0,0,0,0,0,2,6,2,1,0,3,0,6,6,0,0,0,0,1,1,1,0,9,0,0,0,0,0,2,0,0,4,0,18,1,9,0,0,0,0,0,0,0,0,0,0,49,2,21,0,0,6,2,0,0,0,0,0,0,0,1,0,12,17,0,0,0,27,0,0,0,0,4,0,1,0,0,1,0,18,2,0,0,0,0,0,0,7,3,5,0,0,0,0,0,0,3,10,1,0,0,8,0,0,0,0,0,0,19 +0,1,0,0,4,0,0,0,0,0,1,1,1,0,0,2,0,0,0,0,0,1,11,3,5,0,10,2,0,0,2,0,2,0,0,0,0,0,0,0,0,80,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,1,6,1,6,0,2,9,0,63,0,0,7,0,0,1,0,0,49,8,1,0,0,1,1,0,0,0,0,0,0,0,6,0,3,5,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,19,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,2,1,0,0,5,0,0,0,0,0,0,0,0,0,0,2,2,0,3,0,0,0,0,1,0,0,8,3,1,6,0,0,0,2,0,0,2,0,0,0,0,0,0,1,0,0,0,0,16,0,0,3,7,0,0,0,0,0,0,0,51,0,7,1,0,1,0,0,0,1,0,0,0,0,3,0,0,0,5,0,0,1,3,3,1,1,1,0,0,0,0,0,0,2,0,0,1,0,0,1,0,0,0,4,4,0,0,0,0,0,2,1,0,0,0,5,52,0 +0,14,5,0,0,0,0,0,0,0,0,1,0,0,0,2,0,0,7,0,0,1,1,0,2,0,4,0,0,0,17,1,0,0,0,0,0,0,7,4,0,0,7,1,4,0,0,0,1,0,0,6,0,0,19,0,0,0,2,0,1,0,0,42,7,0,5,0,0,0,0,4,2,0,0,22,0,0,0,0,1,0,0,1,3,0,0,0,0,2,10,0,5,11,15,0,0,0,0,0,6,0,7,0,0,0,0,1,2,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,16,0,0,0,0,2,0,0,0,2,0,0,2,1,0,0,1,70,6,2,0,1,0,0,0,0,2,0,3,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,5,28,0,0,2,0,0,0,0,0,0,7,0,0,0,0,3,2,0,0,0,0,14,0,1,0,0,3,0,0,0,0,6,2,0,3,0,0,1,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,0,12,0,0,0,0,1,4,36,0,10,0,5,6,0,1,0,0,0,0,0,0,0,0,19,0,2,0,0,0,0 +0,10,0,0,0,0,0,0,0,1,1,0,0,0,0,0,13,7,0,0,3,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,1,0,0,0,0,0,10,0,1,0,2,0,0,0,5,0,4,5,0,3,0,0,0,0,6,2,5,0,1,0,0,5,2,0,2,26,0,0,11,0,6,0,0,1,18,0,5,0,7,0,4,13,0,10,9,0,0,0,0,0,0,0,2,0,0,0,6,0,0,31,0,0,0,1,0,0,11,9,0,0,0,0,1,5,0,1,1,0,0,3,0,0,4,0,3,0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,6,4,1,0,0,0,13,0,0,1,0,2,0,50,0,2,0,0,0,0,0,0,0,0,1,0,0,0,1,0,5,0,4,0,0,51,0,0,3,0,0,5,27,0,0,0,5,1,17,0,0,0,0,2,0,8,1,0,1,0,5,1,1,1,1,2,0,0,0,0,0,3,0,6,6,0,8,0,0,0,0,0,0,8,0,0,0,0,0,0,0,2,4,0,0 +0,0,0,0,4,0,0,0,0,0,3,2,0,0,11,1,1,6,4,0,0,0,1,0,0,0,0,3,3,0,0,0,0,0,6,0,6,0,0,0,0,0,0,1,3,0,0,0,0,1,0,0,0,2,0,0,0,5,9,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0,0,4,0,0,0,0,0,61,8,25,0,2,0,0,0,0,2,2,0,0,0,0,8,0,6,6,0,1,0,1,0,0,0,0,11,2,0,0,0,1,0,0,1,0,1,0,20,0,0,0,1,0,1,0,0,2,0,0,0,0,0,0,39,0,32,0,0,0,2,0,0,3,0,1,0,0,0,5,0,0,0,0,0,0,10,0,0,0,0,0,0,4,2,0,0,1,0,7,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,65,0,0,0,0,2,7,0,4,0,0,0,0,24,0,0,0,1,3,7,0,0,0,0,16,2,0,4,1,0,1,0,0,0,1,0,0,0,4,0,0,0,0,5,0,0,26,0,0,0,0,3,0,9,0,0,1,0 +0,1,0,4,2,0,0,5,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,0,100,0,0,0,0,0,0,2,1,1,0,0,0,0,0,3,0,2,5,1,0,0,0,0,6,15,0,2,11,0,0,0,0,0,0,1,10,0,0,0,0,12,33,2,0,0,0,9,0,0,4,0,0,0,0,0,0,0,12,0,0,0,0,3,0,0,0,0,0,0,1,0,0,0,1,7,0,0,0,0,0,0,5,2,0,0,1,3,14,2,0,0,0,0,4,0,0,0,0,0,2,30,1,18,0,0,4,1,0,0,0,0,4,1,0,0,0,0,0,0,6,0,0,3,1,0,0,0,1,1,0,4,14,0,2,0,0,0,0,26,0,0,1,0,0,20,0,1,1,0,0,0,0,0,0,0,0,0,1,0,2,1,0,0,0,0,1,0,0,1,1,1,5,8,2,0,0,4,0,5,8,0,4,2,0,7,0,0,0,0,0,1,0,5,0,4,1,0,0,0,0,0,0,1,0,0,9,0,1,0,0,0,8,0,0,9,0,0,0,3,2,2,0,0,0 +0,0,1,0,0,0,0,9,0,0,49,0,47,2,0,1,0,0,0,0,0,0,0,0,0,68,1,1,0,0,47,5,0,0,7,0,0,3,0,0,0,0,0,0,0,0,0,0,5,0,0,0,1,0,0,0,0,0,0,2,0,0,0,0,0,2,0,9,1,0,0,0,4,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,8,0,0,7,4,1,2,0,0,5,0,0,0,0,19,0,1,0,0,0,0,0,0,1,0,9,0,5,4,0,0,0,0,0,2,0,0,0,0,0,1,0,0,0,16,0,1,6,0,2,0,0,0,0,4,0,1,0,0,0,0,1,10,0,0,0,1,0,6,0,0,1,2,4,0,0,0,1,1,9,24,2,1,3,0,1,0,0,1,0,0,0,0,0,7,0,0,0,0,0,4,0,0,7,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,17,0,0,0,1,0,0,1,4,0,0,0,0,0,0,0,0,1,3,0,1,0,1,0,0,0,0,0,0,0,0,0,57,0 +12,0,0,21,1,11,5,0,5,0,0,0,0,7,0,0,0,0,0,0,0,0,6,10,3,0,0,0,0,0,0,9,14,0,0,0,0,0,0,0,0,2,13,0,0,0,0,0,4,0,0,0,0,0,0,0,9,0,0,0,0,47,3,0,0,0,3,1,0,0,0,3,0,0,6,0,0,0,0,0,0,0,3,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,0,0,0,0,6,0,0,0,1,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,18,9,8,0,1,0,0,2,0,0,0,0,1,0,0,0,0,0,0,3,17,0,1,0,0,0,4,0,0,0,0,0,0,0,2,19,0,1,0,0,0,22,21,0,0,0,1,7,1,1,2,1,0,3,0,0,0,8,0,0,0,0,0,1,4,0,0,73,3,0,3,1,0,5,0,0,0,0,1,0,0,0,1,0,0,0,2,0,0,1,0,27,0,0,0,0,0,0,2,0,7,0,0,12,5,0,0,2,0,0,1,0,0,0,0,10,0,1,0,0,11,0,0,0,0,0 +4,0,2,18,0,4,0,0,64,0,0,14,0,0,0,0,4,0,0,0,0,0,0,0,0,0,8,0,0,4,0,0,0,1,0,0,0,5,0,0,0,0,0,31,0,0,0,0,44,0,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,25,1,0,63,0,0,0,0,0,0,0,0,30,0,0,0,0,0,0,0,0,31,0,4,0,0,0,0,0,0,7,0,0,0,0,7,0,0,8,1,0,0,0,1,0,0,0,0,0,0,0,4,5,0,0,0,0,0,2,0,6,0,0,0,0,0,0,6,1,0,0,2,0,0,0,0,0,0,0,1,0,0,0,0,1,0,9,0,0,0,0,0,0,2,0,0,0,0,0,3,0,0,0,0,0,1,0,1,0,1,0,0,8,0,0,1,0,0,13,0,0,3,2,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,1,0,29,0,0,1,0,7,0,0,0,0,14,1,2,15,0,0 +0,3,7,0,0,6,0,0,16,0,2,0,1,0,0,0,0,0,0,21,2,0,0,0,0,0,0,1,0,0,0,0,2,0,0,8,0,0,0,0,0,12,4,11,0,3,0,0,0,0,2,0,6,0,0,0,0,39,0,0,0,0,0,0,0,2,0,0,1,8,0,0,0,17,0,0,48,0,3,0,0,5,0,0,0,0,0,0,0,0,0,0,5,3,0,0,0,30,0,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,18,0,0,1,0,0,0,0,0,14,0,0,0,0,10,2,7,0,0,2,0,0,0,19,0,0,0,0,0,0,1,6,0,0,1,2,0,0,0,0,0,4,0,20,6,1,0,0,0,0,0,6,0,0,1,0,0,4,41,0,0,0,0,0,0,0,0,0,0,0,0,0,0,37,0,0,0,0,0,2,4,0,2,0,0,0,7,0,0,0,0,0,2,0,0,36,0,0,0,1,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0 +0,0,0,0,0,0,2,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,1,8,0,1,1,3,3,0,0,0,1,0,0,0,0,0,0,0,25,4,1,0,0,0,0,0,25,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,0,0,4,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,16,1,0,1,0,2,3,0,0,8,0,0,0,9,0,3,0,0,0,0,0,0,53,0,2,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,3,0,0,0,0,0,69,0,0,0,0,0,12,11,0,0,0,0,0,0,0,0,7,0,2,0,0,0,0,0,0,0,0,53,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1,0,4,0,4,0,0,4,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,3,0,0,0,5,0,0,0,3,2,3,43,1,0,17,0,12,0,0,0,0,0,0,0,14,0,21,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,19,0,1,0,13,0,0,7 diff --git a/server/config/ds4/r9700-strix/prefill-routing.csv b/server/config/ds4/r9700-strix/prefill-routing.csv new file mode 100644 index 000000000..53bb53273 --- /dev/null +++ b/server/config/ds4/r9700-strix/prefill-routing.csv @@ -0,0 +1,45 @@ +# hotness table: n_layer=43 n_expert=256 n_expert_used=6 +# format: one row per layer, columns are expert activation counts (expert 0..N-1) +151,75,16,23,0,12,85,1,0,21,66,11,1,14,130,1,72,58,2,63,2,11,12,4,3,67,0,39,3,65,2,0,1,1,93,0,12,2,65,2,65,32,20,48,19,41,32,63,3,73,264,22,11,2,14,11,4,74,68,33,14,15,78,130,12,101,14,74,1,1,3,12,3,14,75,2,103,2,4,12,10,73,11,17,4,66,41,10,0,29,25,3,4,264,1,64,17,4,11,1,63,37,45,51,76,20,137,82,2,2,2,38,2,11,1,5,64,2,10,42,76,64,24,14,2,69,65,4,40,1,22,11,66,10,1,66,0,2,31,3,0,3,1,3,18,2,0,33,67,21,76,62,79,14,33,3,3,84,64,2,3,2,26,3,2,10,13,63,3,66,23,2,29,1,94,32,4,1,26,13,4,13,130,23,1,29,1,11,2,11,17,11,0,35,107,4,64,11,12,3,1,2,3,5,74,0,1,14,69,2,23,2,16,31,62,68,266,68,67,19,0,264,4,136,76,63,74,66,74,13,65,4,33,3,74,11,66,2,2,24,17,2,95,73,2,4,0,15,22,3,65,21,72,86,3,263 +145,12,69,22,137,2,86,5,6,15,4,2,3,2,27,79,21,3,32,132,1,18,21,63,32,22,4,89,1,64,11,1,11,24,4,75,128,77,1,5,1,38,2,1,64,33,14,13,24,2,10,149,2,10,2,2,19,77,19,147,4,164,10,1,1,17,73,24,88,24,23,13,1,0,11,17,13,0,1,265,32,20,30,52,7,2,2,2,1,254,1,0,0,2,1,2,1,4,11,11,3,4,63,2,66,15,111,264,2,30,2,11,11,1,3,41,21,78,76,2,16,3,65,63,67,20,27,87,1,0,14,11,2,75,31,129,13,150,16,14,2,3,1,12,78,1,30,2,40,3,11,11,135,1,4,132,14,74,66,21,3,78,1,3,15,1,64,32,68,66,65,35,41,11,1,13,0,3,21,12,12,2,67,126,82,20,0,1,3,13,62,40,139,23,79,13,24,74,62,0,1,2,12,80,14,89,0,3,69,2,66,10,4,2,57,263,24,5,1,11,23,27,14,73,63,23,2,2,96,67,2,4,14,14,23,25,0,222,1,0,35,66,0,141,1,1,3,38,1,62,11,26,2,13,17,13 +6,264,75,13,72,4,4,99,77,1,46,1,15,24,7,2,3,5,11,0,148,12,3,3,127,12,2,1,12,84,11,66,28,31,86,14,21,3,3,12,89,3,101,15,3,62,0,1,192,82,3,0,4,130,4,77,11,1,11,66,1,129,0,6,13,1,13,12,39,2,62,254,12,11,158,88,42,126,2,18,6,148,13,30,29,28,13,77,125,13,63,1,2,3,1,22,28,1,2,12,41,0,1,67,64,33,65,31,67,29,0,4,3,1,14,4,72,13,11,62,65,23,20,23,7,1,3,1,22,12,23,3,13,66,11,14,6,15,4,65,152,64,5,31,14,63,253,3,11,27,85,86,13,26,1,1,11,2,3,0,135,43,1,28,1,0,33,63,1,38,64,30,12,23,198,1,1,1,18,13,37,0,29,75,80,137,12,74,69,2,112,13,5,89,12,77,47,23,2,63,1,4,1,13,2,1,1,20,16,41,103,1,3,23,22,1,0,126,0,18,66,79,1,2,252,64,23,65,1,4,24,14,31,154,5,24,10,31,19,14,1,1,29,5,1,1,4,13,12,11,2,3,3,4,3,2 +20,23,6,231,65,2,80,110,0,5,5,27,0,63,1,331,90,29,0,5,1,0,0,109,18,0,7,17,0,0,0,1,114,0,2,0,0,32,23,17,0,0,4,1,252,41,0,1,0,3,20,0,4,0,214,0,0,101,0,26,0,168,255,7,8,0,0,174,75,0,27,3,53,38,17,24,0,4,68,1,8,0,1,68,14,0,5,15,119,3,85,10,0,2,0,0,41,0,88,1,0,152,1,0,70,24,79,2,40,0,0,0,14,22,8,2,117,96,24,22,132,10,0,22,215,0,21,47,13,52,153,30,34,3,0,30,3,1,19,2,0,64,0,8,3,9,1,57,43,280,11,80,88,53,2,123,40,26,0,16,0,0,0,1,14,254,2,0,0,3,0,42,3,187,5,3,22,5,1,24,61,0,0,13,299,0,12,34,6,0,0,2,10,5,134,21,77,4,4,1,0,7,1,2,0,3,2,0,66,3,0,0,14,23,14,0,0,0,39,0,43,6,0,5,1,230,410,4,0,55,20,3,16,4,128,66,9,14,2,0,73,4,0,0,10,279,5,1,9,1,43,3,18,2,1,1 +85,0,116,357,3,28,0,2,34,2,0,9,7,25,0,184,0,2,0,0,14,0,0,0,0,1,5,66,150,2,0,0,16,1,61,0,19,9,72,140,0,0,0,58,5,137,5,0,110,0,154,0,0,4,0,121,124,29,0,0,0,1,0,53,1,0,39,0,7,64,0,0,1,1,0,0,0,27,11,2,129,2,4,1,142,20,0,2,0,0,1,3,0,1,151,133,0,336,133,0,32,0,0,7,88,6,139,0,1,63,11,2,2,182,18,26,94,3,0,1,6,0,6,46,0,47,43,74,177,166,63,0,0,0,2,25,2,0,1,172,43,2,12,10,0,0,3,87,228,0,45,3,0,15,9,0,68,6,1,28,1,25,71,0,55,8,7,0,0,67,13,0,0,0,79,4,2,5,0,53,64,6,0,4,0,113,3,46,54,324,0,0,2,8,0,1,0,0,65,3,0,11,0,2,132,0,52,54,0,0,8,0,38,134,0,3,2,5,18,3,193,6,0,1,19,0,2,41,1,236,7,1,0,1,0,33,10,2,82,694,6,1,20,34,47,3,0,36,68,67,66,14,0,8,1,4 +204,97,1,9,98,11,0,19,0,30,26,78,2,122,13,3,3,69,1,16,12,1,5,18,117,0,0,5,67,5,1,76,108,0,502,2,29,1,170,2,10,0,26,2,18,70,0,7,56,107,2,1,0,38,0,2,38,282,1,14,347,2,0,9,93,6,0,0,23,0,0,0,20,1,17,5,12,1,2,0,2,0,0,20,46,259,0,64,9,0,112,34,42,16,4,8,1,0,5,1,0,2,1,0,11,1,41,13,1,62,33,0,4,4,124,51,0,1,16,77,13,3,72,252,98,247,7,6,0,122,0,11,91,143,0,15,8,2,0,4,1,22,22,1,0,219,37,4,0,3,4,0,25,64,0,17,45,12,8,1,33,10,5,1,10,0,179,106,0,58,31,20,17,0,2,71,4,13,46,6,129,2,0,15,0,164,1,0,0,298,2,0,0,55,0,1,0,0,2,1,36,7,44,1,81,1,21,0,1,0,48,0,7,11,0,39,2,37,10,0,3,26,0,0,69,0,0,0,31,0,0,2,0,33,19,19,9,0,124,18,16,62,2,27,209,131,34,71,0,52,1,71,182,87,18,13 +74,639,0,62,1,13,12,0,12,73,287,64,76,36,0,6,123,4,71,0,5,0,1,0,0,0,0,0,7,51,0,21,132,0,4,5,72,0,108,16,0,0,70,158,0,69,0,128,100,6,39,62,0,1,170,13,4,6,11,0,0,2,46,0,0,0,39,108,169,77,204,0,0,2,0,5,2,84,119,0,0,1,16,29,0,0,36,1,6,41,13,48,33,59,4,4,3,29,0,6,0,0,23,0,10,20,7,18,152,2,0,63,23,543,0,111,0,8,38,0,0,0,2,0,1,4,187,0,3,8,126,0,6,3,33,0,113,15,10,5,0,84,5,0,2,37,34,13,0,7,0,47,0,14,9,0,76,87,0,0,188,95,0,0,61,64,3,44,30,2,126,1,13,0,7,0,0,1,0,16,0,17,0,3,266,0,49,2,2,5,0,1,0,83,13,44,5,85,0,7,22,79,0,0,0,1,0,15,0,6,0,10,21,1,163,59,0,5,1,3,97,2,1,3,24,5,0,64,94,9,16,0,91,0,93,0,9,1,119,32,202,0,1,17,103,52,0,0,10,3,4,1,0,18,59,40 +1,27,16,3,0,0,1,22,1,51,14,52,0,15,196,4,131,2,2,11,0,27,33,1,2,1,10,15,3,20,17,13,27,0,8,3,59,0,1,0,0,9,1,66,4,0,6,51,0,0,31,62,0,1,116,22,5,3,7,75,40,4,0,0,4,111,151,0,297,12,0,7,91,18,0,0,81,48,11,0,4,200,10,94,10,63,5,0,1,62,1,0,0,0,0,10,22,14,46,1,4,5,95,0,33,9,8,0,1,0,4,0,26,3,0,67,6,0,0,105,3,2,91,241,23,1,0,23,13,0,42,0,14,69,1,40,19,66,16,66,5,434,27,0,37,19,0,24,18,32,59,0,1,95,59,1,2,22,3,0,38,17,0,0,2,1,4,0,16,24,12,0,140,38,4,4,2,649,30,35,160,12,0,0,5,7,1,34,20,25,60,4,41,0,2,24,21,3,2,7,67,0,32,150,5,0,4,7,59,7,1,5,107,33,24,245,76,1,0,17,96,0,16,38,157,0,3,36,0,0,0,0,8,297,0,0,109,70,13,19,1,4,11,0,0,195,633,0,4,35,94,5,3,4,1,0 +13,0,103,12,5,1,1,8,9,30,11,0,1,72,0,217,133,96,2,0,22,0,49,1,84,15,5,0,568,14,14,9,22,58,7,2,11,36,0,1,22,2,3,339,6,7,10,6,0,1,20,5,7,1173,1,0,7,28,11,74,1,2,0,0,5,25,0,4,62,0,126,1,0,1,45,54,0,51,36,11,165,4,8,0,5,33,328,24,15,88,3,4,0,3,1,0,0,92,3,0,2,2,8,20,0,2,17,18,1,2,1,0,0,3,18,73,17,0,31,0,27,29,0,0,6,84,6,0,4,31,3,25,108,244,2,2,0,0,0,0,0,1,59,0,0,33,2,1,0,0,0,0,21,0,2,453,1,78,13,57,0,54,8,0,2,8,0,40,126,0,65,10,68,0,0,1,2,5,5,3,0,2,72,0,0,0,0,0,4,51,74,5,0,53,7,71,5,13,23,111,157,0,4,53,0,109,41,1,7,0,76,0,33,84,0,0,3,51,64,0,254,64,8,19,0,2,136,0,0,11,0,3,1,144,0,3,2,2,6,0,0,0,2,247,0,86,33,0,5,0,68,0,0,3,2,3 +134,0,70,5,2,20,0,1,0,0,2,5,21,21,1,19,3,4,8,0,54,1,235,195,4,2,0,41,10,13,177,14,20,1,24,6,4,3,0,0,36,0,24,77,4,1,202,18,27,0,13,32,16,13,98,84,129,106,3,0,0,11,278,0,4,0,0,14,0,1,2,7,1,12,0,2,32,1,3,1,4,3,9,15,30,68,148,7,18,30,25,6,1,53,53,87,2,33,450,5,1,0,9,0,5,0,9,0,4,5,15,12,1,4,1,1,109,10,3,0,24,2,34,1,59,3,0,0,91,0,4,46,0,2,29,7,65,19,9,1,6,0,13,54,5,224,158,36,8,41,3,50,5,35,0,5,10,11,0,66,242,1,4,679,0,7,4,231,1,11,1,0,2,10,1,1,75,7,124,130,39,210,10,0,89,1,5,103,12,0,0,342,12,2,0,0,26,18,0,4,11,0,0,8,38,83,164,23,0,0,0,4,39,64,0,76,0,0,8,88,1,3,0,0,2,1,0,0,1,1,0,1,60,46,128,218,186,9,15,0,34,0,52,4,0,1,47,43,1,20,10,9,10,61,5,23 +19,14,11,0,168,0,0,22,6,0,0,125,2,225,32,25,185,0,79,1,8,3,0,3,4,8,2,0,8,46,1,41,25,4,37,3,23,12,22,2,484,0,30,0,0,1,43,63,6,76,82,4,0,0,1,2,0,0,1,160,0,0,4,1,210,6,1,0,3,4,0,100,15,65,12,0,0,2,101,62,8,19,0,11,74,1,0,8,2,65,97,2,2,4,6,9,230,0,2,0,40,3,0,110,0,22,10,799,2,9,0,0,35,4,23,4,0,2,0,4,0,0,26,70,0,3,29,40,39,3,26,0,5,3,1,35,57,0,3,0,15,13,14,1,5,18,63,21,6,54,20,0,3,8,9,72,8,5,0,0,0,17,9,4,72,137,1,1,37,0,0,0,175,4,40,48,0,0,20,0,220,15,0,84,116,0,0,0,4,254,0,96,54,0,0,0,62,7,14,0,4,13,0,4,0,78,1,29,79,13,38,0,0,357,7,81,4,19,8,1,0,2,85,2,3,0,11,0,0,35,63,1,5,1,1,0,26,144,0,0,26,1,10,0,3,96,4,125,176,0,251,80,100,1,71,199 +0,0,6,16,3,8,15,0,35,0,23,1,0,2,0,0,0,0,1,0,11,0,4,32,3,0,0,55,3,15,29,0,20,36,71,13,5,0,212,22,0,1,0,0,7,0,2,25,2,0,6,1,15,34,1,27,148,30,24,54,4,23,128,51,19,6,175,24,84,3,98,0,7,118,0,0,0,63,58,0,0,0,0,3,30,98,128,0,0,54,0,0,2,0,237,18,56,27,1,29,2,21,37,53,36,1,0,4,0,298,0,8,207,9,53,0,0,10,13,76,0,224,0,0,64,21,0,89,0,8,6,0,10,6,8,81,2,24,66,183,0,0,0,58,1,132,0,4,1,2,11,131,79,4,0,0,1,2,1,42,3,167,12,3,43,13,54,2,2,11,0,21,2,5,0,4,87,86,13,4,126,3,17,0,58,19,52,4,1,3,2,117,12,73,1,1217,31,24,0,2,1,29,0,4,1,0,2,114,30,81,19,24,41,8,97,257,5,5,11,10,16,8,129,1,4,0,0,0,1,51,0,1,40,23,33,0,7,5,0,27,11,0,0,7,22,2,52,0,66,0,68,54,3,0,0,421 +0,0,21,5,0,33,0,73,0,0,0,10,1,0,413,96,0,0,0,1,5,10,12,0,14,159,13,0,74,0,0,0,0,49,8,0,96,0,1,2,51,0,0,11,0,12,0,0,48,0,13,46,0,0,8,0,0,0,26,2,0,0,68,0,0,0,0,4,6,0,2,72,0,0,0,0,103,18,0,0,3,0,15,21,151,16,2,18,6,0,1,22,0,10,8,2,46,108,0,0,2,1,29,78,0,24,0,3,0,1,12,1,0,18,8,12,9,71,0,34,0,0,17,0,480,72,0,0,4,59,1,134,1,29,39,3,173,0,25,18,7,147,277,53,67,0,3,4,15,0,0,3,31,0,39,0,3,4,40,10,0,60,0,18,172,0,7,4,0,53,0,0,73,0,1,485,7,10,0,0,2,0,29,6,174,72,1,25,0,55,0,3,7,1,4,39,0,0,455,0,3,42,0,0,39,10,636,6,0,11,191,3,131,21,0,0,55,7,129,3,10,23,0,1,4,0,1,6,239,41,10,3,13,3,743,3,20,8,180,2,3,0,4,0,38,2,1,1,0,183,34,30,2,0,0,2 +0,0,40,1,338,0,10,0,0,10,0,0,0,28,132,3,9,9,6,310,20,2,11,0,3,4,0,32,1,71,30,1,16,0,63,2,0,0,9,13,2,18,40,0,12,0,6,200,1,0,0,7,0,116,0,5,0,4,35,110,1,4,13,120,72,11,0,3,9,0,0,0,38,0,49,2,14,13,108,90,251,67,133,0,5,0,8,13,3,11,0,96,1,116,106,10,31,5,10,8,8,0,2,17,183,330,0,21,41,4,0,0,0,4,0,8,14,2,0,19,155,94,5,0,5,29,0,9,0,10,1,15,14,2,0,4,6,0,45,0,0,22,0,218,7,57,13,0,112,14,0,0,13,146,34,5,178,3,7,8,58,0,0,1,0,0,0,38,0,5,3,0,1,0,30,57,1,12,0,867,4,2,199,371,76,5,15,10,14,0,1,0,0,8,2,26,0,0,26,3,189,4,0,0,4,28,15,26,59,0,1,0,1,0,0,23,1,2,18,449,1,1,0,0,2,120,0,0,28,1,0,0,0,2,0,257,110,0,4,0,0,1,0,12,9,11,1,261,25,1,2,1,2,1,2,191 +0,103,54,0,6,22,11,15,270,7,2,8,0,0,11,9,70,5,332,48,10,16,0,1,32,0,0,0,136,1,2,69,29,5,0,0,0,146,20,59,0,2,96,3,0,48,0,0,0,148,1,0,0,299,0,0,4,10,9,13,133,7,0,0,2,320,0,1,0,0,8,0,0,18,3,0,0,0,9,0,0,0,0,332,0,6,0,26,558,4,0,0,2,71,0,61,35,59,31,17,0,44,5,1,103,0,14,8,45,27,5,150,2,0,0,0,0,0,0,95,20,0,15,6,35,624,13,36,22,45,21,0,0,7,1,2,0,0,0,134,0,0,0,0,1,0,0,1,14,278,2,3,87,5,172,0,6,0,1,1,0,23,0,0,4,0,8,4,3,0,85,200,10,60,0,28,0,3,2,14,37,16,23,286,62,93,32,0,13,15,0,87,1,0,0,1,6,3,4,0,18,1,100,0,0,19,0,0,0,4,0,1,47,0,0,1,1,0,0,0,14,0,14,36,6,119,0,210,0,9,23,14,2,0,126,457,0,0,0,2,0,9,17,2,37,4,0,0,14,4,312,0,79,0,0,25 +0,11,24,2,3,11,11,150,1,22,6,3,7,0,18,12,0,1,0,16,24,0,4,0,26,1,1,250,0,18,5,316,0,88,0,5,77,92,383,0,0,0,21,0,1,65,69,0,388,11,1,18,0,0,45,21,24,2,0,32,58,0,133,8,5,88,77,0,0,0,0,0,0,0,59,1,45,0,10,1,2,0,0,125,0,76,4,0,37,0,1,5,54,7,4,13,0,0,2,276,0,0,0,0,27,0,50,18,26,0,69,2,1,0,0,66,0,0,0,12,0,11,0,2,437,0,130,44,11,18,16,26,1,2,1,10,6,1,2,6,9,8,5,0,1,1,760,33,0,0,1,29,14,6,0,0,1,7,14,2,0,31,0,174,73,2,2,63,0,6,0,1,1,0,1,0,390,0,96,0,182,0,0,5,0,0,19,27,65,6,4,0,68,0,0,0,0,0,93,102,67,241,1,0,0,43,0,1,0,40,8,0,0,67,5,1,0,0,1,0,2,6,44,106,38,25,2,0,0,9,0,0,6,301,0,1,107,9,4,68,55,78,11,0,3,87,27,0,39,0,328,0,0,92,136,3 +0,0,37,29,19,0,302,0,101,36,71,1,54,0,0,145,0,0,0,905,24,1,0,3,9,627,13,11,0,0,97,1,0,0,0,4,1,0,37,0,0,9,61,7,0,1,0,28,0,1,46,36,8,0,0,1,72,16,1,0,0,63,9,0,0,30,1,0,9,0,0,5,0,0,8,30,19,27,1,6,4,17,0,0,0,456,3,0,0,14,0,14,0,0,0,0,0,8,8,1,603,0,4,0,2,26,0,1,118,3,0,3,0,0,0,0,6,154,0,0,37,0,1,19,36,0,1,42,1,61,11,2,4,0,0,68,0,0,2,0,0,3,1,4,77,24,0,7,15,5,0,1,0,1,10,58,76,0,5,481,0,2,0,0,30,1,0,4,662,1,0,102,60,14,0,277,88,2,11,22,0,56,8,1,182,0,1,80,37,2,0,4,4,0,106,19,0,3,0,0,0,31,0,1,81,450,0,0,0,24,60,0,1,0,0,157,0,0,1,0,19,33,19,8,1,146,0,36,1,1,0,0,73,0,1,7,0,0,1,5,0,0,0,22,10,50,57,10,1,3,13,4,11,11,2,85 +12,0,0,12,38,313,2,0,0,29,0,5,0,3,1,0,16,16,1,55,0,70,47,12,6,88,19,0,10,11,0,3,77,92,59,0,1,0,0,6,0,4,17,0,0,2,2,0,3,0,3,87,0,616,1,33,0,5,0,346,1,212,0,30,4,257,0,43,0,37,1,9,43,15,0,2,1,54,183,0,0,60,2,9,0,0,80,14,1,1,2,0,72,0,0,0,4,2,7,67,0,3,0,431,54,26,0,6,71,1,200,68,0,89,4,17,4,1,1,8,0,10,0,2,6,0,0,8,27,4,14,0,0,221,88,21,62,12,3,0,2,74,4,9,16,82,669,0,5,5,0,0,43,1,9,0,5,0,0,0,0,0,0,6,10,0,0,4,1,77,13,9,4,1,5,0,1,108,89,4,0,27,4,1,0,294,5,2,0,19,0,0,0,2,61,2,169,12,6,0,25,0,43,5,9,1,0,0,0,124,15,7,0,4,43,0,12,2,13,64,31,0,15,22,5,0,0,3,0,12,3,308,5,0,0,0,60,0,67,0,6,1,79,0,185,0,94,281,399,92,3,11,5,1,0,23 +0,1,0,0,5,1,3,1,535,5,14,59,93,270,0,76,60,0,0,0,5,82,0,204,1,19,0,20,0,1,10,61,0,25,23,1,21,10,0,98,5,3,26,116,4,0,3,0,8,0,0,0,2,0,0,3,2,0,18,0,8,0,0,0,3,0,88,13,1,2,0,0,0,10,16,0,0,0,2,0,8,0,2,5,2,15,53,57,0,18,416,4,127,1,0,0,30,0,0,137,5,48,125,2,0,0,0,0,0,244,17,716,1,20,190,0,911,0,2,1,0,0,0,1,96,1,0,0,1,115,39,8,1,0,10,0,4,156,0,85,0,47,0,0,0,1,0,51,66,0,5,60,12,38,57,194,13,4,22,10,19,145,1,2,0,0,56,12,243,374,0,0,3,83,31,2,33,19,1,41,67,6,67,79,6,4,34,1,1,0,50,5,0,2,0,0,0,2,18,254,0,1,0,1,1,2,10,1,29,1,3,20,41,17,11,11,1,0,0,0,6,0,3,0,0,101,0,122,0,0,0,0,0,2,4,4,2,0,23,6,7,1,0,0,0,0,20,2,0,14,0,46,2,2,21,246 +16,31,0,3,6,62,96,1,1,2,189,5,21,7,0,14,0,115,58,1,164,0,5,160,5,52,6,0,10,709,5,45,10,74,0,148,19,0,18,4,67,5,4,2,17,10,556,3,15,5,32,26,47,70,53,79,13,21,138,11,25,1,1,2,5,1,0,0,21,51,2,0,0,49,192,5,0,168,24,6,0,3,29,21,0,4,60,14,0,0,57,0,19,57,23,54,84,180,2,5,7,8,26,4,26,0,14,10,0,7,5,46,2,22,63,7,8,18,2,0,81,0,7,54,76,0,3,0,39,0,1,0,0,170,56,0,0,145,15,37,17,0,0,0,17,1,0,16,29,63,0,0,17,63,4,0,1,0,1,6,29,53,0,50,6,29,3,60,21,1,7,22,10,73,20,5,36,15,9,20,39,56,25,16,158,151,8,18,0,11,16,3,3,1,3,14,0,4,0,0,5,2,1,64,3,0,25,75,97,25,30,2,26,76,0,203,30,72,8,14,64,86,27,61,59,0,20,3,16,150,0,193,69,37,16,0,246,0,64,13,3,51,10,5,4,60,4,5,0,0,1,58,0,62,1,0 +5,0,0,0,0,0,26,155,9,113,0,3,9,2,3,101,1,0,0,0,5,1264,5,0,40,0,54,0,3,44,0,7,16,0,0,64,18,1,1,47,0,0,1,0,1,28,0,411,27,4,24,4,0,1,6,0,58,191,1,1,0,120,0,13,0,0,99,1,5,4,1,0,1,1,0,10,9,0,7,0,50,4,57,0,1,23,8,1,8,0,0,21,6,3,1,14,1,2,1,179,34,3,2,1,0,42,0,2,3,46,90,8,0,0,0,3,6,0,7,0,1,1,57,0,2,0,0,5,1,2,346,56,14,2,1,14,0,0,2,0,21,8,59,3,0,8,7,10,0,54,0,2,0,0,3,2,57,14,0,156,0,0,0,0,0,13,0,23,83,36,1,53,75,0,4,39,432,5,0,0,31,1,7,62,82,214,1,50,3,1,47,6,1,4,0,0,1,181,11,47,8,0,1,0,1,4,0,25,18,45,3,0,0,0,9,0,7,34,1000,444,8,0,0,98,71,140,1,0,0,2,1,0,53,0,21,13,0,25,0,45,69,270,2,1,19,0,2,2,2,14,103,1,6,61,33,0 +17,107,1,0,4,7,159,132,13,37,0,12,0,11,319,0,34,12,0,445,1,10,6,194,140,1,1,9,36,1,22,6,3,0,61,3,0,61,54,1,1,0,0,49,14,1,22,1,0,282,35,0,2,0,60,101,18,2,3,2,5,2,81,8,1,50,1,76,24,3,2,38,63,0,2,19,3,1,34,0,0,79,9,0,46,69,48,93,24,48,1,0,20,29,1,0,137,3,1,0,1,102,17,8,0,42,0,330,19,42,10,11,0,15,1,0,0,0,2,53,11,0,0,6,5,27,28,5,482,61,13,0,0,17,0,104,136,32,0,22,0,16,3,58,0,0,0,0,0,0,22,5,95,158,1,4,6,1,23,0,169,0,0,13,0,3,6,105,39,308,2,4,45,2,0,3,1,14,0,4,443,0,4,135,28,16,14,0,0,26,5,61,426,22,0,0,0,0,103,46,0,0,171,0,0,3,0,3,0,0,0,34,1,69,13,6,0,200,3,59,4,5,90,8,0,1,2,7,1,0,11,44,1,0,62,0,8,1,67,84,0,4,0,11,65,9,0,0,70,10,0,25,7,0,2,0 +304,2,3,119,0,2,198,7,1,0,0,13,48,1,35,47,17,0,164,6,0,13,159,3,2,9,2,3,5,12,0,175,0,2,1,1,0,12,0,1,12,4,0,6,4,13,17,14,40,3,4,114,3,0,39,131,2,0,0,72,27,14,0,0,9,119,8,6,1,1,13,1,296,3,160,0,3,99,17,9,0,233,15,21,22,0,99,0,0,40,1,182,0,0,1,1,31,2,0,34,13,0,0,26,0,0,11,4,3,0,67,0,20,15,21,31,0,3,6,48,0,0,77,0,0,0,89,0,4,0,0,25,139,0,1,8,4,0,34,0,5,0,9,19,0,31,41,323,0,499,0,193,5,0,0,0,38,6,21,62,84,20,31,2,1,0,668,3,0,0,0,0,13,68,68,5,0,50,3,175,3,35,9,0,0,1,18,20,17,49,0,17,9,90,42,0,0,19,20,0,4,0,111,140,1,154,18,98,7,0,0,45,7,0,0,6,19,13,0,22,3,45,76,0,3,10,1,0,12,4,4,3,7,0,0,82,72,0,93,90,1,68,0,0,2,1,175,3,35,153,18,143,1,160,3,69 +21,11,78,52,2,15,71,150,2,0,8,36,46,9,4,207,0,4,25,26,0,5,0,206,13,79,20,1,13,4,19,15,10,26,1,3,99,3,61,0,2,1,0,94,14,5,0,0,221,19,19,198,2,1,32,50,0,10,16,0,35,165,15,0,161,2,57,80,118,2,70,28,8,0,0,17,35,0,7,11,313,169,1,1,44,9,0,0,12,0,24,9,5,35,52,11,132,54,5,1,2,20,0,30,1,0,0,113,13,5,0,0,28,0,15,3,0,1,134,105,87,1,7,27,3,65,4,2,20,2,3,47,79,16,0,12,0,0,13,0,0,17,16,39,1,2,0,6,18,5,7,0,0,5,34,13,11,0,8,3,2,0,30,22,46,61,13,4,79,0,139,0,0,34,6,5,9,2,0,12,6,52,2,5,9,2,8,132,8,11,3,0,67,51,165,5,0,700,5,27,0,3,1,1,214,0,2,49,9,25,21,100,19,19,20,5,60,36,2,17,22,22,54,2,0,175,0,28,22,1,490,5,85,1,0,2,53,25,6,26,28,137,27,175,0,54,2,12,1,4,20,50,10,98,1,13 +75,1,2,38,11,2,11,37,0,4,237,34,4,0,3,53,27,0,1,50,1,9,75,18,31,0,46,135,51,8,4,0,0,19,16,1,0,3,115,0,71,43,1,61,42,6,140,0,6,3,0,108,47,0,2,0,15,316,65,71,31,142,10,4,19,5,18,20,7,14,39,1,289,11,0,35,67,31,0,73,0,18,0,175,0,1,227,276,0,14,0,0,3,78,5,2,0,0,33,0,0,7,89,54,7,35,3,0,0,0,20,7,1,1,29,223,0,0,0,0,1,69,0,696,27,51,329,0,6,0,25,0,2,14,1,40,0,123,1,46,0,47,40,3,0,1,37,21,0,0,0,3,98,30,73,16,4,41,8,9,38,1,0,0,1,2,19,72,13,0,0,26,17,209,1,9,6,108,8,21,4,0,41,61,11,20,36,0,42,1,0,13,0,5,140,0,21,45,14,0,0,0,10,178,1,54,0,0,12,342,8,7,1,0,11,0,0,22,56,0,18,89,11,0,99,48,6,0,75,1,0,0,28,16,0,11,18,0,0,13,28,0,12,1,151,43,3,32,54,4,1,3,126,1,3,29 +0,32,5,0,24,0,25,15,84,515,10,23,43,2,0,27,0,104,17,2,158,71,28,2,30,0,184,17,6,9,0,1,2,62,10,0,37,101,0,217,0,63,2,2,18,20,0,0,11,4,0,54,8,4,25,13,0,3,48,0,7,43,0,0,2,189,35,62,174,5,313,0,5,132,4,0,6,0,23,13,15,1,4,6,9,16,11,0,21,81,60,31,30,14,633,21,19,4,3,0,108,8,212,19,33,6,2,1,2,0,0,52,12,279,65,7,1,64,24,0,159,15,86,1,3,0,12,31,105,14,38,105,1,34,2,10,49,5,0,2,11,28,21,4,3,0,21,0,25,26,8,14,0,170,65,0,82,57,6,0,0,0,5,0,1,1,77,144,11,8,0,0,29,217,5,22,24,2,74,0,33,3,0,37,28,0,23,22,0,9,0,13,20,32,0,7,2,18,0,23,81,3,0,45,6,150,3,1,5,307,8,1,9,1,70,1,4,3,57,116,4,0,0,48,0,0,28,26,2,3,2,4,2,66,0,29,34,129,2,25,1,30,69,5,59,0,0,87,54,0,0,15,2,0,6,5 +9,75,0,3,0,18,0,69,165,0,54,139,6,0,34,20,2,1,58,28,2,0,6,0,5,8,323,0,5,0,0,0,8,0,91,0,3,10,47,6,0,9,0,28,0,22,0,0,36,6,14,7,35,6,87,1,19,89,30,1,0,104,10,1,2,0,62,2,8,4,7,0,10,155,0,331,0,13,34,681,0,32,0,33,0,0,17,64,0,0,5,7,1,477,65,1,0,42,7,11,50,0,304,3,1,0,4,0,0,0,77,6,0,104,0,0,28,15,32,0,33,4,17,0,0,2,12,0,0,213,15,8,27,0,0,40,3,10,1,6,0,75,22,8,0,3,0,0,1,14,320,1,0,16,0,3,19,5,0,177,22,1,0,225,0,22,40,61,2,21,8,14,16,0,2,1,0,9,21,4,0,0,0,2,64,0,35,0,17,4,3,519,7,3,0,83,0,1,0,0,0,256,0,0,2,115,60,0,3,152,0,33,0,41,4,34,2,0,79,62,0,12,28,12,2,0,4,7,4,17,69,0,1,44,78,3,24,2,9,428,1,0,19,288,0,1,0,29,0,1,5,71,0,52,0,25 +62,38,262,184,4,0,0,9,0,33,16,8,19,0,5,39,87,0,2,0,24,15,1,1,1,156,124,6,173,1,0,2,88,1,29,1,2,0,9,76,25,2,1,26,15,0,4,30,0,4,2,1,294,2,24,17,0,19,6,1,7,56,44,7,1,26,153,153,25,184,2,0,90,44,3,27,0,7,0,17,43,0,47,169,124,14,0,20,17,6,0,0,90,8,45,154,11,0,65,3,2,14,33,35,1,77,21,0,72,10,4,0,0,5,4,2,39,33,19,37,1,104,130,0,13,14,17,0,4,6,2,15,6,33,119,24,1,0,5,19,117,30,3,3,25,2,0,8,48,11,24,315,29,10,2,5,58,36,27,118,10,1,3,13,2,1,14,1,23,2,67,28,493,1,13,39,34,6,105,4,3,26,0,3,8,1,5,18,826,15,0,0,45,43,0,21,30,3,4,27,4,35,11,35,1,0,5,0,146,2,30,7,0,1,1,0,0,0,32,14,3,100,54,0,44,7,0,74,100,76,70,50,16,1,9,8,0,0,3,3,48,0,10,7,88,36,13,90,6,18,2,5,48,74,6,0 +0,5,0,8,41,48,290,83,5,1,1,24,264,29,52,0,13,1,0,188,122,82,4,0,112,8,9,29,44,3,0,5,22,0,6,83,54,14,2,0,0,19,5,194,0,75,21,102,54,6,2,30,228,43,11,1,148,7,12,2,0,16,0,0,1,5,12,23,0,238,0,22,63,0,0,1,4,3,3,1,25,9,5,10,2,18,54,138,25,16,38,2,1,43,24,2,11,14,1,31,43,59,36,15,11,0,76,59,0,118,0,29,60,27,21,109,0,9,8,95,0,31,58,0,12,0,0,18,0,13,563,57,80,2,5,26,9,2,39,5,0,1,6,0,15,0,13,102,1,75,12,2,17,0,100,0,12,9,19,0,102,11,26,27,0,20,167,13,0,31,26,125,0,0,16,3,3,45,3,64,3,0,85,2,0,108,152,0,2,30,71,2,30,131,107,0,41,2,76,2,43,14,7,1,146,14,0,47,8,42,0,42,14,45,33,0,3,1,92,21,0,22,0,0,268,42,52,2,30,2,14,26,23,10,109,16,0,0,0,12,72,59,1,0,33,1,20,80,0,6,23,13,0,73,0,76 +0,56,51,15,10,55,0,9,43,6,22,42,62,140,172,0,12,6,108,2,20,0,3,14,5,30,5,0,1,5,50,4,9,0,2,2,16,2,0,0,1,3,0,0,126,4,3,17,16,16,5,4,12,1,41,12,2,0,3,4,58,70,0,2,1,0,11,0,229,0,4,0,0,32,0,161,4,263,9,12,0,72,0,1,2,2,0,6,4,4,44,315,0,68,8,84,17,2,12,0,2,0,0,5,16,17,0,22,1,4,1,0,38,0,0,5,2,186,0,73,2,1,13,265,0,24,2,2,3,24,1,52,20,0,13,29,15,0,18,43,505,14,1,26,0,40,55,16,250,23,4,0,16,2,1,12,128,18,49,0,0,130,5,1,2,316,157,154,0,65,0,127,14,61,52,9,7,1,161,3,52,37,1,0,1,86,0,1,8,8,134,33,8,70,1,38,37,39,145,1,4,0,6,1,96,10,18,33,8,0,0,7,13,77,6,2,11,0,0,8,142,21,5,15,2,1,136,0,0,0,375,0,53,3,13,0,207,55,61,0,5,21,18,115,159,26,0,0,0,1,1,14,195,17,5,0 +2,1,22,9,2,23,1,29,11,18,0,124,31,0,14,1,11,48,2,0,0,0,2,83,4,29,53,0,0,0,2,20,64,6,1,1,14,76,27,42,63,7,4,69,9,8,2,0,0,28,4,72,5,26,38,0,302,9,11,1,3,3,4,3,0,1,42,3,1,1,2,0,6,41,6,91,34,0,293,0,21,0,4,67,7,224,0,5,10,9,13,51,7,5,120,0,24,1,0,53,36,3,4,127,0,27,0,7,0,8,4,11,0,1,4,14,56,169,342,15,3,77,5,67,78,14,40,0,1,0,6,19,34,3,0,87,68,126,11,1,8,42,3,46,2,9,73,42,70,1,25,0,0,26,30,4,1,39,262,4,11,33,1,2,0,1,4,67,31,3,4,189,371,10,5,58,1,47,10,0,103,74,21,15,54,0,1,17,0,15,37,217,35,272,1,0,5,0,0,2,4,7,10,14,0,59,4,1,194,12,40,141,113,50,5,0,16,0,5,51,17,8,2,0,0,22,44,6,22,6,84,1,0,31,201,54,72,79,3,0,9,0,9,13,3,7,163,4,18,23,0,44,241,17,2,347 +119,3,7,15,152,0,4,7,2,5,0,0,4,18,255,0,27,0,31,78,101,42,7,0,14,308,4,35,9,0,21,5,55,34,1,9,120,81,1,27,8,41,7,6,15,11,163,61,0,17,223,2,0,4,16,2,34,34,14,5,67,2,4,1,55,21,42,3,98,7,0,3,63,1,9,60,4,10,164,3,1,8,0,53,15,0,15,13,3,38,65,99,11,114,37,0,17,0,41,0,1,28,10,0,26,28,32,16,0,2,16,42,0,21,14,5,1,0,134,70,2,41,57,99,5,91,41,104,51,2,333,0,19,0,72,1,0,3,0,0,1,26,17,101,1,66,0,49,16,0,13,36,27,28,0,16,55,139,31,4,13,1,17,46,15,2,32,0,189,45,49,6,0,7,8,12,99,191,0,132,5,9,4,16,48,286,0,0,2,170,54,10,12,9,2,2,46,3,4,2,0,0,84,6,4,1,17,78,112,0,22,6,51,77,4,15,10,4,3,1,25,1,166,1,8,0,5,1,0,78,4,1,3,125,30,0,0,7,9,0,85,29,9,71,7,276,9,116,100,1,13,0,0,1,34,74 +3,0,9,2,10,22,38,11,0,3,37,11,9,3,0,4,0,93,0,24,0,1,21,14,61,13,23,6,54,0,119,60,9,162,0,188,12,2,0,83,114,13,0,0,127,19,10,26,1,0,0,16,12,43,1,9,6,294,4,1,0,85,63,0,0,2,4,3,0,0,50,2,0,0,194,13,6,29,42,0,0,9,35,7,0,0,8,23,46,96,6,0,0,2,248,1,9,0,0,0,66,10,36,16,33,0,115,71,0,11,2,0,56,3,7,1,4,392,45,1,34,4,0,1,15,0,20,10,192,93,37,77,2,167,32,4,2,0,0,3,1,33,1,4,7,294,9,8,0,0,5,19,12,158,90,301,8,11,4,36,69,68,6,3,1,43,22,43,3,21,151,4,6,17,2,9,5,3,16,6,64,34,14,7,20,127,21,0,2,6,0,0,1,743,20,42,11,139,34,27,0,10,0,94,6,3,0,0,0,74,53,13,0,11,46,13,2,0,0,175,26,18,9,4,18,2,105,2,1,70,1,2,11,21,44,34,89,16,2,25,13,1,0,19,65,229,41,2,72,0,0,32,17,0,0,86 +1,6,42,1,25,0,0,3,8,103,0,20,8,1,1,9,0,1,4,2,6,11,30,6,24,15,235,620,8,4,37,5,1,0,2,19,11,2,3,14,0,394,1,14,0,1,1,7,1,20,7,3,0,4,0,0,7,24,459,152,8,20,32,46,0,159,38,6,35,0,17,8,9,0,334,49,0,9,72,49,62,10,0,0,4,1,4,13,127,35,47,114,10,5,35,62,2,0,1,0,1,0,2,4,20,43,140,185,4,0,24,22,0,107,3,0,4,8,0,0,4,16,2,31,87,0,3,128,7,18,18,16,2,0,205,0,9,9,22,58,1,0,35,0,3,9,3,5,42,9,11,0,3,108,21,7,132,103,4,23,1,24,0,0,116,7,15,7,15,1,2,110,10,31,121,14,0,9,5,0,0,2,21,0,259,0,280,24,1,27,254,8,2,8,2,123,27,0,160,0,32,4,0,14,16,0,6,128,0,1,0,6,34,51,5,2,31,0,1,39,76,109,13,26,25,2,0,85,7,3,0,17,0,0,5,25,4,6,3,9,0,15,105,0,121,1,26,17,1,11,8,3,3,110,317,0 +0,64,26,68,19,0,11,26,11,1,231,11,1,6,54,15,19,192,116,23,48,14,2,0,22,1,18,0,0,8,182,67,26,2,5,3,4,7,226,39,2,0,139,5,60,11,0,40,44,25,0,87,0,1,32,0,6,11,208,10,33,0,61,232,7,26,14,12,10,11,2,21,2,0,0,174,18,0,1,97,44,17,71,37,26,2,0,0,1,27,32,1,109,55,72,7,0,6,1,34,116,0,14,33,0,3,0,132,7,1,7,7,0,0,0,28,5,9,5,0,11,17,0,4,202,2,7,13,0,32,22,4,0,9,3,0,23,193,0,0,76,89,21,103,24,7,0,3,19,0,3,3,104,1,1,21,5,0,11,2,1,105,22,298,26,0,0,0,2,76,166,3,0,141,0,14,44,15,0,23,161,3,3,2,1,67,16,3,1,0,0,110,1,46,6,35,27,15,38,0,2,75,84,2,64,0,38,50,7,12,0,5,9,2,0,5,2,0,0,14,0,1,6,0,104,235,327,22,0,167,0,48,218,292,2,22,4,54,72,0,41,11,0,16,52,9,3,3,3,177,11,59,0,2,0,19 +31,132,4,0,15,2,3,6,6,1,8,2,1,2,12,0,75,64,8,1,16,23,31,1,17,24,5,1,35,3,9,3,12,0,1,20,0,14,73,14,1,5,130,1,0,7,0,45,9,0,10,2,0,12,5,176,22,68,1,41,2,0,5,54,38,10,21,5,36,0,9,39,2,24,6,151,169,24,224,7,28,78,1,12,113,7,5,75,17,22,1,120,6,100,0,126,76,54,32,44,39,25,30,218,1,65,7,0,3,2,1,32,2,0,10,34,9,4,285,0,6,0,118,20,0,89,74,0,1,18,16,33,67,11,5,82,0,0,141,0,1,22,0,3,0,3,5,33,2,0,2,13,0,8,1,8,28,6,11,13,12,68,23,13,8,0,22,178,2,0,17,4,3,3,711,13,210,32,0,3,1,16,5,1,0,29,1,5,53,15,10,74,1,9,9,6,117,10,5,4,0,26,30,70,0,0,6,97,34,150,0,15,4,12,105,81,91,10,5,39,0,80,113,16,17,8,51,0,6,1,0,1,29,264,108,77,126,318,55,17,2,6,0,37,58,1,13,6,91,19,0,1,7,155,1,24 +1,1,0,38,26,6,32,12,1,2,26,19,0,3,242,0,8,31,27,40,9,124,13,0,16,0,0,85,207,0,3,7,1,0,96,1,109,0,4,75,0,0,6,36,28,0,3,18,11,42,136,39,1,17,0,31,0,74,15,3,1,0,1,4,7,0,8,0,6,0,4,2,3,9,0,1,4,67,1,308,0,0,6,63,0,14,3,6,0,236,128,54,5,74,7,2,36,9,25,16,2,71,1,239,97,4,147,32,8,16,11,68,3,22,2,17,38,7,22,3,0,170,16,88,11,52,105,10,112,0,0,0,4,44,55,182,0,122,11,25,49,13,80,18,84,12,228,18,0,0,84,0,0,62,0,138,42,0,1,15,0,1,73,131,1,3,109,0,1,6,0,10,6,36,118,0,51,14,4,24,4,5,33,0,20,0,51,19,0,2,4,7,0,5,1,3,5,11,159,12,0,1,10,71,10,1,90,0,7,2,0,96,8,0,1,39,63,24,4,374,9,1,305,12,3,41,131,33,43,0,2,2,63,117,11,12,192,17,24,37,0,41,0,4,90,0,0,0,5,11,1,89,0,1,29,184 +1,6,0,3,536,126,23,53,17,78,7,13,0,3,0,83,22,0,4,0,7,2,8,0,1,48,2,0,3,19,21,173,0,17,72,43,6,282,13,1,22,0,6,13,6,65,8,6,20,161,12,54,1,0,1,286,139,2,90,74,8,2,3,7,0,0,66,94,4,3,16,2,37,71,53,0,5,1,94,0,8,15,11,82,0,2,106,0,4,65,1,3,8,5,49,4,3,1,4,91,0,9,8,0,0,6,59,6,63,0,20,0,8,231,39,0,86,14,39,137,33,1,23,92,0,246,2,0,1,21,0,21,124,7,327,72,5,23,16,22,0,0,24,158,72,1,0,0,13,1,0,19,5,1,6,22,0,1,1,87,19,0,17,57,0,74,2,0,16,0,47,3,2,18,5,1,203,68,96,44,0,7,0,5,1,0,9,30,0,3,0,16,1,40,0,0,1,28,0,2,227,47,3,21,61,6,0,7,15,0,27,12,1,44,245,5,23,3,11,18,0,1,442,3,105,11,17,9,79,0,8,1,0,4,8,3,6,158,0,28,2,1,43,13,5,1,126,0,9,20,9,240,17,1,15,1 +0,2,28,2,1,0,26,22,0,0,268,0,250,59,0,47,0,13,6,11,35,14,3,50,0,68,8,157,0,5,152,26,46,1,70,5,0,315,3,3,73,5,5,1,0,146,0,15,15,0,67,1,16,12,0,1,69,0,0,13,17,2,1,1,4,78,1,302,4,28,1,0,10,100,7,27,10,1,12,0,16,11,20,1,4,0,2,65,9,64,0,0,0,18,53,7,0,0,2,1,132,22,0,249,4,0,54,20,105,2,2,0,21,2,10,5,1,438,186,20,20,17,3,0,29,286,8,0,108,5,55,103,0,3,0,17,6,152,75,26,14,1,0,15,21,9,0,57,47,63,54,0,73,88,0,5,19,37,0,16,2,67,6,0,103,47,2,16,0,9,24,45,1,3,99,0,26,0,33,8,38,1,77,94,1,45,59,528,57,0,0,0,0,4,0,0,0,59,0,23,0,3,5,53,2,3,55,0,131,0,28,5,12,0,2,0,9,6,7,3,1,1,184,142,1,1,0,0,98,15,85,2,0,2,22,1,0,0,0,6,7,0,9,1,12,81,0,18,61,4,74,0,0,8,377,3 +91,9,14,201,4,149,54,63,46,9,0,3,45,88,5,12,108,0,197,0,0,2,33,38,160,18,33,10,1,21,2,123,88,125,2,28,18,0,79,1,15,18,71,80,0,17,1,6,25,14,34,11,13,0,0,27,11,17,0,1,7,386,16,2,17,47,51,10,0,1,0,11,0,48,76,7,0,5,4,69,10,25,30,0,48,36,2,1,0,2,9,16,116,6,0,9,9,0,50,1,4,136,4,0,16,120,21,4,11,148,111,0,1,21,80,0,91,7,18,5,0,36,0,0,60,114,5,1,1,14,177,162,34,50,75,2,5,2,100,13,3,0,5,7,1,4,0,59,4,22,208,3,19,0,8,2,30,30,91,13,0,0,6,4,39,88,45,1,33,20,14,81,120,0,1,0,2,36,8,155,22,27,8,72,2,63,8,13,30,0,10,1,35,46,59,0,0,79,29,21,73,41,8,259,26,2,0,13,42,246,5,77,2,7,18,34,14,21,8,30,35,193,20,29,35,13,61,32,5,12,59,0,1,48,47,5,0,16,16,11,31,9,17,44,0,91,7,62,13,21,106,7,54,4,2,0 +24,102,21,96,0,31,150,10,461,1,47,44,2,1,6,14,11,13,0,7,1,5,15,0,11,0,10,13,10,42,4,0,0,476,11,39,26,8,3,33,3,13,1,153,0,13,5,0,50,0,256,0,0,55,17,9,14,25,0,4,74,6,7,0,0,0,0,0,3,3,47,0,15,0,48,1,3,6,0,1,70,3,5,509,2,0,154,4,0,0,1,1,1,3,0,280,1,6,14,10,9,47,36,46,358,3,110,8,9,29,42,0,80,26,0,0,9,3,67,1,0,15,30,0,0,9,44,27,6,103,147,27,6,63,135,36,0,71,1,0,8,9,4,58,25,4,8,28,79,35,17,6,1,311,1,5,34,0,7,2,0,0,100,0,10,0,0,177,1,211,0,0,0,14,2,27,10,1,11,2,6,0,35,7,0,7,0,3,83,1,26,10,28,64,0,18,1,55,83,2,0,101,0,1,26,90,145,16,9,2,10,115,0,0,0,0,0,33,0,22,0,122,2,2,1,0,33,2,9,6,47,61,0,5,1,8,0,22,95,28,176,0,5,29,0,131,36,0,65,0,73,9,209,104,0,1 +4,86,5,1,22,24,14,65,151,24,46,0,65,4,1,0,3,39,0,26,24,53,11,5,0,3,9,106,220,0,1,0,71,0,56,64,4,3,1,20,5,77,29,30,5,56,5,0,1,1,57,0,46,0,28,4,0,104,11,155,0,6,0,4,2,12,6,0,8,12,2,0,20,135,0,5,193,20,50,2,39,139,2,55,63,0,6,5,1,1,24,9,81,92,0,11,6,111,0,45,1,1,4,45,1,1,0,65,188,5,0,11,1,0,0,15,69,3,836,83,110,138,9,4,0,10,74,35,40,66,0,1,18,9,50,0,6,3,11,6,19,98,35,9,26,30,25,0,22,47,0,134,43,72,1,13,0,0,0,28,22,126,34,91,66,14,0,76,24,167,3,0,9,0,4,134,480,1,15,10,31,0,236,4,55,37,13,1,7,1,1,82,3,0,2,30,13,7,4,18,88,0,0,35,123,2,0,16,2,63,39,106,7,66,6,0,8,13,37,2,0,4,28,85,0,5,15,6,5,0,0,0,50,145,0,27,7,1,0,32,0,23,0,41,1,22,5,1,2,201,9,1,2,49,0,2 +3,0,0,1,25,0,16,2,0,1,0,40,28,29,15,41,0,0,6,0,5,2,7,102,0,3,8,58,39,13,1,0,13,22,6,75,0,1,8,3,149,49,34,0,0,24,22,0,150,0,56,192,2,79,27,0,0,0,0,3,2,17,35,2,2,0,111,18,0,0,136,118,11,70,3,5,0,34,8,48,40,154,14,59,54,3,1,3,0,6,81,61,74,18,0,10,14,18,1,12,312,13,0,9,260,4,6,14,0,34,0,0,36,239,5,7,0,1,1,2,1,0,50,80,312,42,310,0,1,17,92,100,139,18,0,0,5,95,9,0,1,0,1,15,111,3,132,28,0,16,110,2,71,143,3,16,98,0,8,45,2,128,14,0,83,1,0,24,7,44,0,0,52,1,10,2,10,123,5,31,0,153,9,16,3,1,115,0,6,10,147,61,16,238,6,2,0,52,9,167,66,0,6,34,0,2,51,0,1,16,45,8,0,9,9,9,36,32,0,182,0,136,2,3,27,11,0,1,118,91,2,131,0,8,55,0,43,14,0,12,51,0,5,0,2,0,1,50,55,6,130,9,77,0,5,11 diff --git a/server/config/hipblaslt/r9700-strix-rocm-7.2.4.txt b/server/config/hipblaslt/r9700-strix-rocm-7.2.4.txt new file mode 100644 index 000000000..48b8ab281 --- /dev/null +++ b/server/config/hipblaslt/r9700-strix-rocm-7.2.4.txt @@ -0,0 +1,81 @@ +Git Version: dabb6df2b9 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,24,2048,16384,1,16384,1,0,16384,1,24,1,24,1,f16_r,f16_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,19509.6,768.162,82.5549,88763,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,256,2048,4096,1,4096,0,0,4096,0,256,0,256,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,12088.1,104.444,355.306,91222,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,133,320,512,1,512,0,0,512,0,133,0,133,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2125.17,49.8641,20.5073,91216,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,320,133,1,133,0,0,133,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2178.02,51.1042,20.0097,91222,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,661,320,512,1,512,0,0,512,0,661,0,661,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1579.77,19.3943,137.106,91216,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,320,661,1,661,0,0,661,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,4622.6,56.7501,46.856,91222,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,149,320,512,1,512,0,0,512,0,149,0,149,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2313.3,50.7994,21.1059,91216,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,320,149,1,149,0,0,149,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2582.89,56.7196,18.903,91216,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,128,64,512,1,512,0,0,512,0,128,0,128,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,517.647,24.4815,16.2053,91215,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,64,128,1,128,0,0,128,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,842.569,39.8482,9.95599,91215,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,640,64,512,1,512,0,0,512,0,640,0,640,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2112.44,75.3132,19.8552,91216,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,64,640,1,640,0,0,640,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1791.92,63.886,23.4067,91215,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,144,64,512,1,512,0,0,512,0,144,0,144,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,494.037,22.5661,19.1022,91216,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,64,144,1,144,0,0,144,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,987.642,45.1124,9.55527,91215,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,677,320,512,1,512,0,0,512,0,677,0,677,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1609.25,19.6491,137.852,91216,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,320,677,1,677,0,0,677,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,4257.54,51.9849,52.105,91222,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,165,320,512,1,512,0,0,512,0,165,0,165,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2107.42,43.7238,25.6556,91216,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,320,165,1,165,0,0,165,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2537.56,52.6481,21.3068,91216,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,661,240,512,1,512,0,0,512,0,661,0,661,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1326.58,18.8599,122.456,91217,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,240,661,1,661,0,0,661,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,4285.87,60.9319,37.903,91222,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,677,240,512,1,512,0,0,512,0,677,0,677,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1447.97,20.4892,114.906,91215,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,240,677,1,677,0,0,677,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,4260.11,60.2819,39.0553,91222,gfx1201,32 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,24,2048,16384,1,16384,1,0,16384,1,24,1,24,1,f16_r,f16_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,4136.3,162.861,389.385,7525,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,256,2048,4096,1,4096,0,0,4096,0,256,0,256,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1951.16,16.8584,2201.24,7602,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,133,320,512,1,512,0,0,512,0,133,0,133,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1795.01,42.1173,24.2793,7600,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,320,133,1,133,0,0,133,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1925.19,45.1719,22.6375,7600,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,661,240,512,1,512,0,0,512,0,661,0,661,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2046.33,29.0925,79.3848,7600,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,240,661,1,661,0,0,661,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2122.52,30.1757,76.5352,7600,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,149,320,512,1,512,0,0,512,0,149,0,149,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1853.91,40.7113,26.3359,7600,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,320,149,1,149,0,0,149,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2002.68,43.9783,24.3795,7600,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,128,64,512,1,512,0,0,512,0,128,0,128,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,538.439,25.4648,15.5795,7599,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,64,128,1,128,0,0,128,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1057.11,49.9945,7.93545,7600,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,640,64,512,1,512,0,0,512,0,640,0,640,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1933.97,68.9503,21.6875,7600,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,64,640,1,640,0,0,640,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1745.43,62.2285,24.0301,7600,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,144,64,512,1,512,0,0,512,0,144,0,144,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,593.985,27.1314,15.8879,7599,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,64,144,1,144,0,0,144,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,917.937,41.9285,10.2809,7602,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,677,240,512,1,512,0,0,512,0,677,0,677,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2055.83,29.0906,80.9306,7600,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,240,677,1,677,0,0,677,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2109.1,29.8444,78.8866,7600,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,165,320,512,1,512,0,0,512,0,165,0,165,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,1888.35,39.1785,28.632,7600,gfx1151,20 + transA,transB,grouped_gemm,batch_count,m,n,k,alpha,lda,stride_a,beta,ldb,stride_b,ldc,stride_c,ldd,stride_d,a_type,b_type,c_type,d_type,compute_type,scaleA,scaleB,scaleC,scaleD,amaxD,swizzle_a,swizzle_b,activation_type,bias_vector,bias_type,aux_type,rotating_buffer,flush,use_gpu_timer,hipblaslt-Gflops,hipblaslt-GB/s,us,solution_index,gcnArchName,CUs + T,N,0,1,512,320,165,1,165,0,0,165,0,512,0,512,0,f32_r,f32_r,f32_r,f32_r,f32_r,0,0,0,0,0,0,0,none,0,f32_r,f32_r,0,1,0,2037.23,42.2674,26.5396,7600,gfx1151,20 diff --git a/server/deps/llama.cpp/ggml/include/ggml.h b/server/deps/llama.cpp/ggml/include/ggml.h index a09f132e6..f5371f5af 100644 --- a/server/deps/llama.cpp/ggml/include/ggml.h +++ b/server/deps/llama.cpp/ggml/include/ggml.h @@ -2485,6 +2485,7 @@ extern "C" { GGML_MOE_FUSED_DEFERRED_PEER_COPY = -3, GGML_MOE_FUSED_OWNER_SPLIT = -4, GGML_MOE_FUSED_ALIGN_IDS = -5, + GGML_MOE_FUSED_BALANCED_OWNER_IDS = -6, }; // Word offsets in ggml_tensor::op_params for the deferred peer-copy op. @@ -2518,7 +2519,8 @@ extern "C" { // Coarse DeepSeek-V4 routed-owner op. gate_up contains concatenated gate // and up output rows; the backend performs fused gate/up MMVQ + clamped - // SwiGLU, down MMVQ, route weighting, and local reduction. + // SwiGLU, down MMVQ, route weighting, and local reduction. An optional + // owner_residual is added in that final reduction without another launch. GGML_API struct ggml_tensor * ggml_ds4_moe_owner( struct ggml_context * ctx, struct ggml_tensor * input, @@ -2526,13 +2528,15 @@ extern "C" { struct ggml_tensor * down_w, struct ggml_tensor * expert_ids, struct ggml_tensor * expert_weights, + struct ggml_tensor * owner_residual, int64_t ff_dim, float swiglu_clamp, float down_scale); // Coarse owner variant for checkpoints that store gate and up tensors // separately. The backend fuses their MMVQ traversal and SwiGLU while - // preserving the checkpoint's three external value scales. + // preserving the checkpoint's three external value scales. The optional + // owner_residual has the same semantics as in ggml_ds4_moe_owner. GGML_API struct ggml_tensor * ggml_ds4_moe_owner_split( struct ggml_context * ctx, struct ggml_tensor * input, @@ -2541,6 +2545,7 @@ extern "C" { struct ggml_tensor * down_w, struct ggml_tensor * expert_ids, struct ggml_tensor * expert_weights, + struct ggml_tensor * owner_residual, int64_t ff_dim, float swiglu_clamp, float gate_scale, @@ -2554,6 +2559,20 @@ extern "C" { struct ggml_context * ctx, struct ggml_tensor * expert_ids); + // Map global route IDs directly to one owner's compact expert stack while + // assigning a batch-wide main-resident quota encoded as four times the + // desired routes per token. The peer owner receives the exact complement. + // This fuses the former repeated-LUT lookup and route masking chain into + // one small device-local operation. + GGML_API struct ggml_tensor * ggml_ds4_moe_balanced_owner_ids( + struct ggml_context * ctx, + struct ggml_tensor * global_ids, + struct ggml_tensor * router_weights, + struct ggml_tensor * local_id_lut, + struct ggml_tensor * main_candidate_lut, + int main_slots_x4, + bool main_owner); + // Copy an F32 peer-GPU tensor only after a scheduler-provided device event // has completed. The scheduler writes the native event handle into the op // parameters and deliberately leaves src on its owner backend. diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu index e4f43f0be..5f1954e0c 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu @@ -719,7 +719,7 @@ static bool rocmfp4_x4_enabled() { return enabled; } -static bool rocmfp4_q5_x4_plus1_enabled() { +static bool rocmfp4_qwide_x4_tail_enabled() { static const bool enabled = []() { const char * value = std::getenv("DFLASH_CUDA_MMVQ_FP4_Q5_X4_PLUS1"); return value && value[0] == '1' && value[1] == '\0'; @@ -767,8 +767,8 @@ static __global__ void mul_mat_vec_q( "packed FP3 MMVQ specialization requires ROCmFP3 weights"); static_assert(!c_fp4_x4 || (type == GGML_TYPE_Q4_0_ROCMFP4_FAST && - (ncols_dst == 4 || ncols_dst == 5)), - "FP4 x4 MMVQ specialization requires ROCmFP4-fast q4/q5"); + (ncols_dst >= 4 && ncols_dst <= 6)), + "FP4 x4 MMVQ specialization requires ROCmFP4-fast q4..q6"); const uint32_t channel_dst = blockIdx.y; @@ -982,10 +982,13 @@ static __global__ void mul_mat_vec_q( tmp[1][i] += dots.y; tmp[2][i] += dots.z; tmp[3][i] += dots.w; - if constexpr (ncols_dst == 5) { - tmp[4][i] += vec_dot_q_mmvq( - vx, &y[4*stride_col_y + kby], - kbx_offset + i*stride_row_x + kbx, kqs); + if constexpr (ncols_dst > 4) { +#pragma unroll + for (int j = 4; j < ncols_dst; ++j) { + tmp[j][i] += vec_dot_q_mmvq( + vx, &y[j*stride_col_y + kby], + kbx_offset + i*stride_row_x + kbx, kqs); + } } if constexpr (has_fusion) { if (use_gate) { @@ -1002,10 +1005,13 @@ static __global__ void mul_mat_vec_q( tmp_gate[1][i] += gate_dots.y; tmp_gate[2][i] += gate_dots.z; tmp_gate[3][i] += gate_dots.w; - if constexpr (ncols_dst == 5) { - tmp_gate[4][i] += vec_dot_q_mmvq( - vgate, &y[4*stride_col_y + kby], - kbx_offset + i*stride_row_x + kbx, kqs); + if constexpr (ncols_dst > 4) { +#pragma unroll + for (int j = 4; j < ncols_dst; ++j) { + tmp_gate[j][i] += vec_dot_q_mmvq( + vgate, &y[j*stride_col_y + kby], + kbx_offset + i*stride_row_x + kbx, kqs); + } } } } @@ -2228,9 +2234,24 @@ static void mul_mat_vec_q_switch_ncols_dst( 0, ids_stride, stream); return; } - if (!has_ids && ncols_dst == 5 && rocmfp4_x4_enabled() && - rocmfp4_q5_x4_plus1_enabled()) { - constexpr int c_ncols_dst = 5; + if (!has_ids && (ncols_dst == 5 || ncols_dst == 6) && + rocmfp4_x4_enabled() && rocmfp4_qwide_x4_tail_enabled()) { + if (ncols_dst == 5) { + constexpr int c_ncols_dst = 5; + std::pair dims = calc_launch_params( + c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, + warp_size, table_id); + mul_mat_vec_q_switch_fusion( + vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, + stride_row_x, stride_col_y, stride_col_dst, + channel_ratio_fd, stride_channel_x, stride_channel_y, + stride_channel_dst, sample_ratio_fd, stride_sample_x, + stride_sample_y, stride_sample_dst, dims.first, dims.second, + 0, ids_stride, stream); + return; + } + constexpr int c_ncols_dst = 6; std::pair dims = calc_launch_params( c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id); diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/moe-fused.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/moe-fused.cu index f153d122f..fef5a0521 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/moe-fused.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/moe-fused.cu @@ -265,6 +265,7 @@ static __global__ void moe_fused_kernel( static __global__ void laguna_moe_combine_kernel( const char * __restrict__ experts, const char * __restrict__ weights, + const char * __restrict__ owner_residual, char * __restrict__ output, const int n_embd, const int n_used, @@ -274,6 +275,8 @@ static __global__ void laguna_moe_combine_kernel( const size_t experts_nb2, const size_t weights_nb0, const size_t weights_nb1, + const size_t residual_nb0, + const size_t residual_nb1, const size_t output_nb0, const size_t output_nb1, const float value_scale) { @@ -304,6 +307,12 @@ static __global__ void laguna_moe_combine_kernel( const float prod = __fmul_rn(scaled, w); sum = (e == 0) ? prod : __fadd_rn(sum, prod); } + if (owner_residual != nullptr) { + const float residual = *(const float *)(owner_residual + + (size_t)h * residual_nb0 + + (size_t)t * residual_nb1); + sum = __fadd_rn(sum, residual); + } *(float *)(output + (size_t)h * output_nb0 + (size_t)t * output_nb1) = sum; @@ -323,8 +332,10 @@ static __global__ void ds4_peer_copy_f32_kernel( // of every valid output encode the original route slot; invalid entries use // the sign bit plus the original slot. The dedicated MoE MMVQ kernel decodes // this metadata and scatters its result back to the original route layout. -// A single thread is deliberate: this is at most a 4 x 6 assignment problem -// and runs once per owner/layer, outside the weight-streaming kernels. +// A single thread is deliberate: practical speculative MoE batches are tiny, +// and this runs once per owner/layer outside the weight-streaming kernels. The +// bounded fast path covers common top-k and speculative widths; larger shapes +// retain the original route order and exact scatter metadata. static __global__ void ds4_align_moe_ids_kernel( const int32_t * __restrict__ ids, int32_t * __restrict__ aligned, @@ -336,8 +347,8 @@ static __global__ void ds4_align_moe_ids_kernel( return; } - constexpr int max_routes = 6; - constexpr int max_tokens = 4; + constexpr int max_routes = 16; + constexpr int max_tokens = 16; if (n_routes > max_routes || n_tokens > max_tokens) { for (int t = 0; t < n_tokens; ++t) { for (int r = 0; r < n_routes; ++r) { @@ -435,6 +446,49 @@ static __global__ void ds4_align_moe_ids_kernel( } } +static __global__ void ds4_balanced_owner_ids_kernel( + const int32_t * __restrict__ global_ids, + const float * __restrict__ router_weights, + const int32_t * __restrict__ local_id_lut, + const float * __restrict__ main_candidate_lut, + int32_t * __restrict__ owner_ids, + int n_routes, + int n_tokens, + int n_expert, + int main_slots_x4, + bool main_owner) { + if (blockIdx.x != 0 || threadIdx.x != 0) { + return; + } + + // Assign one shared quota over the complete verification batch. Quarter- + // route units expose the otherwise unreachable 16/30 split at q=5 while + // keeping the decision device-local and identical on both owners. + const int main_quota = (main_slots_x4 * n_tokens) / 4; + int assigned_main = 0; + for (int token = 0; token < n_tokens; ++token) { + const int row = token * n_routes; + for (int route = 0; route < n_routes; ++route) { + const int index = row + route; + const int32_t global_id = global_ids[index]; + const bool active = router_weights[index] != 0.0f; + const bool valid_global = global_id >= 0 && global_id < n_expert; + const bool main_candidate = active && valid_global && + main_candidate_lut[global_id] != 0.0f; + const bool route_on_main = + main_candidate && assigned_main < main_quota; + if (route_on_main) { + ++assigned_main; + } + + const bool keep = active && valid_global && + (main_owner ? route_on_main : !route_on_main); + const int32_t local_id = keep ? local_id_lut[global_id] : -1; + owner_ids[index] = local_id >= 0 ? local_id : -1; + } + } +} + static ggml_tensor make_contiguous_f32_tensor( float * data, int64_t ne0, @@ -462,6 +516,7 @@ static void ggml_cuda_op_ds4_moe_owner( const ggml_tensor * down_w = dst->src[2]; // [n_ff, n_embd, n_expert] const ggml_tensor * expert_ids = dst->src[3]; // [n_used, n_tokens] const ggml_tensor * weights = dst->src[4]; // [n_used, n_tokens] + const ggml_tensor * residual = dst->src[5]; // optional [n_embd, n_tokens] const int n_embd = (int) input->ne[0]; const int n_tokens = (int) input->ne[1]; @@ -477,6 +532,9 @@ static void ggml_cuda_op_ds4_moe_owner( GGML_ASSERT(expert_ids->ne[1] == n_tokens); GGML_ASSERT(weights->ne[0] == n_used && weights->ne[1] == n_tokens); GGML_ASSERT(dst->ne[0] == n_embd && dst->ne[1] == n_tokens); + GGML_ASSERT(!residual || + (residual->type == GGML_TYPE_F32 && + residual->ne[0] == n_embd && residual->ne[1] == n_tokens)); // The checkpoint concatenates gate rows followed by up rows inside each // expert. Keep the original expert stride while viewing each half. @@ -524,10 +582,13 @@ static void ggml_cuda_op_ds4_moe_owner( laguna_moe_combine_kernel<<>>( (const char *) experts.data, (const char *) weights->data, + residual ? (const char *) residual->data : nullptr, (char *) dst->data, n_embd, n_used, n_tokens, experts.nb[0], experts.nb[1], experts.nb[2], weights->nb[0], weights->nb[1], + residual ? residual->nb[0] : 0, + residual ? residual->nb[1] : 0, dst->nb[0], dst->nb[1], down_scale); } @@ -541,6 +602,7 @@ static void ggml_cuda_op_ds4_moe_owner_split( const ggml_tensor * down_w = dst->src[3]; // [n_ff, n_embd, n_expert] const ggml_tensor * expert_ids = dst->src[4]; // [n_used, n_tokens] const ggml_tensor * weights = dst->src[5]; // [n_used, n_tokens] + const ggml_tensor * residual = dst->src[6]; // optional [n_embd, n_tokens] const int n_embd = (int) input->ne[0]; const int n_tokens = (int) input->ne[1]; @@ -557,6 +619,9 @@ static void ggml_cuda_op_ds4_moe_owner_split( GGML_ASSERT(expert_ids->ne[1] == n_tokens); GGML_ASSERT(weights->ne[0] == n_used && weights->ne[1] == n_tokens); GGML_ASSERT(dst->ne[0] == n_embd && dst->ne[1] == n_tokens); + GGML_ASSERT(!residual || + (residual->type == GGML_TYPE_F32 && + residual->ne[0] == n_embd && residual->ne[1] == n_tokens)); // MUL_MAT_ID consumes token columns in dimension 2. This descriptor is // stack-local, so it deliberately does not participate in q8 memoization. @@ -597,16 +662,49 @@ static void ggml_cuda_op_ds4_moe_owner_split( laguna_moe_combine_kernel<<>>( (const char *) experts.data, (const char *) weights->data, + residual ? (const char *) residual->data : nullptr, (char *) dst->data, n_embd, n_used, n_tokens, experts.nb[0], experts.nb[1], experts.nb[2], weights->nb[0], weights->nb[1], + residual ? residual->nb[0] : 0, + residual ? residual->nb[1] : 0, dst->nb[0], dst->nb[1], down_scale); } void ggml_cuda_op_moe_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const int mode = ggml_get_op_params_i32(dst, 0); + if (mode == GGML_MOE_FUSED_BALANCED_OWNER_IDS) { + const ggml_tensor * global_ids = dst->src[0]; + const ggml_tensor * weights = dst->src[1]; + const ggml_tensor * local_lut = dst->src[2]; + const ggml_tensor * candidate_lut = dst->src[3]; + GGML_ASSERT(global_ids && global_ids->type == GGML_TYPE_I32); + GGML_ASSERT(weights && weights->type == GGML_TYPE_F32); + GGML_ASSERT(local_lut && local_lut->type == GGML_TYPE_I32); + GGML_ASSERT(candidate_lut && candidate_lut->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_I32); + GGML_ASSERT(ggml_is_contiguous(global_ids)); + GGML_ASSERT(ggml_is_contiguous(weights)); + GGML_ASSERT(ggml_is_contiguous(local_lut)); + GGML_ASSERT(ggml_is_contiguous(candidate_lut)); + GGML_ASSERT(ggml_is_contiguous(dst)); + + const int n_routes = (int) global_ids->ne[0]; + const int n_tokens = (int) global_ids->ne[1]; + const int n_expert = (int) ggml_nelements(local_lut); + const int main_slots_x4 = ggml_get_op_params_i32(dst, 1); + const bool main_owner = ggml_get_op_params_i32(dst, 2) != 0; + ds4_balanced_owner_ids_kernel<<<1, 1, 0, ctx.stream()>>>( + (const int32_t *) global_ids->data, + (const float *) weights->data, + (const int32_t *) local_lut->data, + (const float *) candidate_lut->data, + (int32_t *) dst->data, + n_routes, n_tokens, n_expert, main_slots_x4, main_owner); + return; + } if (mode == GGML_MOE_FUSED_ALIGN_IDS) { const ggml_tensor * ids = dst->src[0]; GGML_ASSERT(ids && ids->type == GGML_TYPE_I32); @@ -678,10 +776,12 @@ void ggml_cuda_op_moe_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst) laguna_moe_combine_kernel<<>>( (const char *) experts->data, (const char *) weights->data, + nullptr, (char *) dst->data, n_embd, n_used, n_tokens, experts->nb[0], experts->nb[1], experts->nb[2], weights->nb[0], weights->nb[1], + 0, 0, dst->nb[0], dst->nb[1], 1.0f); return; diff --git a/server/deps/llama.cpp/ggml/src/ggml.c b/server/deps/llama.cpp/ggml/src/ggml.c index 2db6c2bf7..b78b5077c 100644 --- a/server/deps/llama.cpp/ggml/src/ggml.c +++ b/server/deps/llama.cpp/ggml/src/ggml.c @@ -8206,6 +8206,7 @@ struct ggml_tensor * ggml_ds4_moe_owner( struct ggml_tensor * down_w, struct ggml_tensor * expert_ids, struct ggml_tensor * expert_weights, + struct ggml_tensor * owner_residual, int64_t ff_dim, float swiglu_clamp, float down_scale) { @@ -8220,6 +8221,10 @@ struct ggml_tensor * ggml_ds4_moe_owner( GGML_ASSERT(down_w->ne[0] == ff_dim); GGML_ASSERT(down_w->ne[1] == input->ne[0]); GGML_ASSERT(gate_up_w->ne[2] == down_w->ne[2]); + GGML_ASSERT(!owner_residual || + (owner_residual->type == GGML_TYPE_F32 && + owner_residual->ne[0] == input->ne[0] && + owner_residual->ne[1] == input->ne[1])); const int64_t ne[4] = { input->ne[0], input->ne[1], 1, 1 }; struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 2, ne); @@ -8230,6 +8235,7 @@ struct ggml_tensor * ggml_ds4_moe_owner( result->src[2] = down_w; result->src[3] = expert_ids; result->src[4] = expert_weights; + result->src[5] = owner_residual; ggml_set_op_params_i32(result, 0, GGML_MOE_FUSED_OWNER); ggml_set_op_params_i32(result, 1, (int32_t) ff_dim); @@ -8247,6 +8253,7 @@ struct ggml_tensor * ggml_ds4_moe_owner_split( struct ggml_tensor * down_w, struct ggml_tensor * expert_ids, struct ggml_tensor * expert_weights, + struct ggml_tensor * owner_residual, int64_t ff_dim, float swiglu_clamp, float gate_scale, @@ -8265,6 +8272,10 @@ struct ggml_tensor * ggml_ds4_moe_owner_split( GGML_ASSERT(down_w->ne[1] == input->ne[0]); GGML_ASSERT(gate_w->ne[2] == up_w->ne[2]); GGML_ASSERT(gate_w->ne[2] == down_w->ne[2]); + GGML_ASSERT(!owner_residual || + (owner_residual->type == GGML_TYPE_F32 && + owner_residual->ne[0] == input->ne[0] && + owner_residual->ne[1] == input->ne[1])); const int64_t ne[4] = { input->ne[0], input->ne[1], 1, 1 }; struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 2, ne); @@ -8276,6 +8287,7 @@ struct ggml_tensor * ggml_ds4_moe_owner_split( result->src[3] = down_w; result->src[4] = expert_ids; result->src[5] = expert_weights; + result->src[6] = owner_residual; ggml_set_op_params_i32(result, 0, GGML_MOE_FUSED_OWNER_SPLIT); ggml_set_op_params_i32(result, 1, (int32_t) ff_dim); @@ -8301,6 +8313,40 @@ struct ggml_tensor * ggml_ds4_moe_align_ids( return result; } +struct ggml_tensor * ggml_ds4_moe_balanced_owner_ids( + struct ggml_context * ctx, + struct ggml_tensor * global_ids, + struct ggml_tensor * router_weights, + struct ggml_tensor * local_id_lut, + struct ggml_tensor * main_candidate_lut, + int main_slots_x4, + bool main_owner) { + GGML_ASSERT(global_ids->type == GGML_TYPE_I32); + GGML_ASSERT(router_weights->type == GGML_TYPE_F32); + GGML_ASSERT(local_id_lut->type == GGML_TYPE_I32); + GGML_ASSERT(main_candidate_lut->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_are_same_shape(global_ids, router_weights)); + GGML_ASSERT(ggml_nelements(local_id_lut) == + ggml_nelements(main_candidate_lut)); + GGML_ASSERT(ggml_is_contiguous(global_ids)); + GGML_ASSERT(ggml_is_contiguous(router_weights)); + GGML_ASSERT(ggml_is_contiguous(local_id_lut)); + GGML_ASSERT(ggml_is_contiguous(main_candidate_lut)); + GGML_ASSERT(main_slots_x4 > 0 && + main_slots_x4 <= 4 * global_ids->ne[0]); + + struct ggml_tensor * result = ggml_dup_tensor(ctx, global_ids); + result->op = GGML_OP_MOE_FUSED; + result->src[0] = global_ids; + result->src[1] = router_weights; + result->src[2] = local_id_lut; + result->src[3] = main_candidate_lut; + ggml_set_op_params_i32(result, 0, GGML_MOE_FUSED_BALANCED_OWNER_IDS); + ggml_set_op_params_i32(result, 1, main_slots_x4); + ggml_set_op_params_i32(result, 2, main_owner ? 1 : 0); + return result; +} + struct ggml_tensor * ggml_ds4_deferred_peer_copy( struct ggml_context * ctx, struct ggml_tensor * src) { diff --git a/server/docs/DS4_R9700_STRIX_PROFILE.md b/server/docs/DS4_R9700_STRIX_PROFILE.md new file mode 100644 index 000000000..c291a2457 --- /dev/null +++ b/server/docs/DS4_R9700_STRIX_PROFILE.md @@ -0,0 +1,118 @@ +# DS4 q=5 profile for R9700 + Strix Halo + +This profile reproduces the qualified single-process heterogeneous decode on +an R9700 (`gfx1201`) plus Strix Halo (`gfx1151`) running ROCm 7.2.4. It is a +hardware profile, not a portable default for unrelated GPU pairs. + +## Checked-in inputs + +The profile keeps every small input needed for the qualified placement in the +repository: + +- `config/ds4/r9700-strix/prefill-routing.csv`: resident-expert profile; +- `config/ds4/r9700-strix/decode-routing.csv`: decode placement profile; +- `config/hipblaslt/r9700-strix-rocm-7.2.4.txt`: selected hipBLASLt solutions; +- `scripts/qualify_ds4_q5_r9700_strix.sh`: exact launch and measurement policy. + +The wrapper verifies all three file checksums before loading the model. Custom +profile paths remain possible through `HOTNESS_CSV`, `DECODE_HOTNESS_CSV`, and +`HIPBLASLT_TUNING_OVERRIDE_FILE`, but they are a new qualification rather than +the checked-in result. + +## Build + +Compile one binary for both GPU architectures: + +```bash +cmake -S server -B server/build-hip-dual \ + -DCMAKE_BUILD_TYPE=Release \ + -DDFLASH27B_GPU_BACKEND=hip \ + -DDFLASH27B_HIP_ARCHITECTURES='gfx1151;gfx1201' \ + -DDFLASH27B_TESTS=ON + +cmake --build server/build-hip-dual -j \ + --target dflash_server test_tokenizer_harness test_deepseek4_unit +``` + +Before the model run, execute the focused unit suite: + +```bash +server/build-hip-dual/test_deepseek4_unit +``` + +## Run the qualified profile + +Only the two model artifacts and build directory are external inputs: + +```bash +TARGET_MODEL=/path/to/DeepSeek-V4-Flash-ROCMFP2-STRIX.gguf \ +DRAFT_MODEL=/path/to/DeepSeek-V4-Flash-DSpark-draft-Q4RMFP4-denseF16.gguf \ +BUILD_DIR="$PWD/server/build-hip-dual" \ +server/scripts/qualify_ds4_q5_r9700_strix.sh +``` + +The default protocol is one exact 2,048-token prompt, 128 generated tokens, +two warmups, and seven measured requests. Every request must produce 128 +tokens with response SHA-256 +`0f785a7ffa406498aafb14553966eaed0f52220fed0f7cc016b66921d104d194`. +The runner fails on a different length or hash. + +Each run directory records the source commit, server checksum, tuning-table +path and checksum, complete environment, model metadata, individual request +records, client summary, authoritative model-side decode summary, server log, +and ROCm/VRAM state. Set `OUT_ROOT` and `RUN_ID` to choose its location. + +## Measured result + +The final two-warmup/seven-run qualification used source +`ee160092ce106a017d50a11cf8110e3b21e3cc46` and server SHA-256 +`2bd2282c5409f5d6573fbf0763c00bfcbdbe50c547d64231c8213eeee8ec5119`. +All seven measured requests returned 128 tokens with the required response +hash. + +- authoritative server decode: **73.7 tok/s median**, 73.5-73.8 range; +- historical client diagnostic: 83.403 tok/s median, 83.122-83.509 range; +- speculative acceptance: 1.00 median. + +An earlier experiment reported 89.876 tok/s in the client diagnostic, but its +model-side median was 73.2 tok/s. That client formula starts timing at the +first non-empty streamed text event while counting every completion token. A +slow first rejected speculative step therefore fell outside its time window +and inflated the result. The repaired capture path accepts the first block and +is slightly faster model-side, even though that legacy client number is lower. +Use `server-decode-summary.json` for engine performance claims. + +## Qualified placement + +The wrapper fixes the settings that produced the measured profile: + +- critical-path expert placement with a main/peer rate ratio of `4.4`; +- a 14,350 MiB main-GPU expert budget; +- dynamic route balance with 3 main slots and 13 main slots at q=5; +- owner-local residual fusion; +- q=5 verification with the platform-selected ROCmFP4 x4+1 kernel; +- the checked-in hipBLASLt solutions for both GPU architectures. + +The experimental attention-head split and shared-stage width split are not in +this profile: both were slower in full exact-output runs. + +## Wide heterogeneous capture repair + +A rebase onto the newer prefill safety work exposed a reproducibility bug. The +2K heterogeneous prompt was split into 1,920 and 128-token forwards solely to +capture the final DSpark feature window. Sparse heterogeneous prefill already +returns a feature row for every requested token, so the split was unnecessary. +It changed the approximate sparse-prefill state, made the target emit EOS, and +prevented the benchmark from measuring a full completion. + +The runtime now keeps one wide heterogeneous batch and retains the requested +feature rows afterward. Snapshot boundaries still split when required. Unit +tests cover supported modes and size limits, and the model-backed qualifier +checks the full completion and exact response hash. + +## Portability boundary + +Do not silently reuse the hipBLASLt table with another ROCm release or GPU +architecture. Retune and requalify there. The routing profiles are also tied +to this model, quantization, workload, and expert layout. The general +`qualify_ds4_q5_amd.sh` runner remains the starting point for a new pair. diff --git a/server/docs/ENVIRONMENT.md b/server/docs/ENVIRONMENT.md index a877c3812..3eee06200 100644 --- a/server/docs/ENVIRONMENT.md +++ b/server/docs/ENVIRONMENT.md @@ -45,6 +45,10 @@ consolidation of this list into CLI flags is tracked as follow-up work. | `GGML_DS4_FA_SERIAL_INDEX_SCAN` | unset | DEBUG/A-B: restore the serial indexed-attention mask scan instead of the long-context HIP parallel scan. | | `DFLASH_MOE_PREFILL_PERSISTENT_OWNER_ALLOC` | 1 for qualified long heterogeneous prefill | KILL SWITCH: =0 restores per-layer route/owner scratch allocation. | | `DFLASH_MOE_TP_*` / `DFLASH_MOE_HYBRID_PREFILL_EAGER` | unset | BURN-IN: model-neutral names for common heterogeneous-MoE scheduling and kernel policy. Existing `DFLASH_DS4_*` names remain compatibility aliases. | +| `DFLASH_MOE_TP_SHARED_FFN_PEER_FRACTION` | 0 | OPT-IN: aligned shared/dense FFN stage partition (`0`, `(0,1]`, or `auto`). Keep disabled unless exact model-backed qualification wins; see `HETEROGENEOUS_STAGE_PLANNER.md`. | +| `DFLASH_MOE_TP_MAIN_RATE` / `DFLASH_MOE_TP_PEER_RATE` | 1 / 1 | Calibrated owner rates used by automatic stage partitioning. | +| `DFLASH_MOE_TP_MAIN_FIXED_WORK` / `DFLASH_MOE_TP_PEER_FIXED_WORK` | 0 / 0 | Concurrent work already assigned to each owner, in the planner's stage-work units. | +| `DFLASH_MOE_TP_FUSED_OWNER_RESIDUAL` | unset | OPT-IN: merge an owner-local shared/dense contribution into a supported routed-owner reduction without another cross-device value. | | `DFLASH_MMID_TELEMETRY` | unset | DEBUG: report MUL_MAT_ID dispatch, MMVQ variant, and per-node graph compatibility. | | `DFLASH_KVFLASH` | unset | Prefer the CLI: `--kvflash` (token count or `auto`). | | `DFLASH_PREFIX_CACHE_SLOTS` | 32 | Container-entrypoint equivalent of `--prefix-cache-slots`; not read directly by the native binary. | diff --git a/server/docs/HETEROGENEOUS_STAGE_PLANNER.md b/server/docs/HETEROGENEOUS_STAGE_PLANNER.md new file mode 100644 index 000000000..84d139cc3 --- /dev/null +++ b/server/docs/HETEROGENEOUS_STAGE_PLANNER.md @@ -0,0 +1,146 @@ +# Generic Heterogeneous Stage Planner + +This note records the reusable two-owner stage partition added to the common +MoE runtime, its correctness contract, and the measured decisions that should +not be rediscovered on each model or hardware pair. + +## Scope + +`heterogeneous_stage_planner.{h,cpp}` is independent of ggml, model family, +quantization, and device vendor. A caller supplies: + +- a splittable work width; +- the alignment required by the tensor format or kernel; +- either an explicit peer fraction or calibrated owner rates; +- optional work already assigned to each owner. + +The output is a complete, aligned partition. Fractions `0` and `1` represent +whole-stage ownership. Interior fractions keep at least one aligned unit on +both owners. Invalid or unsplittable inputs conservatively remain on the main +owner. + +The balanced plan minimizes the predicted critical path: + +```text +max((main_fixed_work + main_width) / main_rate, + (peer_fixed_work + peer_width) / peer_rate) +``` + +Work and rate units are deliberately abstract. An adapter may use bytes and +bytes/us, FLOPs and FLOPs/us, or calibrated route-equivalents, provided every +input uses the same unit. + +## Common MoE integration + +The first consumer is the shared/dense SwiGLU FFN in +`moe_hybrid_storage.cpp` and `moe_hybrid_ffn_eval.cpp`: + +1. Storage validates the gate/up/down tensor shapes, contiguity, and block + alignment. +2. It copies only the selected peer rows into peer-owned weight storage. +3. Both owners compute their full gate/up/activation/down chain concurrently. +4. Each owner reduces its local routed and shared contributions before the + heterogeneous boundary. +5. One deferred peer copy and one final join remain per layer. + +The implementation supports an aligned width split and both whole-stage +endpoints. It deliberately refuses a shared FFN with an input-dependent output +gate: applying that gate independently to partial down projections would alter +the floating-point graph and is not bit-identical. + +`DFLASH_MOE_TP_FUSED_OWNER_RESIDUAL=1` lets a supported coarse routed-owner +kernel consume its owner-local shared/dense result in the existing final +reduction. Unsupported backends keep the ordinary add, so the common scheduler +does not require a model-specific kernel. + +## Policy controls + +These are burn-in controls, not a permanent public configuration API: + +| Variable | Meaning | +|---|---| +| `DFLASH_MOE_TP_SHARED_FFN_PEER_FRACTION` | `0` disables, `(0,1)` requests an aligned split, `1` assigns the stage to peer, and `auto` uses the rate model. | +| `DFLASH_MOE_TP_MAIN_RATE` | Calibrated main-owner rate for `auto`. | +| `DFLASH_MOE_TP_PEER_RATE` | Calibrated peer-owner rate for `auto`. | +| `DFLASH_MOE_TP_MAIN_TO_PEER_RATE` | Shorthand ratio when separate rates are unavailable. | +| `DFLASH_MOE_TP_MAIN_FIXED_WORK` | Work already assigned to main when the stage begins. | +| `DFLASH_MOE_TP_PEER_FIXED_WORK` | Work already assigned to peer when the stage begins. | +| `DFLASH_MOE_TP_FUSED_OWNER_RESIDUAL` | Fuse an owner-local dense/shared result into a supported routed-owner reduction. | + +The legacy `DFLASH_DS4_*` spellings remain compatibility aliases. New model +adapters should use the common names or populate the equivalent +`MoeHybridConfig` fields. + +## Qualification result + +All entries below used the same q=5, 2K-context, 128-token exact-output run. +The expected response SHA-256 was +`0f785a7ffa406498aafb14553966eaed0f52220fed0f7cc016b66921d104d194`. + +| Plan | Legacy client tok/s | Decision | +|---|---:|---| +| Established route balance, shared stage on main | 88.602 | Qualified default | +| Owner-local residual fusion, no shared split | 88.637 | Exact final binary; neutral, retained as opt-in primitive | +| 25% shared-width peer shard, materialized owner adds | 86.750 | Reject | +| 12.5% shared-width peer shard, materialized owner adds | 86.557 | Reject | +| 12.5% shared-width peer shard, four-output join | 85.159 | Reject; implementation removed | +| 12.5% shared-width peer shard, single-join owner fusion | 86.636 | Reject | +| Repeated-expert route-slot alignment | 85.117 | Reject; remains disabled | +| Complete shared stage on peer, all routed work on main | 84.022 | Reject; unstable in this profile | + +These historical client values divide all completion tokens by the interval +from the first non-empty streamed text event to `[DONE]`. They are useful as a +transport diagnostic, but speculative block boundaries can move work before +that first event and inflate the value. Use the model-side DSpark rate from +`server-decode-summary.json` for engine comparisons. + +The narrow shared shards lose more kernel efficiency than their extra overlap +can recover. The complete peer stage also extends verification latency. These +results mean width sharding must remain disabled by default; the planner is a +portable mechanism to qualify other shapes and hardware, not evidence that one +fraction is universally beneficial. + +## Current critical path + +The qualified route-owner profile measured main-owner overlap at 82.63%, peer +overlap at 100%, launch skew at 8.41 us, and a main tail of about 58.35 us per +MoE layer (roughly 2.5 ms per speculative step). Verification is about 54.7 ms +of a 65.3 ms speculative step. Removing the measured owner tail alone is not +enough to reach 100 tok/s; the next optimization must reduce full-width expert +kernel time or overlap another independent stage, not add synchronization or +small matrix shards. + +## Reproduction + +For the qualified R9700 + Strix Halo pair, use +`scripts/qualify_ds4_q5_r9700_strix.sh`. It supplies the checked-in routing +profiles, verifies their checksums and the ROCm 7.2.4 hipBLASLt table, and then +calls the general runner. Only `TARGET_MODEL`, `DRAFT_MODEL`, and `BUILD_DIR` +are required. See `DS4_R9700_STRIX_PROFILE.md` for the exact build and launch. + +For a new hardware pair, use `scripts/qualify_ds4_q5_amd.sh` with newly +qualified routing inputs. The runner records the source commit, +binary checksum, all policy controls, exact response hashes, individual runs, +and summary statistics. Relevant switches are: + +```bash +SHARED_FFN_PEER_FRACTION=0 +FUSED_OWNER_RESIDUAL=1 +ALIGN_SHARED_IDS=0 +EXPERT_TOP_K=4 +DYNAMIC_ROUTE_BALANCE=1 +DYNAMIC_MAIN_SLOTS_X4=13 +``` + +Do not promote a candidate from a microbenchmark alone. It must pass unit/GPU +oracles, the exact response hash, warmups, and at least three measured model +runs. Keep rejected candidates disabled and record their result here. + +The literal rebased profile at `ee160092ce106a017d50a11cf8110e3b21e3cc46` +passed all seven measured exact-output requests. Its authoritative server +decode median was **73.7 tok/s** (73.5-73.8); the legacy client field was +83.403 tok/s. The earlier 89.876 client result had a 73.2 tok/s server median: +it excluded a slow first rejected speculative step from its time window while +still counting all 128 tokens. The newer capture path has full first-block +acceptance and slightly higher actual server throughput, despite the lower +legacy client number. diff --git a/server/docs/moe_hybrid.md b/server/docs/moe_hybrid.md index 5fc7bc131..19da76e59 100644 --- a/server/docs/moe_hybrid.md +++ b/server/docs/moe_hybrid.md @@ -24,6 +24,7 @@ The same mechanism supports GPU+CPU offload on a memory-constrained card and GPU | `moe_hybrid_swap_manager.{h,cpp}` | Runtime expert promotion/demotion between requests | | `moe_hybrid_storage.{h,cpp}` | Compact owner-local buffers for split expert tensors | | `moe_hybrid_ffn_eval.{h,cpp}` | Concurrent owner execution and partial-result joining | +| `heterogeneous_stage_planner.{h,cpp}` | Model/vendor-neutral aligned stage partitioning and rate balancing | | `moe_expert_compute.{h,cpp}` | Backend-neutral selected-expert compute interface | | `moe_expert_compute_ipc.cpp` | Second-GPU process transport and architecture adapter registry | @@ -167,6 +168,10 @@ Adding a compatible MoE architecture requires a thin adapter, not another schedu Hardware policy remains outside this contract: placement budgets, which GPU is primary, peer-copy mode, verification width, and kernel qualification are Lucebox profile choices. A model adapter must not duplicate owner scheduling, storage, peer transport, or join logic. +The common shared/dense-stage partition, its safety constraints, and measured +qualification history are documented in +[`HETEROGENEOUS_STAGE_PLANNER.md`](HETEROGENEOUS_STAGE_PLANNER.md). + Common execution switches use the `DFLASH_MOE_*` namespace. The corresponding `DFLASH_DS4_*` names remain accepted as compatibility aliases for existing DeepSeek V4 profiles. Currently integrated with: diff --git a/server/scripts/ds4_publication_decode_client.py b/server/scripts/ds4_publication_decode_client.py index 76097d3d3..0cea55d77 100755 --- a/server/scripts/ds4_publication_decode_client.py +++ b/server/scripts/ds4_publication_decode_client.py @@ -147,6 +147,12 @@ def stream_request( "wall_s": round(wall_s, 6), "ttft_s": round(ttft_s, 6) if ttft_s is not None else None, "client_decode_s": round(decode_s, 6) if decode_s is not None else None, + # This transport diagnostic intentionally retains its historical + # formula. It is not model throughput: speculative block boundaries + # can move work before the first non-empty streamed text event while + # the numerator still includes every completion token. + "client_decode_rate_window": "first_nonempty_text_event_to_done", + "client_decode_rate_numerator_tokens": completion_tokens, "client_decode_tok_s": ( round(client_decode_tps, 3) if client_decode_tps is not None else None ), diff --git a/server/scripts/qualify_ds4_q5_amd.sh b/server/scripts/qualify_ds4_q5_amd.sh index d86d01b1e..7be27481d 100755 --- a/server/scripts/qualify_ds4_q5_amd.sh +++ b/server/scripts/qualify_ds4_q5_amd.sh @@ -13,7 +13,9 @@ TOKENIZER_HARNESS="${TOKENIZER_HARNESS:-$BUILD_DIR/test_tokenizer_harness}" TARGET_MODEL="${TARGET_MODEL:?set TARGET_MODEL to the target GGUF path}" DRAFT_MODEL="${DRAFT_MODEL:?set DRAFT_MODEL to the DSpark draft GGUF path}" HOTNESS_CSV="${HOTNESS_CSV:?set HOTNESS_CSV to the expert hotness CSV path}" +DECODE_HOTNESS_CSV="${DECODE_HOTNESS_CSV:-}" CONTEXT_CLIENT="${CONTEXT_CLIENT:-$SCRIPT_DIR/ds4_context_sweep.py}" +SERVER_DECODE_SUMMARIZER="${SERVER_DECODE_SUMMARIZER:-$SCRIPT_DIR/summarize_ds4_server_decode.py}" EXPECTED_SHA256="${EXPECTED_SHA256:-0f785a7ffa406498aafb14553966eaed0f52220fed0f7cc016b66921d104d194}" PORT="${PORT:-18109}" MAX_CTX="${MAX_CTX:-18432}" @@ -25,6 +27,7 @@ DIRECT_INDEXER_TOPK="${DIRECT_INDEXER_TOPK:-1}" BLOCK_RADIX_TOPK="${BLOCK_RADIX_TOPK:-1}" PACK_Q4_INDEXER="${PACK_Q4_INDEXER:-0}" Q5_VERIFY="${Q5_VERIFY:-1}" +Q6_VERIFY="${Q6_VERIFY:-0}" FP4_Q5_X4_PLUS1="${FP4_Q5_X4_PLUS1:-auto}" CRITICAL_PATH_PLACEMENT="${CRITICAL_PATH_PLACEMENT:-0}" MAIN_TO_PEER_RATE="${MAIN_TO_PEER_RATE:-3.4}" @@ -35,21 +38,45 @@ RUNS="${RUNS:-3}" MAX_TOKENS="${MAX_TOKENS:-128}" TARGETS="${TARGETS:-2048 4096 8192 16384 2048}" VRAM_MONITOR_SECONDS="${VRAM_MONITOR_SECONDS:-2}" +SET_PERF_LEVEL="${SET_PERF_LEVEL:-1}" HASH_MODELS="${HASH_MODELS:-0}" CUDA_GRAPH_STATS_EVERY="${CUDA_GRAPH_STATS_EVERY:-200}" CUDA_DISABLE_GRAPHS_DEVICES="${CUDA_DISABLE_GRAPHS_DEVICES:-}" -RUN_ID="${RUN_ID:-ds4-q5-fr${FORCE_GRAPH_REPLAY}-direct${DIRECT_INDEXER_TOPK}-radix${BLOCK_RADIX_TOPK}-x4p1${FP4_Q5_X4_PLUS1}-cp${CRITICAL_PATH_PLACEMENT}-r${MAIN_TO_PEER_RATE}-$(date -u +%Y%m%dT%H%M%SZ)}" +ROCBLAS_USE_HIPBLASLT="${ROCBLAS_USE_HIPBLASLT:-}" +HIPBLASLT_LOG_MASK="${HIPBLASLT_LOG_MASK:-}" +HIPBLASLT_TUNING_FILE="${HIPBLASLT_TUNING_FILE:-}" +HIPBLASLT_TUNING_OVERRIDE_FILE="${HIPBLASLT_TUNING_OVERRIDE_FILE:-}" +DYNAMIC_ROUTE_BALANCE="${DYNAMIC_ROUTE_BALANCE:-0}" +DYNAMIC_MAIN_SLOTS="${DYNAMIC_MAIN_SLOTS:-3}" +DYNAMIC_MAIN_SLOTS_X2="${DYNAMIC_MAIN_SLOTS_X2:-}" +DYNAMIC_MAIN_SLOTS_X4="${DYNAMIC_MAIN_SLOTS_X4:-}" +SHARED_FFN_PEER_FRACTION="${SHARED_FFN_PEER_FRACTION:-0}" +FUSED_OWNER_RESIDUAL="${FUSED_OWNER_RESIDUAL:-0}" +ALIGN_SHARED_IDS="${ALIGN_SHARED_IDS:-0}" +EXPERT_TOP_K="${EXPERT_TOP_K:-4}" +VERIFY_WIDTH=$((4 + Q5_VERIFY + 2 * Q6_VERIFY)) +RUN_ID="${RUN_ID:-ds4-q${VERIFY_WIDTH}-fr${FORCE_GRAPH_REPLAY}-direct${DIRECT_INDEXER_TOPK}-radix${BLOCK_RADIX_TOPK}-x4p1${FP4_Q5_X4_PLUS1}-cp${CRITICAL_PATH_PLACEMENT}-r${MAIN_TO_PEER_RATE}-sf${SHARED_FFN_PEER_FRACTION}-or${FUSED_OWNER_RESIDUAL}-ai${ALIGN_SHARED_IDS}-$(date -u +%Y%m%dT%H%M%SZ)}" OUT_ROOT="${OUT_ROOT:-$CHECKOUT/results/ds4_q5_context_qualification}" OUT_DIR="$OUT_ROOT/$RUN_ID" SERVER_LOG="$OUT_DIR/server.log" for required in "$SERVER_BIN" "$TOKENIZER_HARNESS" "$TARGET_MODEL" \ - "$DRAFT_MODEL" "$HOTNESS_CSV" "$CONTEXT_CLIENT"; do + "$DRAFT_MODEL" "$HOTNESS_CSV" "$CONTEXT_CLIENT" \ + "$SERVER_DECODE_SUMMARIZER"; do if [[ ! -e "$required" ]]; then echo "missing required path: $required" >&2 exit 2 fi done +if [[ -n "$DECODE_HOTNESS_CSV" && ! -e "$DECODE_HOTNESS_CSV" ]]; then + echo "missing decode hotness path: $DECODE_HOTNESS_CSV" >&2 + exit 2 +fi +if [[ -n "$HIPBLASLT_TUNING_OVERRIDE_FILE" && + ! -f "$HIPBLASLT_TUNING_OVERRIDE_FILE" ]]; then + echo "missing hipBLASLt tuning override: $HIPBLASLT_TUNING_OVERRIDE_FILE" >&2 + exit 2 +fi case "$FORCE_GRAPH_REPLAY:$SERIAL_INDEX_SCAN" in 0:0|0:1|1:0|1:1) ;; @@ -71,6 +98,55 @@ case "$Q5_VERIFY" in 0|1) ;; *) echo "Q5_VERIFY must be 0 or 1" >&2; exit 2 ;; esac +case "$Q6_VERIFY" in + 0|1) ;; + *) echo "Q6_VERIFY must be 0 or 1" >&2; exit 2 ;; +esac +if ((Q5_VERIFY + Q6_VERIFY > 1)); then + echo "Q5_VERIFY and Q6_VERIFY are mutually exclusive" >&2 + exit 2 +fi +case "$DYNAMIC_ROUTE_BALANCE" in + 0|1) ;; + *) echo "DYNAMIC_ROUTE_BALANCE must be 0 or 1" >&2; exit 2 ;; +esac +case "$FUSED_OWNER_RESIDUAL" in + 0|1) ;; + *) echo "FUSED_OWNER_RESIDUAL must be 0 or 1" >&2; exit 2 ;; +esac +case "$ALIGN_SHARED_IDS" in + 0|1) ;; + *) echo "ALIGN_SHARED_IDS must be 0 or 1" >&2; exit 2 ;; +esac +if [[ ! "$EXPERT_TOP_K" =~ ^[1-9][0-9]*$ ]] || ((EXPERT_TOP_K > 16)); then + echo "EXPERT_TOP_K must be an integer from 1 through 16" >&2 + exit 2 +fi +if [[ ! "$DYNAMIC_MAIN_SLOTS" =~ ^[1-9][0-9]*$ ]] || + ((DYNAMIC_MAIN_SLOTS > EXPERT_TOP_K)); then + echo "DYNAMIC_MAIN_SLOTS must be an integer from 1 through EXPERT_TOP_K ($EXPERT_TOP_K)" >&2 + exit 2 +fi +if [[ -n "$DYNAMIC_MAIN_SLOTS_X2" ]] && + { [[ ! "$DYNAMIC_MAIN_SLOTS_X2" =~ ^[1-9][0-9]*$ ]] || + ((DYNAMIC_MAIN_SLOTS_X2 < 2 || DYNAMIC_MAIN_SLOTS_X2 > 2 * EXPERT_TOP_K)); }; then + echo "DYNAMIC_MAIN_SLOTS_X2 must be empty or an integer from 2 through $((2 * EXPERT_TOP_K))" >&2 + exit 2 +fi +if [[ -n "$DYNAMIC_MAIN_SLOTS_X4" ]] && + { [[ ! "$DYNAMIC_MAIN_SLOTS_X4" =~ ^[1-9][0-9]*$ ]] || + ((DYNAMIC_MAIN_SLOTS_X4 < 4 || DYNAMIC_MAIN_SLOTS_X4 > 4 * EXPERT_TOP_K)); }; then + echo "DYNAMIC_MAIN_SLOTS_X4 must be empty or an integer from 4 through $((4 * EXPERT_TOP_K))" >&2 + exit 2 +fi +if [[ "$SHARED_FFN_PEER_FRACTION" != 0 && + "$SHARED_FFN_PEER_FRACTION" != auto ]] && + { [[ ! "$SHARED_FFN_PEER_FRACTION" =~ ^(0[.][0-9]+|1([.]0+)?)$ ]] || + ! awk -v value="$SHARED_FFN_PEER_FRACTION" \ + 'BEGIN { exit !(value > 0 && value <= 1) }'; }; then + echo "SHARED_FFN_PEER_FRACTION must be 0, auto, or a value in (0,1]" >&2 + exit 2 +fi case "$FP4_Q5_X4_PLUS1" in auto|0|1) ;; *) echo "FP4_Q5_X4_PLUS1 must be auto, 0, or 1" >&2; exit 2 ;; @@ -100,6 +176,10 @@ case "$HASH_MODELS" in 0|1) ;; *) echo "HASH_MODELS must be 0 or 1" >&2; exit 2 ;; esac +case "$SET_PERF_LEVEL" in + 0|1) ;; + *) echo "SET_PERF_LEVEL must be 0 or 1" >&2; exit 2 ;; +esac if pgrep -f "dflash_server .*--port ${PORT}([[:space:]]|$)" >/dev/null; then echo "benchmark port $PORT is already owned by another dflash_server" >&2 @@ -122,8 +202,10 @@ cleanup() { } trap cleanup EXIT -rocm-smi -d 0 --setperflevel auto >/dev/null 2>&1 || true -rocm-smi -d 1 --setperflevel high >/dev/null 2>&1 || true +if [[ "$SET_PERF_LEVEL" == 1 ]]; then + rocm-smi -d 0 --setperflevel auto >/dev/null 2>&1 || true + rocm-smi -d 1 --setperflevel high >/dev/null 2>&1 || true +fi printf '0\n' >/tmp/ds4_awidth rm -f /tmp/ds4_spec_q @@ -177,7 +259,7 @@ server_env=( "DFLASH_DS4_PINNED_ROLLBACK=1" "DFLASH_DS4_GPU_ARGMAX_VERIFY=1" "DFLASH_DS4_SPEC=1" - "DFLASH_DS4_SPEC_Q=$((4 + Q5_VERIFY))" + "DFLASH_DS4_SPEC_Q=$VERIFY_WIDTH" "DFLASH_DS4_ADAPTIVE_WIDTH=0" "DFLASH_DS4_DRAFT=$DRAFT_MODEL" "DFLASH_DS4_DRAFT_GPU=0" @@ -185,11 +267,58 @@ server_env=( "DFLASH_MOE_FUSED_COMBINE=0" ) +if [[ -n "$DECODE_HOTNESS_CSV" ]]; then + server_env+=( + "DFLASH_DS4_DECODE_HOTNESS_CSV=$DECODE_HOTNESS_CSV" + ) +fi +if [[ "$DYNAMIC_ROUTE_BALANCE" == 1 ]]; then + server_env+=( + "DFLASH_MOE_TP_DYNAMIC_ROUTE_BALANCE=1" + "DFLASH_MOE_TP_DYNAMIC_MAIN_SLOTS=$DYNAMIC_MAIN_SLOTS" + ) + if [[ -n "$DYNAMIC_MAIN_SLOTS_X2" ]]; then + server_env+=( + "DFLASH_MOE_TP_DYNAMIC_MAIN_SLOTS_X2=$DYNAMIC_MAIN_SLOTS_X2" + ) + fi + if [[ -n "$DYNAMIC_MAIN_SLOTS_X4" ]]; then + server_env+=( + "DFLASH_MOE_TP_DYNAMIC_MAIN_SLOTS_X4=$DYNAMIC_MAIN_SLOTS_X4" + ) + fi +fi +if [[ "$SHARED_FFN_PEER_FRACTION" != 0 ]]; then + server_env+=( + "DFLASH_MOE_TP_SHARED_FFN_PEER_FRACTION=$SHARED_FFN_PEER_FRACTION" + "DFLASH_MOE_TP_MAIN_TO_PEER_RATE=$MAIN_TO_PEER_RATE" + ) +fi +if [[ "$FUSED_OWNER_RESIDUAL" == 1 ]]; then + server_env+=("DFLASH_MOE_TP_FUSED_OWNER_RESIDUAL=1") +fi +if [[ "$ALIGN_SHARED_IDS" == 1 ]]; then + server_env+=("DFLASH_CUDA_MMVQ_MOE_ALIGN_SHARED_IDS=1") +fi if [[ -n "$CUDA_DISABLE_GRAPHS_DEVICES" ]]; then server_env+=( "GGML_CUDA_DISABLE_GRAPHS_DEVICES=$CUDA_DISABLE_GRAPHS_DEVICES" ) fi +if [[ -n "$ROCBLAS_USE_HIPBLASLT" ]]; then + server_env+=("ROCBLAS_USE_HIPBLASLT=$ROCBLAS_USE_HIPBLASLT") +fi +if [[ -n "$HIPBLASLT_LOG_MASK" ]]; then + server_env+=("HIPBLASLT_LOG_MASK=$HIPBLASLT_LOG_MASK") +fi +if [[ -n "$HIPBLASLT_TUNING_FILE" ]]; then + server_env+=("HIPBLASLT_TUNING_FILE=$HIPBLASLT_TUNING_FILE") +fi +if [[ -n "$HIPBLASLT_TUNING_OVERRIDE_FILE" ]]; then + server_env+=( + "HIPBLASLT_TUNING_OVERRIDE_FILE=$HIPBLASLT_TUNING_OVERRIDE_FILE" + ) +fi # Preserve only the explicit profiler-wrapper controls across env -i. Ordinary # qualification runs leave these unset and retain the exact established env. @@ -225,6 +354,9 @@ fi if [[ "$Q5_VERIFY" == 1 ]]; then server_env+=("DFLASH_DS4_Q5_VERIFY=1") fi +if [[ "$Q6_VERIFY" == 1 ]]; then + server_env+=("DFLASH_DS4_Q6_VERIFY=1") +fi if [[ "$FP4_Q5_X4_PLUS1" != auto ]]; then server_env+=("DFLASH_CUDA_MMVQ_FP4_Q5_X4_PLUS1=$FP4_Q5_X4_PLUS1") fi @@ -245,7 +377,7 @@ server_args=( --hard-limit-reply-budget 0 --chunk 2048 --ds4-fused-decode - --ds4-expert-top-k 4 + --ds4-expert-top-k "$EXPERT_TOP_K" --ds4-prefill sparse --peer-access ) @@ -260,10 +392,27 @@ server_args=( echo "block_radix_topk=$BLOCK_RADIX_TOPK" echo "pack_q4_indexer=$PACK_Q4_INDEXER" echo "q5_verify=$Q5_VERIFY" + echo "q6_verify=$Q6_VERIFY" + echo "verify_width=$VERIFY_WIDTH" echo "fp4_q5_x4_plus1=$FP4_Q5_X4_PLUS1" echo "critical_path_placement=$CRITICAL_PATH_PLACEMENT" echo "main_to_peer_rate=$MAIN_TO_PEER_RATE" echo "balance_min_hot=$BALANCE_MIN_HOT" + echo "hotness_csv=$HOTNESS_CSV" + echo "hotness_sha256=$(sha256sum "$HOTNESS_CSV" | awk '{print $1}')" + echo "decode_hotness_csv=$DECODE_HOTNESS_CSV" + if [[ -n "$DECODE_HOTNESS_CSV" ]]; then + echo "decode_hotness_sha256=$(sha256sum \ + "$DECODE_HOTNESS_CSV" | awk '{print $1}')" + fi + echo "dynamic_route_balance=$DYNAMIC_ROUTE_BALANCE" + echo "dynamic_main_slots=$DYNAMIC_MAIN_SLOTS" + echo "dynamic_main_slots_x2=$DYNAMIC_MAIN_SLOTS_X2" + echo "dynamic_main_slots_x4=$DYNAMIC_MAIN_SLOTS_X4" + echo "shared_ffn_peer_fraction=$SHARED_FFN_PEER_FRACTION" + echo "fused_owner_residual=$FUSED_OWNER_RESIDUAL" + echo "align_shared_ids=$ALIGN_SHARED_IDS" + echo "expert_top_k=$EXPERT_TOP_K" echo "cache_slots=$CACHE_SLOTS" echo "mmvq_max_ncols=$MMVQ_MAX_NCOLS" echo "targets=$TARGETS" @@ -273,6 +422,14 @@ server_args=( echo "max_ctx=$MAX_CTX" echo "cuda_graph_stats_every=$CUDA_GRAPH_STATS_EVERY" echo "cuda_disable_graphs_devices=$CUDA_DISABLE_GRAPHS_DEVICES" + echo "rocblas_use_hipblaslt=$ROCBLAS_USE_HIPBLASLT" + echo "hipblaslt_log_mask=$HIPBLASLT_LOG_MASK" + echo "hipblaslt_tuning_file=$HIPBLASLT_TUNING_FILE" + echo "hipblaslt_tuning_override_file=$HIPBLASLT_TUNING_OVERRIDE_FILE" + if [[ -n "$HIPBLASLT_TUNING_OVERRIDE_FILE" ]]; then + echo "hipblaslt_tuning_override_sha256=$(sha256sum \ + "$HIPBLASLT_TUNING_OVERRIDE_FILE" | awk '{print $1}')" + fi sha256sum "$SERVER_BIN" stat -c 'target_model=%n bytes=%s mtime=%y' "$TARGET_MODEL" stat -c 'draft_model=%n bytes=%s mtime=%y' "$DRAFT_MODEL" @@ -331,6 +488,14 @@ python3 "$CONTEXT_CLIENT" \ --json-out "$OUT_DIR/decode-client.json" \ 2>&1 | tee "$OUT_DIR/decode-client.log" +python3 "$SERVER_DECODE_SUMMARIZER" \ + --server-log "$SERVER_LOG" \ + --targets "${target_args[@]}" \ + --warmup "$WARMUP" --runs "$RUNS" \ + --expected-tokens "$MAX_TOKENS" \ + --json-out "$OUT_DIR/server-decode-summary.json" \ + 2>&1 | tee "$OUT_DIR/server-decode-summary.log" + rocm-smi --showperflevel --showclocks --showmeminfo vram \ >"$OUT_DIR/rocm-smi-after.txt" 2>&1 || true date -u '+finished_utc=%Y-%m-%dT%H:%M:%SZ' >>"$OUT_DIR/manifest.txt" diff --git a/server/scripts/qualify_ds4_q5_r9700_strix.sh b/server/scripts/qualify_ds4_q5_r9700_strix.sh new file mode 100755 index 000000000..60be698ee --- /dev/null +++ b/server/scripts/qualify_ds4_q5_r9700_strix.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Reproduce the qualified q=5 profile on the LuceBox R9700 + Strix Halo pair. +# The hipBLASLt table is tied to these GPU architectures and ROCm 7.2.4. + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +CHECKOUT="${CHECKOUT:-$(cd "$SCRIPT_DIR/../.." && pwd)}" +DEFAULT_TUNING_FILE="$CHECKOUT/server/config/hipblaslt/r9700-strix-rocm-7.2.4.txt" +EXPECTED_TUNING_SHA256="7b909b84adaf8a24fdfb760a8d66cd5adf95bf6e8c0270e1a10e929f21499e98" +TUNING_FILE="${HIPBLASLT_TUNING_OVERRIDE_FILE:-$DEFAULT_TUNING_FILE}" +PROFILE_DIR="$CHECKOUT/server/config/ds4/r9700-strix" +DEFAULT_HOTNESS_CSV="$PROFILE_DIR/prefill-routing.csv" +DEFAULT_DECODE_HOTNESS_CSV="$PROFILE_DIR/decode-routing.csv" +EXPECTED_HOTNESS_SHA256="927d1449906d7881097e818c6e0210a7834b4231594e4daefb18f40e37955dc2" +EXPECTED_DECODE_HOTNESS_SHA256="f5989e0d5a5ef91d5ffd3068f81ab568c5b0bc6b755461acbfc32536dda47e15" +HOTNESS_CSV="${HOTNESS_CSV:-$DEFAULT_HOTNESS_CSV}" +DECODE_HOTNESS_CSV="${DECODE_HOTNESS_CSV:-$DEFAULT_DECODE_HOTNESS_CSV}" + +verify_default_file() { + local path=$1 + local expected_sha256=$2 + local label=$3 + if [[ ! -f "$path" ]]; then + echo "missing $label: $path" >&2 + exit 2 + fi + local actual_sha256 + actual_sha256=$(sha256sum "$path" | awk '{print $1}') + if [[ "$actual_sha256" != "$expected_sha256" ]]; then + echo "unexpected $label checksum: $actual_sha256" >&2 + exit 2 + fi +} + +if [[ "$TUNING_FILE" == "$DEFAULT_TUNING_FILE" ]]; then + verify_default_file \ + "$TUNING_FILE" "$EXPECTED_TUNING_SHA256" "hipBLASLt tuning table" +elif [[ ! -f "$TUNING_FILE" ]]; then + echo "missing hipBLASLt tuning file: $TUNING_FILE" >&2 + exit 2 +fi +if [[ "$HOTNESS_CSV" == "$DEFAULT_HOTNESS_CSV" ]]; then + verify_default_file \ + "$HOTNESS_CSV" "$EXPECTED_HOTNESS_SHA256" "prefill routing profile" +fi +if [[ "$DECODE_HOTNESS_CSV" == "$DEFAULT_DECODE_HOTNESS_CSV" ]]; then + verify_default_file \ + "$DECODE_HOTNESS_CSV" "$EXPECTED_DECODE_HOTNESS_SHA256" \ + "decode routing profile" +fi + +exec env -u HIPBLASLT_TUNING_FILE \ + ROCBLAS_USE_HIPBLASLT=1 \ + HIPBLASLT_TUNING_OVERRIDE_FILE="$TUNING_FILE" \ + HOTNESS_CSV="$HOTNESS_CSV" \ + DECODE_HOTNESS_CSV="$DECODE_HOTNESS_CSV" \ + CRITICAL_PATH_PLACEMENT="${CRITICAL_PATH_PLACEMENT:-1}" \ + MAIN_TO_PEER_RATE="${MAIN_TO_PEER_RATE:-4.4}" \ + EXPERT_BUDGET_MB="${EXPERT_BUDGET_MB:-14350}" \ + DYNAMIC_ROUTE_BALANCE="${DYNAMIC_ROUTE_BALANCE:-1}" \ + DYNAMIC_MAIN_SLOTS="${DYNAMIC_MAIN_SLOTS:-3}" \ + DYNAMIC_MAIN_SLOTS_X4="${DYNAMIC_MAIN_SLOTS_X4:-13}" \ + FUSED_OWNER_RESIDUAL="${FUSED_OWNER_RESIDUAL:-1}" \ + TARGETS="${TARGETS:-2048}" \ + WARMUP="${WARMUP:-2}" \ + RUNS="${RUNS:-7}" \ + RUN_ID="${RUN_ID:-ds4-q5-r9700-strix-hipblaslt}" \ + "$SCRIPT_DIR/qualify_ds4_q5_amd.sh" "$@" diff --git a/server/scripts/summarize_ds4_server_decode.py b/server/scripts/summarize_ds4_server_decode.py new file mode 100644 index 000000000..8a37d3c3c --- /dev/null +++ b/server/scripts/summarize_ds4_server_decode.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Summarize authoritative model-side DSpark decode rates from a server log.""" + +from __future__ import annotations + +import argparse +import json +import re +import statistics +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Sequence + + +DECODE_PATTERN = re.compile( + r"\[deepseek4\] DSpark decode: " + r"(?P\d+) tok in (?P\d+(?:\.\d+)?)s " + r"\((?P\d+(?:\.\d+)?) tok/s\) " + r"accept_rate=(?P\d+(?:\.\d+)?)" +) + + +@dataclass(frozen=True) +class DecodeRecord: + tokens: int + seconds: float + tok_s: float + acceptance: float + + +def parse_records(log_text: str) -> list[DecodeRecord]: + return [ + DecodeRecord( + tokens=int(match.group("tokens")), + seconds=float(match.group("seconds")), + tok_s=float(match.group("rate")), + acceptance=float(match.group("acceptance")), + ) + for match in DECODE_PATTERN.finditer(log_text) + ] + + +def summarize_records( + records: Sequence[DecodeRecord], + targets: Sequence[int], + warmup: int, + runs: int, + expected_tokens: int, +) -> dict[str, object]: + if warmup < 0 or runs < 1: + raise ValueError("warmup must be non-negative and runs must be positive") + requests_per_target = warmup + runs + expected_records = len(targets) * requests_per_target + if len(records) != expected_records: + raise ValueError( + f"found {len(records)} DSpark decode records; expected {expected_records}" + ) + + groups: list[dict[str, object]] = [] + offset = 0 + for target in targets: + all_rows = list(records[offset : offset + requests_per_target]) + offset += requests_per_target + if any(row.tokens != expected_tokens for row in all_rows): + actual = sorted({row.tokens for row in all_rows}) + raise ValueError( + f"target {target} completion lengths {actual}; expected {expected_tokens}" + ) + measured = all_rows[warmup:] + rates = [row.tok_s for row in measured] + total_seconds = sum(row.seconds for row in measured) + groups.append({ + "target_context": target, + "n": len(measured), + "completion_tokens": expected_tokens, + "server_decode_tok_s_median": round(statistics.median(rates), 3), + "server_decode_tok_s_min": round(min(rates), 3), + "server_decode_tok_s_max": round(max(rates), 3), + "server_decode_tok_s_weighted": round( + sum(row.tokens for row in measured) / total_seconds, 3 + ), + "acceptance_median": round( + statistics.median(row.acceptance for row in measured), 4 + ), + "records": [asdict(row) for row in measured], + }) + return { + "schema_version": 1, + "metric": "model-side DSpark decode", + "warmup": warmup, + "runs": runs, + "groups": groups, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--server-log", type=Path, required=True) + parser.add_argument("--targets", type=int, nargs="+", required=True) + parser.add_argument("--warmup", type=int, required=True) + parser.add_argument("--runs", type=int, required=True) + parser.add_argument("--expected-tokens", type=int, required=True) + parser.add_argument("--json-out", type=Path, required=True) + args = parser.parse_args() + + try: + payload = summarize_records( + parse_records(args.server_log.read_text(encoding="utf-8")), + args.targets, + args.warmup, + args.runs, + args.expected_tokens, + ) + except (OSError, ValueError) as exc: + parser.error(str(exc)) + + args.json_out.parent.mkdir(parents=True, exist_ok=True) + rendered = json.dumps(payload, indent=2) + "\n" + args.json_out.write_text(rendered, encoding="utf-8") + print(rendered, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/src/common/heterogeneous_stage_planner.cpp b/server/src/common/heterogeneous_stage_planner.cpp new file mode 100644 index 000000000..1641e0e1b --- /dev/null +++ b/server/src/common/heterogeneous_stage_planner.cpp @@ -0,0 +1,80 @@ +#include "heterogeneous_stage_planner.h" + +#include +#include + +namespace dflash::common { + +namespace { + +HeterogeneousStagePlan unsplit_plan(int total_width, int alignment) { + HeterogeneousStagePlan plan; + plan.total_width = std::max(0, total_width); + plan.main_width = plan.total_width; + plan.alignment = std::max(1, alignment); + return plan; +} + +} // namespace + +HeterogeneousStagePlan plan_heterogeneous_stage_width( + int total_width, + int alignment, + double peer_fraction) { + alignment = std::max(1, alignment); + HeterogeneousStagePlan plan = unsplit_plan(total_width, alignment); + if (total_width <= 0 || total_width % alignment != 0 || + !std::isfinite(peer_fraction) || peer_fraction < 0.0 || + peer_fraction > 1.0) { + return plan; + } + + if (peer_fraction == 0.0) return plan; + if (peer_fraction == 1.0) { + plan.main_width = 0; + plan.peer_width = total_width; + plan.peer_fraction = 1.0; + return plan; + } + + const double requested_units = + peer_fraction * (double) total_width / (double) alignment; + const int total_units = total_width / alignment; + if (total_units < 2) return plan; + int peer_units = std::clamp( + (int) std::llround(requested_units), 1, total_units - 1); + + plan.peer_width = peer_units * alignment; + plan.main_width = total_width - plan.peer_width; + plan.peer_fraction = + (double) plan.peer_width / (double) plan.total_width; + return plan; +} + +HeterogeneousStagePlan plan_balanced_heterogeneous_stage_width( + int total_width, + int alignment, + double main_rate, + double peer_rate, + double main_fixed_work, + double peer_fixed_work) { + if (total_width <= 0 || alignment <= 0 || + total_width % alignment != 0 || + !std::isfinite(main_rate) || !std::isfinite(peer_rate) || + !std::isfinite(main_fixed_work) || !std::isfinite(peer_fixed_work) || + main_rate <= 0.0 || peer_rate <= 0.0 || + main_fixed_work < 0.0 || peer_fixed_work < 0.0) { + return unsplit_plan(total_width, alignment); + } + + // Solve (main_fixed + total - peer) / main_rate == + // (peer_fixed + peer) / peer_rate. + const double peer_work = std::clamp( + (peer_rate * (main_fixed_work + (double) total_width) - + main_rate * peer_fixed_work) / + (main_rate + peer_rate), 0.0, (double) total_width); + return plan_heterogeneous_stage_width( + total_width, alignment, peer_work / (double) total_width); +} + +} // namespace dflash::common diff --git a/server/src/common/heterogeneous_stage_planner.h b/server/src/common/heterogeneous_stage_planner.h new file mode 100644 index 000000000..3d50a1a59 --- /dev/null +++ b/server/src/common/heterogeneous_stage_planner.h @@ -0,0 +1,54 @@ +// Generic work splitter for two heterogeneous execution owners. +// +// The planner is deliberately independent of ggml, model architecture, and +// device vendor. Callers describe a splittable width and the granularity +// required by their kernels or quantization format. + +#pragma once + +namespace dflash::common { + +struct HeterogeneousStagePlan { + int total_width = 0; + int main_width = 0; + int peer_width = 0; + int alignment = 1; + double peer_fraction = 0.0; + + bool split() const { + return total_width > 0 && main_width > 0 && peer_width > 0 && + main_width + peer_width == total_width; + } + + bool valid() const { + return total_width > 0 && main_width >= 0 && peer_width >= 0 && + main_width + peer_width == total_width; + } + + bool uses_peer() const { + return valid() && peer_width > 0; + } +}; + +// Plan an explicit partition. The result is rounded to the closest legal peer +// width. Fractions 0 and 1 assign the whole stage to one owner; fractions in +// between leave at least one aligned unit on each. An invalid request returns +// an unsplit plan owned entirely by `main`. +HeterogeneousStagePlan plan_heterogeneous_stage_width( + int total_width, + int alignment, + double peer_fraction); + +// Balance a stage from owner throughput and already-scheduled fixed work. +// Work and rates may use any consistent unit (bytes and bytes/us, FLOPs and +// FLOPs/us, or calibrated route-equivalents). The selected split minimizes +// the estimated maximum of the two owner completion times before alignment. +HeterogeneousStagePlan plan_balanced_heterogeneous_stage_width( + int total_width, + int alignment, + double main_rate, + double peer_rate, + double main_fixed_work = 0.0, + double peer_fixed_work = 0.0); + +} // namespace dflash::common diff --git a/server/src/common/moe_hybrid_ffn_eval.cpp b/server/src/common/moe_hybrid_ffn_eval.cpp index 727d87307..b27a9a94b 100644 --- a/server/src/common/moe_hybrid_ffn_eval.cpp +++ b/server/src/common/moe_hybrid_ffn_eval.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace dflash::common { @@ -129,6 +130,66 @@ const MoeHybridGraphPolicy & moe_hybrid_graph_policy() { return policy; } +// Merge an owner-local dense/shared contribution in the routed owner's +// existing final reduction. Unsupported kernels retain an ordinary local add. +static bool fused_owner_residual_enabled() { + return moe_policy_flag( + "DFLASH_MOE_TP_FUSED_OWNER_RESIDUAL", + "DFLASH_DS4_TP_FUSED_OWNER_RESIDUAL"); +} + +static int dynamic_route_balance_main_slots_x4(int n_used) { + static const long requested_x4 = [] { + const char * enabled = moe_policy_env( + "DFLASH_MOE_TP_DYNAMIC_ROUTE_BALANCE", + "DFLASH_DS4_TP_DYNAMIC_ROUTE_BALANCE"); + if (!enabled || !*enabled || std::strcmp(enabled, "0") == 0) { + return 0L; + } + const char * raw_slots_x4 = moe_policy_env( + "DFLASH_MOE_TP_DYNAMIC_MAIN_SLOTS_X4", + "DFLASH_DS4_TP_DYNAMIC_MAIN_SLOTS_X4"); + const char * raw_slots_x2 = moe_policy_env( + "DFLASH_MOE_TP_DYNAMIC_MAIN_SLOTS_X2", + "DFLASH_DS4_TP_DYNAMIC_MAIN_SLOTS_X2"); + const char * raw_slots = moe_policy_env( + "DFLASH_MOE_TP_DYNAMIC_MAIN_SLOTS", + "DFLASH_DS4_TP_DYNAMIC_MAIN_SLOTS"); + const char * raw = raw_slots; + long scale = 4L; + if (raw_slots_x4 && *raw_slots_x4) { + raw = raw_slots_x4; + scale = 1L; + } else if (raw_slots_x2 && *raw_slots_x2) { + raw = raw_slots_x2; + scale = 2L; + } + if (!raw || !*raw) return 12L; + char * end = nullptr; + const long value = std::strtol(raw, &end, 10); + return end && end != raw && *end == '\0' && value > 0 + ? scale * value : -1L; + }(); + if (requested_x4 == 0) return 0; + if (n_used <= 0 || requested_x4 < 4 || requested_x4 > 4L * n_used) { + static std::once_flag invalid_log; + std::call_once(invalid_log, [n_used] { + std::fprintf(stderr, + "[moe-hybrid] invalid dynamic route balance: four times the " + "main slot quota must be in [4,%d] for top-k=%d\n", + 4 * std::max(0, n_used), n_used); + }); + return -1; + } + static std::once_flag active_log; + std::call_once(active_log, [] { + std::fprintf(stderr, + "[moe-hybrid] dynamic route balance active: main_slots=%.2f\n", + 0.25 * (double) requested_x4); + }); + return (int) requested_x4; +} + static void add_hybrid_telemetry(MoeHybridFfnTelemetry & dst, const MoeHybridFfnTelemetry & src) { dst.ffn_wall_us += src.ffn_wall_us; @@ -200,16 +261,28 @@ static MoeExpertComputeIpcMode parse_moe_expert_compute_ipc_mode() { // Returns the output tensor (or nullptr if no shared expert is present). static ggml_tensor * build_shared_expert_subgraph( ggml_context * ctx, const MoeLayerDesc & desc, ggml_tensor * inp, - float swiglu_clamp = 0.0f) { + float swiglu_clamp = 0.0f, + std::vector * backend_nodes = nullptr) { if (!desc.ffn_up_shexp || !desc.ffn_gate_shexp || !desc.ffn_down_shexp) return nullptr; - ggml_tensor * sh_gate = apply_scale2(ctx, - ggml_mul_mat(ctx, desc.ffn_gate_shexp, inp), desc.ffn_gate_shexp_s); - ggml_tensor * sh_up = apply_scale2(ctx, - ggml_mul_mat(ctx, desc.ffn_up_shexp, inp), desc.ffn_up_shexp_s); - ggml_tensor * sh_gu = swiglu_maybe_clamped(ctx, sh_gate, sh_up, swiglu_clamp); - ggml_tensor * shared = apply_scale2(ctx, - ggml_mul_mat(ctx, desc.ffn_down_shexp, sh_gu), desc.ffn_down_shexp_s); + const auto track = [backend_nodes](ggml_tensor * tensor) { + if (tensor && backend_nodes) backend_nodes->push_back(tensor); + return tensor; + }; + ggml_tensor * sh_gate = track(ggml_mul_mat( + ctx, desc.ffn_gate_shexp, inp)); + sh_gate = track(apply_scale2( + ctx, sh_gate, desc.ffn_gate_shexp_s)); + ggml_tensor * sh_up = track(ggml_mul_mat( + ctx, desc.ffn_up_shexp, inp)); + sh_up = track(apply_scale2( + ctx, sh_up, desc.ffn_up_shexp_s)); + ggml_tensor * sh_gu = track( + swiglu_maybe_clamped(ctx, sh_gate, sh_up, swiglu_clamp)); + ggml_tensor * shared = track(ggml_mul_mat( + ctx, desc.ffn_down_shexp, sh_gu)); + shared = track(apply_scale2( + ctx, shared, desc.ffn_down_shexp_s)); if (desc.ffn_gate_inp_shexp) { // The shared-expert gate is a single-row weight (M=1): out[0,n] = sum_k W[k]*inp[k,n]. // Computing it as ggml_mul_mat routes to cublas, and on the shipped CUDA 12.0 @@ -219,15 +292,54 @@ static ggml_tensor * build_shared_expert_subgraph( // the stream and surfaces as an illegal access in the next op. Compute the gate as // broadcast elementwise-mul + sum_rows instead: identical math, ggml kernels only, // no cublas. This is what unblocks single-pass full-batch verify. - ggml_tensor * gate_prod = ggml_mul(ctx, inp, desc.ffn_gate_inp_shexp); - ggml_tensor * shared_gate = apply_scale2(ctx, - ggml_sum_rows(ctx, gate_prod), desc.ffn_gate_inp_shexp_s); - shared_gate = ggml_sigmoid(ctx, shared_gate); - shared = ggml_mul(ctx, shared, shared_gate); + ggml_tensor * gate_prod = track( + ggml_mul(ctx, inp, desc.ffn_gate_inp_shexp)); + ggml_tensor * shared_gate = track(ggml_sum_rows(ctx, gate_prod)); + shared_gate = track(apply_scale2( + ctx, shared_gate, desc.ffn_gate_inp_shexp_s)); + shared_gate = track(ggml_sigmoid(ctx, shared_gate)); + shared = track(ggml_mul(ctx, shared, shared_gate)); } return shared; } +static MoeLayerDesc make_shared_ffn_main_shard( + ggml_context * ctx, + const MoeLayerDesc & desc, + const MoeHybridLayerStorage & storage) { + MoeLayerDesc shard; + const int64_t width = storage.shared_ffn_main_width; + shard.ffn_gate_shexp = ggml_view_2d( + ctx, desc.ffn_gate_shexp, + desc.ffn_gate_shexp->ne[0], width, + desc.ffn_gate_shexp->nb[1], 0); + shard.ffn_up_shexp = ggml_view_2d( + ctx, desc.ffn_up_shexp, + desc.ffn_up_shexp->ne[0], width, + desc.ffn_up_shexp->nb[1], 0); + shard.ffn_down_shexp = ggml_view_2d( + ctx, desc.ffn_down_shexp, + width, desc.ffn_down_shexp->ne[1], + desc.ffn_down_shexp->nb[1], 0); + shard.ffn_gate_shexp_s = desc.ffn_gate_shexp_s; + shard.ffn_up_shexp_s = desc.ffn_up_shexp_s; + shard.ffn_down_shexp_s = desc.ffn_down_shexp_s; + return shard; +} + +static MoeLayerDesc make_shared_ffn_peer_shard( + const MoeLayerDesc & desc, + const MoeHybridLayerStorage & storage) { + MoeLayerDesc shard; + shard.ffn_gate_shexp = storage.gate_shexp_peer; + shard.ffn_up_shexp = storage.up_shexp_peer; + shard.ffn_down_shexp = storage.down_shexp_peer; + shard.ffn_gate_shexp_s = desc.ffn_gate_shexp_s; + shard.ffn_up_shexp_s = desc.ffn_up_shexp_s; + shard.ffn_down_shexp_s = desc.ffn_down_shexp_s; + return shard; +} + static int fixed_slot_graphs_mode() { static const int mode = [] { const char * env = std::getenv("DFLASH_MOE_FIXED_SLOT_GRAPHS"); @@ -577,7 +689,8 @@ static bool build_batched_routed_graph( std::vector * backend_nodes = nullptr, bool allow_fused_combine = false, bool force_fused_combine = false, - bool defer_route_reduction = false) + bool defer_route_reduction = false, + ggml_tensor * owner_residual = nullptr) { const auto track = [&](ggml_tensor * t) -> ggml_tensor * { if (backend_nodes && t) backend_nodes->push_back(t); @@ -592,6 +705,12 @@ static bool build_batched_routed_graph( ctx, sel, n_used, 1, sel->nb[1], (size_t) t * sel->nb[1])); ggml_tensor * wts_col = ggml_cont(ctx, ggml_view_2d( ctx, wts, n_used, 1, wts->nb[1], (size_t) t * wts->nb[1])); + ggml_tensor * residual_col = owner_residual + ? track(ggml_cont(ctx, ggml_view_2d( + ctx, owner_residual, n_embd, 1, + owner_residual->nb[1], + (size_t) t * owner_residual->nb[1]))) + : nullptr; ggml_tensor * routed_col = nullptr; if (!build_batched_routed_graph( ctx, gate_tensor, up_tensor, down_tensor, gate_up_tensor, @@ -600,7 +719,7 @@ static bool build_batched_routed_graph( n_embd, n_ff_exp, n_used, 1, swiglu_clamp, &routed_col, false, backend_nodes, allow_fused_combine, force_fused_combine, - defer_route_reduction)) { + defer_route_reduction, residual_col)) { return false; } // Reduced owner outputs concatenate by token columns. Canonical @@ -645,6 +764,7 @@ static bool build_batched_routed_graph( if (coarse_split_requested && coarse_split_eligible) { *out_routed = track(ggml_ds4_moe_owner_split( ctx, inp, gate_tensor, up_tensor, down_tensor, sel, wts, + owner_residual, n_ff_exp, swiglu_clamp, gate_scale, up_scale, down_scale)); return *out_routed != nullptr; @@ -656,6 +776,7 @@ static bool build_batched_routed_graph( down_tensor->type == GGML_TYPE_Q3_0_ROCMFPX) { *out_routed = track(ggml_ds4_moe_owner( ctx, inp, gate_up_tensor, down_tensor, sel, wts, + owner_residual, n_ff_exp, swiglu_clamp, down_scale)); return *out_routed != nullptr; } else if (gate_up_tensor && @@ -706,6 +827,9 @@ static bool build_batched_routed_graph( if (!defer_route_reduction && allow_fused_combine && (force_fused_combine || moe_hybrid_graph_policy().fused_combine)) { *out_routed = track(ggml_laguna_moe_combine(ctx, experts, wts)); + if (owner_residual) { + *out_routed = track(ggml_add(ctx, *out_routed, owner_residual)); + } return *out_routed != nullptr; } @@ -724,6 +848,9 @@ static bool build_batched_routed_graph( ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd, 1, n_tokens)); ggml_tensor * moe_sum = track(ggml_repeat_back(ctx, experts, sum_shape)); *out_routed = track(ggml_reshape_2d(ctx, moe_sum, n_embd, n_tokens)); + if (owner_residual) { + *out_routed = track(ggml_add(ctx, *out_routed, owner_residual)); + } return true; } @@ -756,6 +883,8 @@ static bool build_moe_owner_remap( ggml_tensor * global_ids, ggml_tensor * router_weights, int n_tokens, + int dynamic_main_slots_x4, + bool main_owner, MoeOwnerGraphSpec & owner) { if (!owner.local_by_global || (int) owner.local_by_global->size() != cfg.n_expert || @@ -784,6 +913,24 @@ static bool build_moe_owner_remap( ggml_set_output(*owner.valid_lut); } + if (dynamic_main_slots_x4 > 0) { + // The secondary owner must hold every expert because it receives the + // exact complement of the capped primary routes. + if (!main_owner && std::any_of( + owner.local_by_global->begin(), owner.local_by_global->end(), + [](int32_t local) { return local < 0; })) { + return false; + } + owner.local_ids = track(ggml_ds4_moe_balanced_owner_ids( + ctx, global_ids, router_weights, + *owner.local_lut, *owner.valid_lut, + dynamic_main_slots_x4, main_owner)); + // Negative owner IDs suppress non-owned routes exactly in the + // dedicated MMVQ kernels, so the canonical route weights can be reused. + owner.masked_weights = router_weights; + return owner.local_ids != nullptr; + } + // Store immutable q-replicated lookup rows as graph inputs instead of // running owner-local REPEAT kernels in every layer and verifier step. ggml_tensor * mapped = track(ggml_get_rows( @@ -805,9 +952,12 @@ static bool prepare_moe_owner_branch( ggml_tensor * global_ids, ggml_tensor * router_weights, int n_tokens, + int dynamic_main_slots_x4, + bool main_owner, MoeOwnerGraphSpec & owner) { return !owner.available() || build_moe_owner_remap( - ctx, cfg, global_ids, router_weights, n_tokens, owner); + ctx, cfg, global_ids, router_weights, n_tokens, + dynamic_main_slots_x4, main_owner, owner); } static void align_moe_owner_routes( @@ -832,6 +982,7 @@ static bool build_moe_owner_branch( int n_tokens, bool canonical_route_join, bool allow_fused_combine, + ggml_tensor * owner_residual, MoeOwnerGraphSpec & owner) { if (!owner.available()) { return true; @@ -850,7 +1001,8 @@ static bool build_moe_owner_branch( cfg.n_embd, cfg.n_ff_exp, cfg.n_expert_used, n_tokens, cfg.swiglu_clamp, &owner.output, tokenwise, owner.branch_nodes, allow_fused_combine, - /*force_fused_combine=*/false, canonical_route_join); + /*force_fused_combine=*/false, canonical_route_join, + owner_residual); } static ggml_tensor * build_moe_owner_join( @@ -995,6 +1147,10 @@ bool build_moe_hybrid_ffn_graph( const bool canonical_route_join = join_mode == MoeHybridJoinMode::CanonicalRouteOrder; + const int n_used = cfg.n_expert_used; + const int dynamic_main_slots_x4 = + dynamic_route_balance_main_slots_x4(n_used); + if (dynamic_main_slots_x4 < 0) return false; // Both owner remaps consume the same normalized top-k route weights. // Expose the canonical tensor so the scheduler can keep it on the primary // backend rather than discovering it late through the secondary branch. @@ -1015,25 +1171,79 @@ bool build_moe_hybrid_ffn_graph( // Keep graph construction order stable: both remaps, then both optional ID // alignments, then both expert branches. if (!prepare_moe_owner_branch( - ctx, cfg, global_ids, router_weights, n_tokens, primary_owner) || + ctx, cfg, global_ids, router_weights, n_tokens, + dynamic_main_slots_x4, true, primary_owner) || !prepare_moe_owner_branch( - ctx, cfg, global_ids, router_weights, n_tokens, secondary_owner)) { + ctx, cfg, global_ids, router_weights, n_tokens, + dynamic_main_slots_x4, false, secondary_owner)) { return false; } align_moe_owner_routes(ctx, n_tokens, primary_owner); align_moe_owner_routes(ctx, n_tokens, secondary_owner); + + // Shared/dense FFNs can be width-partitioned only when each owner reduces + // its routes locally. Canonical cross-runtime joins retain the established + // full shared stage on the primary backend to preserve route-order math. + ggml_tensor * shared_main = nullptr; + ggml_tensor * shared_peer = nullptr; + if (!canonical_route_join && include_shared && + storage.has_shared_ffn_peer_partition()) { + const MoeLayerDesc peer_shard = + make_shared_ffn_peer_shard(desc, storage); + if (storage.shared_ffn_main_width > 0) { + const MoeLayerDesc main_shard = + make_shared_ffn_main_shard(ctx, desc, storage); + shared_main = build_shared_expert_subgraph( + ctx, main_shard, inp, cfg.swiglu_clamp, &out.hot_nodes); + } + shared_peer = build_shared_expert_subgraph( + ctx, peer_shard, inp, cfg.swiglu_clamp, &out.cold_nodes); + if ((storage.shared_ffn_main_width > 0 && !shared_main) || + !shared_peer) { + return false; + } + } else if (!canonical_route_join && include_shared) { + shared_main = build_shared_expert_subgraph( + ctx, desc, inp, cfg.swiglu_clamp, &out.hot_nodes); + if (!shared_main) return false; + } + + const bool fuse_owner_residual = + !canonical_route_join && fused_owner_residual_enabled(); + const bool main_shared_consumed = + fuse_owner_residual && shared_main && primary_owner.available(); + const bool peer_shared_consumed = + fuse_owner_residual && shared_peer && secondary_owner.available(); if (!build_moe_owner_branch( ctx, cfg, desc, inp, n_tokens, canonical_route_join, - allow_fused_combine, primary_owner) || + allow_fused_combine, + main_shared_consumed ? shared_main : nullptr, + primary_owner) || !build_moe_owner_branch( ctx, cfg, desc, inp, n_tokens, canonical_route_join, - allow_fused_combine, secondary_owner)) { + allow_fused_combine, + peer_shared_consumed ? shared_peer : nullptr, + secondary_owner)) { return false; } + if (!main_shared_consumed && shared_main) { + primary_owner.output = primary_owner.output + ? ggml_add(ctx, primary_owner.output, shared_main) + : shared_main; + out.hot_nodes.push_back(primary_owner.output); + } + if (!peer_shared_consumed && shared_peer) { + secondary_owner.output = secondary_owner.output + ? ggml_add(ctx, secondary_owner.output, shared_peer) + : shared_peer; + out.cold_nodes.push_back(secondary_owner.output); + } + ggml_tensor * combined = build_moe_owner_join( ctx, schedule_graph, cfg, desc, inp, global_ids, router_weights, - n_tokens, include_shared, canonical_route_join, + n_tokens, canonical_route_join && include_shared, + canonical_route_join, primary_owner.output, secondary_owner.output, out); if (!combined) return false; diff --git a/server/src/common/moe_hybrid_placement.cpp b/server/src/common/moe_hybrid_placement.cpp index f9237f709..414423b91 100644 --- a/server/src/common/moe_hybrid_placement.cpp +++ b/server/src/common/moe_hybrid_placement.cpp @@ -263,6 +263,111 @@ bool MoeHybridPlacement::build_from_stats_with_layer_bytes( return true; } +bool MoeHybridPlacement::expand_from_stats_with_layer_bytes( + const MoeHybridRoutingStats & stats, + const std::vector & layer_expert_bytes, + uint64_t total_hot_budget_bytes, + MoeHybridPlacement & in_out, + std::string * err) { + if (stats.empty() || stats.n_layer <= 0 || stats.n_expert <= 0) { + if (err) *err = "stats not initialized"; + return false; + } + if ((int) layer_expert_bytes.size() != stats.n_layer) { + if (err) *err = "layer_expert_bytes size mismatch"; + return false; + } + if (!in_out.matches( + stats.n_layer, stats.n_expert, stats.n_expert_used)) { + if (err) *err = "existing placement shape does not match stats"; + return false; + } + if (total_hot_budget_bytes == 0) { + if (err) *err = "total_hot_budget_bytes must be > 0"; + return false; + } + + MoeHybridPlacement tmp = in_out; + std::vector> resident( + (size_t) stats.n_layer, + std::vector((size_t) stats.n_expert, 0)); + uint64_t used_bytes = 0; + for (int il = 0; il < stats.n_layer; ++il) { + const auto & ids = tmp.hot_expert_ids[(size_t) il]; + if ((int) ids.size() != tmp.hot_counts[(size_t) il]) { + if (err) *err = "existing placement count does not match ids"; + return false; + } + const uint64_t expert_bytes = layer_expert_bytes[(size_t) il]; + if (expert_bytes > 0 && ids.size() > + (std::numeric_limits::max() - used_bytes) / + expert_bytes) { + if (err) *err = "existing placement byte count overflow"; + return false; + } + used_bytes += (uint64_t) ids.size() * expert_bytes; + for (int32_t id : ids) { + if (id < 0 || id >= stats.n_expert || + resident[(size_t) il][(size_t) id]) { + if (err) *err = "existing placement contains an invalid expert"; + return false; + } + resident[(size_t) il][(size_t) id] = 1; + } + } + if (used_bytes > total_hot_budget_bytes) { + if (err) *err = "existing placement exceeds byte budget"; + return false; + } + + std::vector> ranked((size_t) stats.n_layer); + std::vector next((size_t) stats.n_layer, 0); + for (int il = 0; il < stats.n_layer; ++il) { + ranked[(size_t) il] = stats.ranked_experts(il); + } + + uint64_t remaining = total_hot_budget_bytes - used_bytes; + while (true) { + int best_layer = -1; + int best_expert = -1; + double best_value = -1.0; + uint64_t best_gain = 0; + for (int il = 0; il < stats.n_layer; ++il) { + const uint64_t bytes = layer_expert_bytes[(size_t) il]; + if (bytes == 0 || bytes > remaining) continue; + auto & cursor = next[(size_t) il]; + const auto & layer_ranked = ranked[(size_t) il]; + while (cursor < layer_ranked.size() && + resident[(size_t) il] + [(size_t) layer_ranked[cursor]]) { + ++cursor; + } + if (cursor == layer_ranked.size()) continue; + const int expert = layer_ranked[cursor]; + const uint64_t gain = stats.count(il, expert); + const double value = (double) gain / (double) bytes; + if (best_layer < 0 || value > best_value || + (value == best_value && gain > best_gain)) { + best_layer = il; + best_expert = expert; + best_value = value; + best_gain = gain; + } + } + if (best_layer < 0) break; + tmp.hot_expert_ids[(size_t) best_layer].push_back( + (int32_t) best_expert); + tmp.hot_counts[(size_t) best_layer]++; + tmp.total_hot++; + resident[(size_t) best_layer][(size_t) best_expert] = 1; + remaining -= layer_expert_bytes[(size_t) best_layer]; + ++next[(size_t) best_layer]; + } + + in_out = std::move(tmp); + return true; +} + bool MoeHybridPlacement::build_critical_path_balanced_from_stats( const MoeHybridRoutingStats & stats, const std::vector & layer_expert_bytes, diff --git a/server/src/common/moe_hybrid_placement.h b/server/src/common/moe_hybrid_placement.h index d726b10a5..c237c8527 100644 --- a/server/src/common/moe_hybrid_placement.h +++ b/server/src/common/moe_hybrid_placement.h @@ -74,6 +74,18 @@ struct MoeHybridPlacement { MoeHybridPlacement & out, std::string * err = nullptr); + // Preserve an existing placement and spend any remaining byte budget on + // experts ranked by a second routing profile. This is useful when the + // experts needed to balance a latency-sensitive phase (for example, + // decode) must remain resident while spare capacity is filled for a + // different phase (for example, prefill). + static bool expand_from_stats_with_layer_bytes( + const MoeHybridRoutingStats & stats, + const std::vector & layer_expert_bytes, + uint64_t total_hot_budget_bytes, + MoeHybridPlacement & in_out, + std::string * err = nullptr); + // Distribute main-owner experts to minimize the sum of predicted per-layer // fork times, max(main, peer), rather than merely maximizing aggregate hit // rate. layer_main_fixed_bytes accounts for owner-local work such as the diff --git a/server/src/common/moe_hybrid_storage.cpp b/server/src/common/moe_hybrid_storage.cpp index b1ad20f35..5795d1d43 100644 --- a/server/src/common/moe_hybrid_storage.cpp +++ b/server/src/common/moe_hybrid_storage.cpp @@ -1,11 +1,13 @@ #include "moe_hybrid_storage.h" #include "moe_hybrid_types.h" +#include "heterogeneous_stage_planner.h" #include "ggml-cpu.h" #include "ggml-backend.h" #include "ggml-cuda.h" #include +#include #include #include #include @@ -146,6 +148,223 @@ static ggml_tensor * new_like_with_expert_count(ggml_context * ctx, ggml_tensor return ggml_new_tensor(ctx, src->type, 4, ne); } +struct SharedFfnSplitPolicy { + double peer_fraction = 0.0; + double main_rate = 1.0; + double peer_rate = 1.0; + double main_fixed_work = 0.0; + double peer_fixed_work = 0.0; +}; + +static const char * first_nonempty_env( + const char * primary, const char * legacy = nullptr) { + const char * raw = std::getenv(primary); + if (raw && *raw) return raw; + raw = legacy ? std::getenv(legacy) : nullptr; + return raw && *raw ? raw : nullptr; +} + +static bool parse_positive_double(const char * raw, double & value) { + if (!raw || !*raw) return false; + char * end = nullptr; + const double parsed = std::strtod(raw, &end); + if (end == raw || *end != '\0' || !std::isfinite(parsed) || parsed <= 0.0) { + return false; + } + value = parsed; + return true; +} + +static bool parse_nonnegative_double(const char * raw, double & value) { + if (!raw || !*raw) return false; + char * end = nullptr; + const double parsed = std::strtod(raw, &end); + if (end == raw || *end != '\0' || !std::isfinite(parsed) || parsed < 0.0) { + return false; + } + value = parsed; + return true; +} + +static SharedFfnSplitPolicy resolve_shared_ffn_split_policy( + const MoeHybridConfig & cfg) { + SharedFfnSplitPolicy policy; + policy.peer_fraction = cfg.shared_ffn_peer_fraction; + policy.main_rate = cfg.heterogeneous_main_rate > 0.0f + ? cfg.heterogeneous_main_rate : 1.0; + policy.peer_rate = cfg.heterogeneous_peer_rate > 0.0f + ? cfg.heterogeneous_peer_rate : 1.0; + policy.main_fixed_work = std::max( + 0.0, (double) cfg.heterogeneous_main_fixed_work); + policy.peer_fixed_work = std::max( + 0.0, (double) cfg.heterogeneous_peer_fixed_work); + + if (const char * raw = first_nonempty_env( + "DFLASH_MOE_TP_SHARED_FFN_PEER_FRACTION", + "DFLASH_DS4_TP_SHARED_FFN_PEER_FRACTION")) { + if (std::strcmp(raw, "auto") == 0 || + std::strcmp(raw, "AUTO") == 0) { + policy.peer_fraction = -1.0; + } else { + char * end = nullptr; + const double parsed = std::strtod(raw, &end); + if (end == raw || *end != '\0' || !std::isfinite(parsed) || + parsed <= 0.0 || parsed > 1.0) { + std::fprintf(stderr, + "[hybrid-storage] ignoring invalid shared-FFN peer " + "fraction '%s' (expected auto or a value in (0,1])\n", + raw); + policy.peer_fraction = 0.0; + } else { + policy.peer_fraction = parsed; + } + } + } + + double rate = 0.0; + if (parse_positive_double(first_nonempty_env( + "DFLASH_MOE_TP_MAIN_RATE", "DFLASH_DS4_TP_MAIN_RATE"), rate)) { + policy.main_rate = rate; + } else if (parse_positive_double(first_nonempty_env( + "DFLASH_MOE_TP_MAIN_TO_PEER_RATE", + "DFLASH_DS4_TP_MAIN_TO_PEER_RATE"), rate)) { + policy.main_rate = rate; + policy.peer_rate = 1.0; + } + if (parse_positive_double(first_nonempty_env( + "DFLASH_MOE_TP_PEER_RATE", "DFLASH_DS4_TP_PEER_RATE"), rate)) { + policy.peer_rate = rate; + } + double fixed_work = 0.0; + if (parse_nonnegative_double(first_nonempty_env( + "DFLASH_MOE_TP_MAIN_FIXED_WORK", + "DFLASH_DS4_TP_MAIN_FIXED_WORK"), fixed_work)) { + policy.main_fixed_work = fixed_work; + } + if (parse_nonnegative_double(first_nonempty_env( + "DFLASH_MOE_TP_PEER_FIXED_WORK", + "DFLASH_DS4_TP_PEER_FIXED_WORK"), fixed_work)) { + policy.peer_fixed_work = fixed_work; + } + return policy; +} + +static HeterogeneousStagePlan plan_shared_ffn_peer_shard( + const MoeHybridConfig & cfg, + const MoeLayerDesc & desc, + const SharedFfnSplitPolicy & policy, + bool distinct_gpu_peer) { + if (!distinct_gpu_peer || policy.peer_fraction == 0.0 || + !cfg.materialize_cold_experts || desc.ffn_gate_inp_shexp || + !desc.ffn_gate_shexp || !desc.ffn_up_shexp || + !desc.ffn_down_shexp) { + return {}; + } + + const ggml_tensor * gate = desc.ffn_gate_shexp; + const ggml_tensor * up = desc.ffn_up_shexp; + const ggml_tensor * down = desc.ffn_down_shexp; + const int64_t width = gate->ne[1]; + if (width <= 1 || width > std::numeric_limits::max() || + gate->ne[0] != cfg.n_embd || up->ne[0] != cfg.n_embd || + up->ne[1] != width || down->ne[0] != width || + down->ne[1] != cfg.n_embd || + gate->ne[2] != 1 || gate->ne[3] != 1 || + up->ne[2] != 1 || up->ne[3] != 1 || + down->ne[2] != 1 || down->ne[3] != 1 || + !ggml_is_contiguous(gate) || !ggml_is_contiguous(up) || + !ggml_is_contiguous(down)) { + return {}; + } + + const int alignment = std::max(1, (int) ggml_blck_size(down->type)); + HeterogeneousStagePlan plan = policy.peer_fraction < 0.0 + ? plan_balanced_heterogeneous_stage_width( + (int) width, alignment, policy.main_rate, policy.peer_rate, + policy.main_fixed_work, policy.peer_fixed_work) + : plan_heterogeneous_stage_width( + (int) width, alignment, policy.peer_fraction); + if (!plan.uses_peer() || + gate->nb[1] != ggml_row_size(gate->type, gate->ne[0]) || + up->nb[1] != ggml_row_size(up->type, up->ne[0]) || + down->nb[1] != ggml_row_size(down->type, down->ne[0])) { + return {}; + } + return plan; +} + +static void create_shared_ffn_peer_shard( + ggml_context * ctx, + const MoeLayerDesc & desc, + const HeterogeneousStagePlan & plan, + MoeHybridLayerStorage & dst, + int layer) { + if (!ctx || !plan.uses_peer()) return; + dst.gate_shexp_peer = ggml_new_tensor_2d( + ctx, desc.ffn_gate_shexp->type, + desc.ffn_gate_shexp->ne[0], plan.peer_width); + dst.up_shexp_peer = ggml_new_tensor_2d( + ctx, desc.ffn_up_shexp->type, + desc.ffn_up_shexp->ne[0], plan.peer_width); + dst.down_shexp_peer = ggml_new_tensor_2d( + ctx, desc.ffn_down_shexp->type, + plan.peer_width, desc.ffn_down_shexp->ne[1]); + dst.shared_ffn_main_width = plan.main_width; + dst.shared_ffn_peer_width = plan.peer_width; + ggml_format_name(dst.gate_shexp_peer, "blk.%d.ffn_gate_shexp.peer", layer); + ggml_format_name(dst.up_shexp_peer, "blk.%d.ffn_up_shexp.peer", layer); + ggml_format_name(dst.down_shexp_peer, "blk.%d.ffn_down_shexp.peer", layer); +} + +static bool copy_shared_ffn_peer_shard( + const MoeLayerDesc & desc, + MoeHybridLayerStorage & dst, + std::vector & staging, + std::string * err) { + if (!dst.has_shared_ffn_peer_partition()) return true; + + auto copy_tail_rows = [&](ggml_tensor * source, ggml_tensor * destination, + const char * label) { + const size_t offset = + (size_t) dst.shared_ffn_main_width * source->nb[1]; + const size_t bytes = ggml_nbytes(destination); + if (offset > ggml_nbytes(source) || bytes > ggml_nbytes(source) - offset) { + if (err) *err = std::string("shared-FFN ") + label + + " tail is outside the source tensor"; + return false; + } + staging.resize(bytes); + ggml_backend_tensor_get(source, staging.data(), offset, bytes); + ggml_backend_tensor_set(destination, staging.data(), 0, bytes); + return true; + }; + if (!copy_tail_rows( + desc.ffn_gate_shexp, dst.gate_shexp_peer, "gate") || + !copy_tail_rows( + desc.ffn_up_shexp, dst.up_shexp_peer, "up")) { + return false; + } + + const size_t main_row_bytes = ggml_row_size( + desc.ffn_down_shexp->type, dst.shared_ffn_main_width); + const size_t peer_row_bytes = ggml_row_size( + desc.ffn_down_shexp->type, dst.shared_ffn_peer_width); + const size_t destination_bytes = ggml_nbytes(dst.down_shexp_peer); + if (main_row_bytes + peer_row_bytes != desc.ffn_down_shexp->nb[1] || + peer_row_bytes != dst.down_shexp_peer->nb[1]) { + if (err) *err = "shared-FFN down-projection row packing mismatch"; + return false; + } + staging.resize(destination_bytes); + ggml_backend_tensor_get_2d( + desc.ffn_down_shexp, staging.data(), main_row_bytes, + peer_row_bytes, (size_t) desc.ffn_down_shexp->ne[1], + desc.ffn_down_shexp->nb[1], dst.down_shexp_peer->nb[1]); + ggml_backend_tensor_set( + dst.down_shexp_peer, staging.data(), 0, destination_bytes); + return true; +} + } // namespace MoeHybridStorage::~MoeHybridStorage() { @@ -196,6 +415,11 @@ MoeHybridStorage::~MoeHybridStorage() { layer.up_cold = nullptr; layer.down_cold = nullptr; layer.gate_up_cold = nullptr; + layer.gate_shexp_peer = nullptr; + layer.up_shexp_peer = nullptr; + layer.down_shexp_peer = nullptr; + layer.shared_ffn_main_width = 0; + layer.shared_ffn_peer_width = 0; } if (cpu_backend) { ggml_backend_free(cpu_backend); @@ -260,6 +484,11 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, if (err) *err = "failed to select cold expert backend"; return false; } + const SharedFfnSplitPolicy shared_policy = + resolve_shared_ffn_split_policy(cfg); + out.shared_ffn_peer_fraction = (float) shared_policy.peer_fraction; + int shared_split_layers = 0; + size_t shared_split_bytes = 0; const bool duplicate_hot_on_cold = out.cold_backend_kind == MoeHybridColdBackend::Gpu && duplicate_hot_experts_on_cold_gpu(); @@ -294,6 +523,7 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, dst.hot_local_by_global[(size_t)expert] = (int32_t)i; is_hot[(size_t)expert] = 1; } + dst.decode_hot_local_by_global = dst.hot_local_by_global; for (int expert = 0; expert < cfg.n_expert; ++expert) { if (duplicate_hot_on_cold || !is_hot[(size_t)expert]) { dst.cold_local_by_global[(size_t)expert] = (int32_t)dst.cold_expert_ids.size(); @@ -317,6 +547,11 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, const int cold_count = (int)dst.cold_expert_ids.size(); const int hot_count = (int)dst.hot_expert_ids.size(); + const HeterogeneousStagePlan shared_plan = + plan_shared_ffn_peer_shard( + cfg, desc, shared_policy, + out.cold_backend_kind == MoeHybridColdBackend::Gpu && + out.cold_backend != gpu_backend); // Allocate hot expert tensors on GPU if (hot_count > 0 && cfg.materialize_hot_experts) { @@ -370,9 +605,10 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, } // Allocate cold expert tensors on the selected cold backend. - if (cold_count > 0 && cfg.materialize_cold_experts) { + if ((cold_count > 0 || shared_plan.uses_peer()) && + cfg.materialize_cold_experts) { ggml_init_params ip{}; - ip.mem_size = 16 * ggml_tensor_overhead(); + ip.mem_size = 24 * ggml_tensor_overhead(); ip.mem_buffer = nullptr; ip.no_alloc = true; dst.cold_ctx = ggml_init(ip); @@ -388,12 +624,9 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, dst.up_cold = new_like_with_expert_count(dst.cold_ctx, desc.ffn_up_exps, cold_count); dst.down_cold = new_like_with_expert_count(dst.cold_ctx, desc.ffn_down_exps, cold_count); } + create_shared_ffn_peer_shard( + dst.cold_ctx, desc, shared_plan, dst, il); dst.cold_buf = ggml_backend_alloc_ctx_tensors(dst.cold_ctx, out.cold_backend); - - - - - if (!dst.cold_buf) { if (err) { *err = (out.cold_backend_kind == MoeHybridColdBackend::Gpu) @@ -402,9 +635,11 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, } return false; } + ggml_backend_buffer_set_usage( + dst.cold_buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); std::vector cold_bytes; - if (dst.fused_gate_up) { + if (cold_count > 0 && dst.fused_gate_up) { if (!read_expert_slices(gpu_backend, desc.ffn_gate_up_exps, dst.cold_expert_ids, dst.gate_up_expert_bytes, cold_bytes, err)) return false; @@ -413,7 +648,7 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, dst.down_expert_bytes, cold_bytes, err)) return false; ggml_backend_tensor_set(dst.down_cold, cold_bytes.data(), 0, cold_bytes.size()); - } else { + } else if (cold_count > 0) { if (!read_expert_slices(gpu_backend, desc.ffn_gate_exps, dst.cold_expert_ids, dst.gate_expert_bytes, cold_bytes, err)) return false; @@ -427,9 +662,33 @@ bool build_moe_hybrid_storage(const MoeHybridConfig & cfg, return false; ggml_backend_tensor_set(dst.down_cold, cold_bytes.data(), 0, cold_bytes.size()); } + if (!copy_shared_ffn_peer_shard(desc, dst, cold_bytes, err)) { + return false; + } + if (dst.has_shared_ffn_peer_partition()) { + ++shared_split_layers; + shared_split_bytes += ggml_nbytes(dst.gate_shexp_peer) + + ggml_nbytes(dst.up_shexp_peer) + + ggml_nbytes(dst.down_shexp_peer); + } } } + if (shared_split_layers > 0) { + const auto split_it = std::find_if( + out.layers.begin(), out.layers.end(), + [](const MoeHybridLayerStorage & layer) { + return layer.has_shared_ffn_peer_partition(); + }); + std::fprintf(stderr, + "[hybrid-storage] shared-FFN partition: layers=%d " + "main=%d peer=%d peer_weights=%.1f MiB\n", + shared_split_layers, + split_it != out.layers.end() ? split_it->shared_ffn_main_width : 0, + split_it != out.layers.end() ? split_it->shared_ffn_peer_width : 0, + (double) shared_split_bytes / (1024.0 * 1024.0)); + } + return true; } @@ -472,6 +731,11 @@ bool build_moe_hybrid_storage_from_file( if (err) *err = "failed to select cold expert backend"; return false; } + const SharedFfnSplitPolicy shared_policy = + resolve_shared_ffn_split_policy(cfg); + out.shared_ffn_peer_fraction = (float) shared_policy.peer_fraction; + int shared_split_layers = 0; + size_t shared_split_bytes = 0; const bool duplicate_hot_on_cold = out.cold_backend_kind == MoeHybridColdBackend::Gpu && allocate_cold && duplicate_hot_experts_on_cold_gpu(); @@ -507,6 +771,7 @@ bool build_moe_hybrid_storage_from_file( dst.hot_local_by_global[(size_t)expert] = (int32_t)i; is_hot[(size_t)expert] = 1; } + dst.decode_hot_local_by_global = dst.hot_local_by_global; if (allocate_cold) { for (int expert = 0; expert < cfg.n_expert; ++expert) { if (duplicate_hot_on_cold || !is_hot[(size_t)expert]) { @@ -532,6 +797,12 @@ bool build_moe_hybrid_storage_from_file( const int hot_count = (int)dst.hot_expert_ids.size(); const int cold_count = (int)dst.cold_expert_ids.size(); + const HeterogeneousStagePlan shared_plan = + plan_shared_ffn_peer_shard( + cfg, desc, shared_policy, + allocate_cold && + out.cold_backend_kind == MoeHybridColdBackend::Gpu && + out.cold_backend != gpu_backend); const int spare = (cold_count > 0 && cache_slots > 0) ? std::min(cache_slots, cold_count) : 0; const int hot_alloc = hot_count + spare; @@ -595,9 +866,10 @@ bool build_moe_hybrid_storage_from_file( } // Allocate cold expert tensors on the selected cold backend. - if (allocate_cold && cold_count > 0 && cfg.materialize_cold_experts) { + if (allocate_cold && (cold_count > 0 || shared_plan.uses_peer()) && + cfg.materialize_cold_experts) { ggml_init_params ip{}; - ip.mem_size = 16 * ggml_tensor_overhead(); + ip.mem_size = 24 * ggml_tensor_overhead(); ip.mem_buffer = nullptr; ip.no_alloc = true; dst.cold_ctx = ggml_init(ip); @@ -613,6 +885,8 @@ bool build_moe_hybrid_storage_from_file( dst.up_cold = new_like_with_expert_count(dst.cold_ctx, desc.ffn_up_exps, cold_count); dst.down_cold = new_like_with_expert_count(dst.cold_ctx, desc.ffn_down_exps, cold_count); } + create_shared_ffn_peer_shard( + dst.cold_ctx, desc, shared_plan, dst, il); dst.cold_buf = ggml_backend_alloc_ctx_tensors(dst.cold_ctx, out.cold_backend); if (!dst.cold_buf) { if (err) { @@ -622,9 +896,11 @@ bool build_moe_hybrid_storage_from_file( } return false; } + ggml_backend_buffer_set_usage( + dst.cold_buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); std::vector slice_buf; - if (dst.fused_gate_up) { + if (cold_count > 0 && dst.fused_gate_up) { if (!read_expert_slices_from_mem(fd.gate_up_exps.data, fd.gate_up_exps.size, dst.cold_expert_ids, dst.gate_up_expert_bytes, slice_buf, err)) return false; @@ -633,7 +909,7 @@ bool build_moe_hybrid_storage_from_file( dst.cold_expert_ids, dst.down_expert_bytes, slice_buf, err)) return false; ggml_backend_tensor_set(dst.down_cold, slice_buf.data(), 0, slice_buf.size()); - } else { + } else if (cold_count > 0) { if (!read_expert_slices_from_mem(fd.gate_exps.data, fd.gate_exps.size, dst.cold_expert_ids, dst.gate_expert_bytes, slice_buf, err)) return false; @@ -647,9 +923,33 @@ bool build_moe_hybrid_storage_from_file( return false; ggml_backend_tensor_set(dst.down_cold, slice_buf.data(), 0, slice_buf.size()); } + if (!copy_shared_ffn_peer_shard(desc, dst, slice_buf, err)) { + return false; + } + if (dst.has_shared_ffn_peer_partition()) { + ++shared_split_layers; + shared_split_bytes += ggml_nbytes(dst.gate_shexp_peer) + + ggml_nbytes(dst.up_shexp_peer) + + ggml_nbytes(dst.down_shexp_peer); + } } } + if (shared_split_layers > 0) { + const auto split_it = std::find_if( + out.layers.begin(), out.layers.end(), + [](const MoeHybridLayerStorage & layer) { + return layer.has_shared_ffn_peer_partition(); + }); + std::fprintf(stderr, + "[hybrid-storage] shared-FFN partition: layers=%d " + "main=%d peer=%d peer_weights=%.1f MiB\n", + shared_split_layers, + split_it != out.layers.end() ? split_it->shared_ffn_main_width : 0, + split_it != out.layers.end() ? split_it->shared_ffn_peer_width : 0, + (double) shared_split_bytes / (1024.0 * 1024.0)); + } + return true; } diff --git a/server/src/common/moe_hybrid_storage.h b/server/src/common/moe_hybrid_storage.h index 12c8fb83d..55401b185 100644 --- a/server/src/common/moe_hybrid_storage.h +++ b/server/src/common/moe_hybrid_storage.h @@ -90,9 +90,33 @@ struct MoeHybridLayerStorage { ggml_backend_t cold_backend = nullptr; // Alias: either CPU backend or caller-owned GPU/HIP backend. MoeHybridColdBackend cold_backend_kind = MoeHybridColdBackend::Cpu; + // Peer-owned partition of a shared gate/up/down FFN. For a tensor split, + // main uses zero-copy prefix views and peer stores a compact tail. For a + // whole-stage assignment, main_width is zero and peer stores the complete + // tensors. Both lowerings retain one value per routed owner at the join. + ggml_tensor * gate_shexp_peer = nullptr; + ggml_tensor * up_shexp_peer = nullptr; + ggml_tensor * down_shexp_peer = nullptr; + int shared_ffn_main_width = 0; + int shared_ffn_peer_width = 0; + + bool has_shared_ffn_peer_partition() const { + return gate_shexp_peer && up_shexp_peer && down_shexp_peer && + shared_ffn_main_width >= 0 && shared_ffn_peer_width > 0; + } + + bool has_shared_ffn_peer_shard() const { + return has_shared_ffn_peer_partition() && + shared_ffn_main_width > 0; + } + std::vector hot_expert_ids; std::vector cold_expert_ids; std::vector hot_local_by_global; + // Optional decode ownership is a subset of the physically resident hot + // experts. Prefill can use every resident expert while decode retains a + // separately balanced fork. Empty means identical to hot_local_by_global. + std::vector decode_hot_local_by_global; std::vector cold_local_by_global; // --- Bounded GPU expert cache (laguna) --- @@ -195,6 +219,7 @@ struct MoeHybridStorage { MoeHybridColdBackend cold_backend_kind = MoeHybridColdBackend::Cpu; bool materialized_hot_experts = true; bool materialized_cold_experts = true; + float shared_ffn_peer_fraction = 0.0f; MoeHybridPlacement placement; std::vector layers; diff --git a/server/src/common/moe_hybrid_types.h b/server/src/common/moe_hybrid_types.h index bdb22e47e..6b67c8ece 100644 --- a/server/src/common/moe_hybrid_types.h +++ b/server/src/common/moe_hybrid_types.h @@ -38,6 +38,22 @@ struct MoeHybridConfig { bool materialize_hot_experts = true; bool materialize_cold_experts = true; + // Optional tensor-parallel shard of the shared FFN intermediate width. + // 0 disables it; values in (0,1) request an explicit tensor partition; + // 1 assigns the complete stage to peer; a negative value asks the generic + // planner to balance from owner rates. + // The storage builder also accepts DFLASH_MOE_TP_SHARED_FFN_PEER_FRACTION + // (and the legacy DS4 alias), allowing every model adapter that uses the + // common hybrid path to opt in without architecture-specific graph code. + float shared_ffn_peer_fraction = 0.0f; + float heterogeneous_main_rate = 1.0f; + float heterogeneous_peer_rate = 1.0f; + // Work already assigned to each owner when the splittable stage begins. + // Values use the same arbitrary work unit as the stage width; adapters can + // therefore account for routed experts or other concurrent dense stages. + float heterogeneous_main_fixed_work = 0.0f; + float heterogeneous_peer_fixed_work = 0.0f; + // When true, MMQ mul_mat_id works correctly with reduced hot stacks // (n_hot < n_expert). Safe on sm_80+ (Ampere/Ada/Hopper/Blackwell). // On sm_75 (Turing) and gfx1151, the kernel has illegal memory accesses diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 3aafad282..e56d40a78 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -66,16 +66,18 @@ static void configure_dspark_mmvq_defaults(int gpu) { return; } - // q=5 is an explicit AMD-only experiment and needs the plain quantized + // Wide verification is an explicit AMD-only experiment and needs the plain quantized // verifier matmuls to stay on MMVQ. The process-wide crossover applies to // both owners in the heterogeneous graph, so set it before inspecting the // target device (which is gfx1201 in the R9700 + gfx1151 launch). - if (env_flag_enabled("DFLASH_DS4_Q5_VERIFY")) { + const bool q6_verify = env_flag_enabled("DFLASH_DS4_Q6_VERIFY"); + if (env_flag_enabled("DFLASH_DS4_Q5_VERIFY") || q6_verify) { if (std::getenv("LUCE_MMVQ_MAX_NCOLS") == nullptr && - ::setenv("LUCE_MMVQ_MAX_NCOLS", "5", 0) == 0) { + ::setenv("LUCE_MMVQ_MAX_NCOLS", q6_verify ? "6" : "5", 0) == 0) { std::fprintf(stderr, - "[deepseek4] AMD DSpark q5: defaulting " - "LUCE_MMVQ_MAX_NCOLS=5\n"); + "[deepseek4] AMD DSpark wide verify: defaulting " + "LUCE_MMVQ_MAX_NCOLS=%d\n", + q6_verify ? 6 : 5); } cudaDeviceProp prop{}; @@ -876,6 +878,21 @@ int DeepSeek4Backend::capture_safe_prefill_tokens( return safe_tokens; } +bool DeepSeek4Backend::supports_batched_spec_feature_capture( + bool hybrid, + PrefillAttentionMode mode, + int n_tokens) { + if (mode == PrefillAttentionMode::Exact || n_tokens <= 4 || + n_tokens > DS4_MAX_LAYER_MAJOR_PREFILL_TOKENS) { + return false; + } + // The monolithic layer-major path reads only the requested token range. + // Sparse heterogeneous prefill returns every requested capture row; the + // caller then retains the final/snapshot window. Other hybrid modes are + // tokenwise and must still split at capture boundaries. + return !hybrid || mode == PrefillAttentionMode::Sparse; +} + bool DeepSeek4Backend::init() { // The shared MMVQ/MMQ crossover defaults to q=3 for NVIDIA. On gfx1151, // DSpark q=4 is faster through MMVQ. Keep AR and other devices unchanged, @@ -1030,7 +1047,9 @@ bool DeepSeek4Backend::init_moe_tensor_parallel() { bool DeepSeek4Backend::compute_uniform_hybrid_placement(const DeepSeek4Weights & w, int max_ctx, MoeHybridPlacement & out, + MoeHybridPlacement * decode_out, std::string * err) const { + if (decode_out) *decode_out = {}; Ds4HybridBudgetInfo budget; if (!compute_ds4_hybrid_budget_info(w, backend_, max_ctx, budget, err)) { return false; @@ -1047,6 +1066,10 @@ bool DeepSeek4Backend::compute_uniform_hybrid_placement(const DeepSeek4Weights & bool concentrated = false; int retained_local = 0; const char * profile_path = std::getenv("DFLASH_DS4_HOTNESS_CSV"); + const char * decode_profile_path = + std::getenv("DFLASH_DS4_DECODE_HOTNESS_CSV"); + const bool phase_aware_placement = decode_profile_path && + *decode_profile_path; const bool critical_path_placement = !tp.all_on_secondary && !concentrate_requested && env_flag_enabled("DFLASH_DS4_TP_CRITICAL_PATH_PLACEMENT"); @@ -1068,8 +1091,11 @@ bool DeepSeek4Backend::compute_uniform_hybrid_placement(const DeepSeek4Weights & } return false; } + const char * balance_profile_path = phase_aware_placement + ? decode_profile_path : profile_path; MoeHybridRoutingStats stats; - if (!MoeHybridRoutingStats::load_csv(profile_path, stats, err)) { + if (!MoeHybridRoutingStats::load_csv( + balance_profile_path, stats, err)) { return false; } if (stats.n_layer != w.n_layer || stats.n_expert != w.n_expert) { @@ -1115,6 +1141,39 @@ bool DeepSeek4Backend::compute_uniform_hybrid_placement(const DeepSeek4Weights & return false; } + if (phase_aware_placement) { + if (!decode_out) { + if (err) *err = "phase-aware placement requires decode output"; + return false; + } + *decode_out = out; + MoeHybridRoutingStats residency_stats; + if (!MoeHybridRoutingStats::load_csv( + profile_path, residency_stats, err)) { + return false; + } + if (residency_stats.n_layer != w.n_layer || + residency_stats.n_expert != w.n_expert || + residency_stats.n_expert_used != w.n_expert_used) { + if (err) { + *err = "residency routing profile shape does not match " + "DeepSeek V4 target"; + } + return false; + } + if (!MoeHybridPlacement::expand_from_stats_with_layer_bytes( + residency_stats, budget.mem.layer_expert_bytes, + budget.expert_budget, out, err)) { + return false; + } + std::fprintf(stderr, + "[deepseek4] hybrid phase-aware placement: " + "decode_profile=%s resident_profile=%s " + "decode=%d resident=%d\n", + decode_profile_path, profile_path, + decode_out->total_hot, out.total_hot); + } + const auto [min_hot, max_hot] = std::minmax_element( out.hot_counts.begin(), out.hot_counts.end()); const double mean_hot = out.hot_counts.empty() ? 0.0 @@ -1123,7 +1182,7 @@ bool DeepSeek4Backend::compute_uniform_hybrid_placement(const DeepSeek4Weights & "[deepseek4] hybrid critical-path placement: " "profile=%s active=%d main/peer=%.3f " "hot/layer=%.1f [%d,%d]\n", - profile_path, active_experts, main_to_peer_rate, + balance_profile_path, active_experts, main_to_peer_rate, mean_hot, min_hot != out.hot_counts.end() ? *min_hot : 0, max_hot != out.hot_counts.end() ? *max_hot : 0); @@ -1209,7 +1268,8 @@ bool DeepSeek4Backend::init_hybrid_model() { std::string err; const int max_ctx = cfg_.max_ctx > 0 ? cfg_.max_ctx : 8192; - if (!compute_uniform_hybrid_placement(w_, max_ctx, moe_placement_, &err)) { + if (!compute_uniform_hybrid_placement( + w_, max_ctx, moe_placement_, &moe_decode_placement_, &err)) { std::fprintf(stderr, "[deepseek4] failed to compute hybrid placement: %s\n", err.c_str()); return false; } @@ -1283,6 +1343,36 @@ bool DeepSeek4Backend::init_hybrid_model() { return false; } + // The physical placement is shared by both phases. Decode may own only a + // subset so its fast main branch does not outrun and then wait on the peer; + // prefill continues to consume every resident expert. + if (!moe_decode_placement_.empty()) { + if (!moe_decode_placement_.matches( + w_.n_layer, w_.n_expert, w_.n_expert_used)) { + std::fprintf(stderr, + "[deepseek4] decode placement dimensions are invalid\n"); + return false; + } + for (int il = 0; il < w_.n_layer; ++il) { + MoeHybridLayerStorage & layer = hybrid->layers[(size_t) il]; + layer.decode_hot_local_by_global.assign( + (size_t) w_.n_expert, -1); + for (int32_t expert : + moe_decode_placement_.hot_expert_ids[(size_t) il]) { + if (expert < 0 || expert >= w_.n_expert || + layer.hot_local_by_global[(size_t) expert] < 0) { + std::fprintf(stderr, + "[deepseek4] decode owner expert %d in layer " + "%d is not resident\n", + (int) expert, il); + return false; + } + layer.decode_hot_local_by_global[(size_t) expert] = + layer.hot_local_by_global[(size_t) expert]; + } + } + } + if (hybrid->has_mmap() && !hybrid->materialized_cold_experts) { size_t max_expert_bytes = 0; for (const auto & layer : hybrid->layers) { @@ -1348,6 +1438,7 @@ bool DeepSeek4Backend::park(ParkTarget target) { expert_backend_ = nullptr; } moe_placement_ = {}; + moe_decode_placement_ = {}; free_deepseek4_weights(w_); parked_ = true; if (spec_drafter_) { @@ -1375,6 +1466,7 @@ bool DeepSeek4Backend::unpark(ParkTarget target) { expert_backend_ = nullptr; } moe_placement_ = {}; + moe_decode_placement_ = {}; return false; } @@ -1392,6 +1484,7 @@ bool DeepSeek4Backend::unpark(ParkTarget target) { expert_backend_ = nullptr; } moe_placement_ = {}; + moe_decode_placement_ = {}; return false; } @@ -1407,6 +1500,7 @@ bool DeepSeek4Backend::unpark(ParkTarget target) { expert_backend_ = nullptr; } moe_placement_ = {}; + moe_decode_placement_ = {}; return false; } @@ -1419,6 +1513,7 @@ bool DeepSeek4Backend::unpark(ParkTarget target) { stream_engine_.destroy(); moe_hybrid_.reset(); moe_placement_ = {}; + moe_decode_placement_ = {}; return false; } @@ -1572,10 +1667,8 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, } if (spec_enabled_ && spec_drafter_) { const bool batch_final_capture = - !w_.moe_hybrid && - cache_.prefill_mode != PrefillAttentionMode::Exact && - n_tok > 4 && - n_tok <= DS4_MAX_LAYER_MAJOR_PREFILL_TOKENS; + supports_batched_spec_feature_capture( + w_.moe_hybrid, cache_.prefill_mode, n_tok); n_tok = capture_safe_prefill_tokens( i, n_tok, spec_final_from, batch_final_capture, save_snapshot && !snapshot_saved, @@ -2103,6 +2196,7 @@ void DeepSeek4Backend::shutdown() { routing_stats_.reset(); routing_stats_out_path_.clear(); moe_placement_ = {}; + moe_decode_placement_ = {}; free_deepseek4_weights(w_); if (snap_backend_) { ggml_backend_free(snap_backend_); snap_backend_ = nullptr; } if (backend_) { ggml_backend_free(backend_); backend_ = nullptr; } diff --git a/server/src/deepseek4/deepseek4_backend.h b/server/src/deepseek4/deepseek4_backend.h index 98e239e42..a9c58a18a 100644 --- a/server/src/deepseek4/deepseek4_backend.h +++ b/server/src/deepseek4/deepseek4_backend.h @@ -114,8 +114,14 @@ class DeepSeek4Backend : public ModelBackend { void release_spec_drafter(bool mark_parked); void keep_spec_feature_tail(std::vector & features, size_t max_rows) const; + // True when a wide prefill path returns per-token DSpark features and the + // caller can retain only the requested capture window without splitting. + static bool supports_batched_spec_feature_capture( + bool hybrid, + PrefillAttentionMode mode, + int n_tokens); // Limit a prefill batch to a region with a uniform DSpark capture policy. - // Layer-major prefill can capture a subrange without splitting the final + // Wide GPU paths can capture a subrange without splitting the final // feature window; other paths still stop exactly at capture boundaries. static int capture_safe_prefill_tokens(int token_offset, int requested_tokens, @@ -153,11 +159,13 @@ class DeepSeek4Backend : public ModelBackend { bool compute_uniform_hybrid_placement(const DeepSeek4Weights & w, int max_ctx, MoeHybridPlacement & out, + MoeHybridPlacement * decode_out, std::string * err) const; void maybe_save_routing_stats(); std::shared_ptr moe_hybrid_; MoeHybridPlacement moe_placement_; + MoeHybridPlacement moe_decode_placement_; MoeHybridStreamEngine stream_engine_; MoeExpertComputeRuntime expert_runtime_; std::shared_ptr routing_stats_; diff --git a/server/src/deepseek4/deepseek4_dspark.h b/server/src/deepseek4/deepseek4_dspark.h index 0e373aca4..628a464ee 100644 --- a/server/src/deepseek4/deepseek4_dspark.h +++ b/server/src/deepseek4/deepseek4_dspark.h @@ -179,7 +179,7 @@ bool deepseek4_dspark_verify_forward(ggml_backend_t backend, // the physical SWA rows they overwrote after the ring wraps; otherwise a later // causal verify reads rejected-token KV as if it were older committed history. // This remains much smaller than a full target-cache snapshot because the -// verifier width is bounded by the DSpark block (currently q <= 5). +// verifier width is bounded by the DSpark block (currently q <= 6). struct DeepSeek4SpecRollback { int raw_pos = 0; int raw_count = 0; diff --git a/server/src/deepseek4/deepseek4_dspark_spec.cpp b/server/src/deepseek4/deepseek4_dspark_spec.cpp index 06a86f7b1..d4e409fda 100644 --- a/server/src/deepseek4/deepseek4_dspark_spec.cpp +++ b/server/src/deepseek4/deepseek4_dspark_spec.cpp @@ -298,7 +298,9 @@ bool init_pinned_rollback(const DeepSeek4Cache & cache, DeepSeek4SpecRollback & s.pinned_idx_sc, prev_half_bytes(lc.indexer_compressor.state_score), total); s.raw_row_bytes = lc.raw_kv ? ggml_row_size(lc.raw_kv->type, lc.raw_kv->ne[0]) : 0; - assign_pinned_span(s.pinned_raw_rows, s.raw_row_bytes * 5, total); + // The DSpark artifact exposes five proposal rows, so the widest + // verifier batch is seed + five candidates. + assign_pinned_span(s.pinned_raw_rows, s.raw_row_bytes * 6, total); } assign_pinned_span( rb.pinned_hc, cache.hc_state ? ggml_nbytes(cache.hc_state) : 0, total); @@ -365,7 +367,7 @@ void spec_rollback_save(const DeepSeek4Cache & cache, DeepSeek4SpecRollback & rb ggml_backend_t backend, bool async_copy, bool pinned_copy, int raw_pos, int raw_count) { rb.raw_pos = raw_pos; - rb.raw_count = std::clamp(raw_count, 0, 5); + rb.raw_count = std::clamp(raw_count, 0, 6); rb.layers.resize(cache.layers.size()); if (async_copy || pinned_copy) { rb.async_backend = backend; @@ -666,7 +668,11 @@ bool run_deepseek4_dspark_spec_decode( spec_env_flag("DFLASH_DS4_SEQ_VERIFY"); const bool async_rollback = spec_env_flag("DFLASH_DS4_ASYNC_ROLLBACK"); const bool pinned_rollback = spec_env_flag("DFLASH_DS4_PINNED_ROLLBACK"); - const bool q5_verify = spec_env_flag("DFLASH_DS4_Q5_VERIFY") && block >= 4; + const bool q6_verify = + spec_env_flag("DFLASH_DS4_Q6_VERIFY") && block >= 5; + const bool q5_verify = + (spec_env_flag("DFLASH_DS4_Q5_VERIFY") || q6_verify) && block >= 4; + const bool wide_verify = q5_verify || q6_verify; const bool draft_overlap_probe = spec_env_flag("DFLASH_DS4_DRAFT_OVERLAP_PROBE"); const bool draft_overlap_reuse_context = @@ -710,10 +716,11 @@ bool run_deepseek4_dspark_spec_decode( double ewma_accept = 1.5; // The conservative fast path remains capped at the compression ratio. - // The explicit q5 path handles the second ratio-4 boundary in-graph and - // restores/replays only a rejected q5 prefix, avoiding full snapshots on - // the overwhelmingly common all-accepted path. - const int fast_cap = q5_verify ? block + 1 : 4; + // The explicit wide path handles a second ratio-4 boundary in-graph and + // restores/replays only a rejected prefix, avoiding full snapshots on the + // overwhelmingly common all-accepted path. + const int fast_cap = std::min( + block + 1, q6_verify ? 6 : (q5_verify ? 5 : 4)); int q_cap = full_snap ? block + 1 : fast_cap; if (const char * qs = std::getenv("DFLASH_DS4_SPEC_Q")) { const int v = std::atoi(qs); @@ -828,8 +835,8 @@ bool run_deepseek4_dspark_spec_decode( } } tm_draft += spec_ms_since(t0); - if (q5_verify && steps == 0) { - std::fprintf(stderr, "[ds4-q5] draft-ready block=%d hidden=%zu\n", + if (wide_verify && steps == 0) { + std::fprintf(stderr, "[ds4-wide] draft-ready block=%d hidden=%zu\n", block, local_hidden.size()); } @@ -862,8 +869,14 @@ bool run_deepseek4_dspark_spec_decode( return v && *v && *v != '0'; }(); int q_step_cap = (seq_verify_mode || fused_verify_mode) - ? std::min(q_cap, q5_verify ? 5 : 4) + ? std::min(q_cap, q6_verify ? 6 : (q5_verify ? 5 : 4)) : std::min(q_cap, 4 - (pos & 3)); + // Starting q6 on the final ratio-4 position would flush twice and + // then overwrite the first row of the next window before its second + // pool. The q5 shape ends exactly at that second boundary. + if (q6_verify && (pos & 3) == 3) { + q_step_cap = std::min(q_step_cap, 5); + } if (adaptive_width && !use_confidence_width && !seq_verify_mode) { const int w_cap = (int) ewma_accept + 2; if (w_cap < q_step_cap) q_step_cap = w_cap; @@ -927,8 +940,8 @@ bool run_deepseek4_dspark_spec_decode( if ((int) draft_tok.size() > q_step_cap) draft_tok.resize(q_step_cap); const int q = (int) draft_tok.size(); // seed + candidates tm_head += spec_ms_since(t0); - if (q5_verify && steps == 0) { - std::fprintf(stderr, "[ds4-q5] head-ready q=%d\n", q); + if (wide_verify && steps == 0) { + std::fprintf(stderr, "[ds4-wide] head-ready q=%d\n", q); } if (debug) { @@ -971,8 +984,8 @@ bool run_deepseek4_dspark_spec_decode( pos, q); } tm_save += spec_ms_since(t0); - if (q5_verify && steps == 0) { - std::fprintf(stderr, "[ds4-q5] rollback-ready rows=%d\n", + if (wide_verify && steps == 0) { + std::fprintf(stderr, "[ds4-wide] rollback-ready rows=%d\n", rollback.raw_count); } @@ -983,8 +996,8 @@ bool run_deepseek4_dspark_spec_decode( // ── ONE batched verify (writes cache + captures features for all q) ── t0 = SpecClock::now(); int verify_last = -1; - if (q5_verify && steps == 0) { - std::fprintf(stderr, "[ds4-q5] verify-begin q=%d pos=%d\n", q, pos); + if (wide_verify && steps == 0) { + std::fprintf(stderr, "[ds4-wide] verify-begin q=%d pos=%d\n", q, pos); } const bool verify_ok = target.verify_batch(draft_tok, pos, verify_last, &tgt_am); @@ -1054,9 +1067,10 @@ bool run_deepseek4_dspark_spec_decode( break; } } else if (!full_snap && accept < q && q > 4) { - // A rejected q5 may have crossed two ratio-4 boundaries. Restore - // the compact pre-verify state and replay only the accepted prefix - // (at most q4), which is exact and rare at high acceptance. + // A rejected wide verify may have crossed two ratio-4 boundaries. + // Restore the compact pre-verify state and replay only the + // accepted prefix (at most q5), which is exact and rare at high + // acceptance. spec_rollback_apply( rollback, target_w, target_cache, pos, true, backend, async_rollback || pinned_rollback, @@ -1068,7 +1082,7 @@ bool run_deepseek4_dspark_spec_decode( int replay_last = -1; std::vector replay_am; if (!target.verify_batch(kv_toks, pos, replay_last, &replay_am)) { - std::fprintf(stderr, "[ds4-spec] q5 rollback replay failed\n"); + std::fprintf(stderr, "[ds4-spec] wide rollback replay failed\n"); ok = false; break; } diff --git a/server/src/deepseek4/deepseek4_fused_verify.inc b/server/src/deepseek4/deepseek4_fused_verify.inc index 8cd1655ef..ec16a459b 100644 --- a/server/src/deepseek4/deepseek4_fused_verify.inc +++ b/server/src/deepseek4/deepseek4_fused_verify.inc @@ -160,26 +160,40 @@ static void ds4_fused_verify_refresh_hybrid_luts( MoeHybridStorage * hybrid, DeepSeek4FusedDecodeGraph & fg) { if (!hybrid) return; + const bool dynamic_route_balance = + ds4_env_flag("DFLASH_DS4_TP_DYNAMIC_ROUTE_BALANCE") || + ds4_env_flag("DFLASH_MOE_TP_DYNAMIC_ROUTE_BALANCE"); const int32_t invalid_route = ds4_env_flag("DFLASH_DS4_TP_MASKED_ROUTES") ? -1 : 0; for (size_t hi = 0; hi < fg.hybrid_inputs.size(); ++hi) { const int il = (int) hi; const MoeHybridLayerStorage & layer = hybrid->layers[(size_t) il]; const MoeHybridGraphInputs & inputs = fg.hybrid_inputs[hi]; + const std::vector & decode_hot = + dynamic_route_balance || + layer.decode_hot_local_by_global.empty() + ? layer.hot_local_by_global + : layer.decode_hot_local_by_global; std::vector hot_lut((size_t) w.n_expert, invalid_route); std::vector cold_lut((size_t) w.n_expert, invalid_route); std::vector hot_valid((size_t) w.n_expert, 0.0f); std::vector cold_valid((size_t) w.n_expert, 0.0f); for (int ie = 0; ie < w.n_expert; ++ie) { - const int32_t hot = layer.hot_local_by_global[(size_t) ie]; + const int32_t hot = decode_hot[(size_t) ie]; const int32_t cold = layer.cold_local_by_global[(size_t) ie]; if (hot >= 0) { hot_lut[(size_t) ie] = hot; hot_valid[(size_t) ie] = 1.0f; } - if (cold >= 0 && hot < 0) { + if (cold >= 0) { cold_lut[(size_t) ie] = cold; - cold_valid[(size_t) ie] = 1.0f; + // In dynamic mode this LUT is the main-residency candidate + // set, copied independently to peer. The peer selector keeps + // its exact complement after the per-token main cap. Static + // placement retains the ordinary cold-validity mask. + cold_valid[(size_t) ie] = dynamic_route_balance + ? (hot >= 0 ? 1.0f : 0.0f) + : (hot < 0 ? 1.0f : 0.0f); } } ds4_fv_set_repeated_rows(inputs.hot_local_lut, hot_lut); @@ -253,14 +267,18 @@ static void ds4_fused_consume_route_diagnostics( route.layer < (int) hybrid->layers.size()) { const MoeHybridLayerStorage & layer = hybrid->layers[(size_t) route.layer]; + const std::vector & decode_hot = + layer.decode_hot_local_by_global.empty() + ? layer.hot_local_by_global + : layer.decode_hot_local_by_global; for (int i = 0; i < route.width; ++i) { const int32_t id = token_ids[i]; if (token_weights[i] == 0.0f || id < 0 || - id >= (int32_t) layer.hot_local_by_global.size()) { + id >= (int32_t) decode_hot.size()) { continue; } ++cache_total; - if (layer.hot_local_by_global[(size_t) id] >= 0) { + if (decode_hot[(size_t) id] >= 0) { ++cache_hot; } } @@ -293,7 +311,8 @@ static size_t ds4_fused_verify_hybrid_slot_limit() { // model-qualified at 28.5 GiB peak on the 31.9 GiB R9700. q<=4 keeps // the conservative two-slot default. const long default_slots = - ds4_env_flag("DFLASH_DS4_Q5_VERIFY") ? 9 : 2; + (ds4_env_flag("DFLASH_DS4_Q5_VERIFY") || + ds4_env_flag("DFLASH_DS4_Q6_VERIFY")) ? 9 : 2; const long requested = raw ? std::strtol(raw, nullptr, 10) : default_slots; return (size_t) std::max( diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index 3fa4e4e9a..19ce98f6c 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -6747,11 +6747,15 @@ bool deepseek4_step_layer_range( moe_hybrid->materialized_cold_experts && moe_hybrid->cold_backend_kind == MoeHybridColdBackend::Gpu && moe_hybrid->cold_backend && moe_hybrid->cold_backend != backend; - const bool q5_verify_candidate = - n_tokens == 5 && ds4_env_flag("DFLASH_DS4_Q5_VERIFY"); + const bool q6_verify_enabled = + ds4_env_flag("DFLASH_DS4_Q6_VERIFY"); + const bool wide_verify_candidate = + (n_tokens == 5 && + (ds4_env_flag("DFLASH_DS4_Q5_VERIFY") || q6_verify_enabled)) || + (n_tokens == 6 && q6_verify_enabled); const bool fused_verify_candidate = (!moe_hybrid || fused_hybrid_ready) && - n_tokens >= 2 && (n_tokens <= 4 || q5_verify_candidate) && verify_hooks && + n_tokens >= 2 && (n_tokens <= 4 || wide_verify_candidate) && verify_hooks && layer_begin == 0 && is_last_shard && out_logits && ds4_backend_is_gpu(backend) && ds4_fused_verify_enabled(); const bool heterogeneous_sparse_prefill = @@ -6987,7 +6991,7 @@ bool deepseek4_step_layer_range( ? &fused_hybrid_decode_hooks : verify_hooks; if ((!moe_hybrid || fused_hybrid_ready) && ((n_tokens >= 2 && - (n_tokens <= 4 || q5_verify_candidate) && verify_hooks) || + (n_tokens <= 4 || wide_verify_candidate) && verify_hooks) || fused_hybrid_decode) && layer_begin == 0 && is_last_shard && out_logits && ds4_backend_is_gpu(backend) && ds4_fused_verify_enabled()) { diff --git a/server/test/test_moe_hybrid_storage.cpp b/server/test/test_moe_hybrid_storage.cpp index b09948e3d..deea21b2a 100644 --- a/server/test/test_moe_hybrid_storage.cpp +++ b/server/test/test_moe_hybrid_storage.cpp @@ -1,4 +1,5 @@ #include "CppUnitTestFramework.hpp" +#include "../src/common/heterogeneous_stage_planner.h" #include "../src/common/moe_hybrid_storage.h" #include @@ -35,3 +36,72 @@ TEST_CASE(MoeHybridStorageFixture, expert_residency_tracks_model_sized_expert_se REQUIRE(!storage.is_expert_hot(256)); REQUIRE(!storage.all_routed_are_hot(all_hot.data(), (int)all_hot.size())); } + +TEST_CASE(MoeHybridStorageFixture, heterogeneous_stage_split_is_aligned_and_complete) { + const HeterogeneousStagePlan plan = + plan_heterogeneous_stage_width(2048, 32, 0.23); + + REQUIRE(plan.split()); + REQUIRE(plan.main_width == 1568); + REQUIRE(plan.peer_width == 480); + REQUIRE(plan.main_width + plan.peer_width == 2048); + REQUIRE(plan.main_width % 32 == 0); + REQUIRE(plan.peer_width % 32 == 0); +} + +TEST_CASE(MoeHybridStorageFixture, heterogeneous_stage_split_rejects_unsafe_shapes) { + const HeterogeneousStagePlan unaligned = + plan_heterogeneous_stage_width(2050, 32, 0.25); + const HeterogeneousStagePlan too_narrow = + plan_heterogeneous_stage_width(32, 32, 0.5); + const HeterogeneousStagePlan empty_balanced = + plan_balanced_heterogeneous_stage_width(0, 32, 3.0, 1.0); + + REQUIRE(!unaligned.split()); + REQUIRE(unaligned.main_width == 2050); + REQUIRE(!too_narrow.split()); + REQUIRE(too_narrow.main_width == 32); + REQUIRE(!empty_balanced.valid()); +} + +TEST_CASE(MoeHybridStorageFixture, heterogeneous_stage_split_keeps_both_owners_for_interior_fraction) { + const HeterogeneousStagePlan small_peer = + plan_heterogeneous_stage_width(2048, 32, 0.001); + const HeterogeneousStagePlan small_main = + plan_heterogeneous_stage_width(2048, 32, 0.999); + + REQUIRE(small_peer.split()); + REQUIRE(small_peer.peer_width == 32); + REQUIRE(small_main.split()); + REQUIRE(small_main.main_width == 32); +} + +TEST_CASE(MoeHybridStorageFixture, heterogeneous_stage_partition_supports_whole_stage_ownership) { + const HeterogeneousStagePlan main = + plan_heterogeneous_stage_width(2048, 32, 0.0); + const HeterogeneousStagePlan peer = + plan_heterogeneous_stage_width(2048, 32, 1.0); + + REQUIRE(main.valid()); + REQUIRE(!main.uses_peer()); + REQUIRE(main.main_width == 2048); + REQUIRE(peer.valid()); + REQUIRE(peer.uses_peer()); + REQUIRE(!peer.split()); + REQUIRE(peer.main_width == 0); + REQUIRE(peer.peer_width == 2048); +} + +TEST_CASE(MoeHybridStorageFixture, heterogeneous_stage_balance_accounts_for_owner_rates_and_fixed_work) { + const HeterogeneousStagePlan rate_only = + plan_balanced_heterogeneous_stage_width( + 2048, 32, 3.0, 1.0); + const HeterogeneousStagePlan main_already_busy = + plan_balanced_heterogeneous_stage_width( + 2048, 32, 3.0, 1.0, 128.0, 0.0); + + REQUIRE(rate_only.split()); + REQUIRE(rate_only.peer_width == 512); + REQUIRE(main_already_busy.split()); + REQUIRE(main_already_busy.peer_width == 544); +} diff --git a/server/test/test_qwen35moe_expert_placement.cpp b/server/test/test_qwen35moe_expert_placement.cpp index e0ac3f0b7..d44981a27 100644 --- a/server/test/test_qwen35moe_expert_placement.cpp +++ b/server/test/test_qwen35moe_expert_placement.cpp @@ -80,6 +80,34 @@ TEST_CASE(Qwen35MoeExpertPlacementFixture, moe_expert_placement_suite) { REQUIRE(balanced.is_hot(0, 2)); REQUIRE(balanced.is_hot(1, 0)); + // A phase-specific decode placement remains intact while a second profile + // spends otherwise-unused bytes on prefill residency. + MoeHybridRoutingStats residency_stats; + residency_stats.n_layer = 2; + residency_stats.n_expert = 4; + residency_stats.n_expert_used = 2; + residency_stats.counts = { + 1, 2, 3, 500, + 1, 300, 400, 1000, + }; + residency_stats.layer_totals = {506, 1701}; + MoeHybridPlacement expanded = balanced; + REQUIRE(MoeHybridPlacement::expand_from_stats_with_layer_bytes( + residency_stats, {100, 100}, 600, expanded, &err)); + REQUIRE(expanded.total_hot == 6); + REQUIRE(expanded.hot_counts == std::vector({4, 2})); + REQUIRE(expanded.is_hot(0, 0)); + REQUIRE(expanded.is_hot(0, 1)); + REQUIRE(expanded.is_hot(0, 2)); + REQUIRE(expanded.is_hot(0, 3)); + REQUIRE(expanded.is_hot(1, 0)); + REQUIRE(expanded.is_hot(1, 3)); + REQUIRE(!expanded.is_hot(1, 2)); + + MoeHybridPlacement over_budget = balanced; + REQUIRE(!MoeHybridPlacement::expand_from_stats_with_layer_bytes( + residency_stats, {100, 100}, 300, over_budget, &err)); + balance_cfg.main_to_peer_rate = 0.0; REQUIRE(!MoeHybridPlacement::build_critical_path_balanced_from_stats( balance_stats, {100, 100}, {100, 100}, 600, diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index 179773775..20d22a77d 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1727,8 +1727,8 @@ static void test_dspark_raw_ring_rollback_after_wrap(ggml_backend_t backend) { layer.n_comp = 2; layer.n_index_comp = 2; DeepSeek4SpecRollback rollback; - deepseek4_spec_rollback_save(cache, rollback, 10, 5); - TEST_ASSERT(rollback.raw_count == 5); + deepseek4_spec_rollback_save(cache, rollback, 10, 6); + TEST_ASSERT(rollback.raw_count == 6); auto overwrite_row = [&](int absolute_pos, uint8_t value) { const int row = absolute_pos % weights.n_swa; @@ -1742,7 +1742,7 @@ static void test_dspark_raw_ring_rollback_after_wrap(ggml_backend_t backend) { expected.begin() + (size_t) row * layer.raw_kv->nb[1] + row_bytes, value); }; - for (int t = 0; t < 5; ++t) { + for (int t = 0; t < 6; ++t) { overwrite_row(10 + t, (uint8_t) (0xa0 + t)); } @@ -2018,8 +2018,23 @@ static void test_dspark_prefill_capture_boundaries() { std::fprintf(stderr, " test_dspark_prefill_capture_boundaries ..."); using Backend = DeepSeek4Backend; - // Layer-major prefill captures only the requested tail from a wide graph, - // while generic paths still stop exactly at the final feature window. + // Both monolithic layer-major and sparse heterogeneous prefill return the + // per-token capture rows needed to retain a tail from one wide graph. + TEST_ASSERT(Backend::supports_batched_spec_feature_capture( + false, PrefillAttentionMode::Sparse, 2048)); + TEST_ASSERT(Backend::supports_batched_spec_feature_capture( + true, PrefillAttentionMode::Sparse, 2048)); + TEST_ASSERT(!Backend::supports_batched_spec_feature_capture( + true, PrefillAttentionMode::Dense, 2048)); + TEST_ASSERT(!Backend::supports_batched_spec_feature_capture( + true, PrefillAttentionMode::Exact, 2048)); + TEST_ASSERT(!Backend::supports_batched_spec_feature_capture( + true, PrefillAttentionMode::Sparse, 4)); + TEST_ASSERT(!Backend::supports_batched_spec_feature_capture( + true, PrefillAttentionMode::Sparse, + DS4_MAX_LAYER_MAJOR_PREFILL_TOKENS + 1)); + + // Generic paths still stop exactly at the final feature window. TEST_ASSERT(Backend::capture_safe_prefill_tokens( 0, 2048, 1920, true, false, 0, 0) == 2048); TEST_ASSERT(Backend::capture_safe_prefill_tokens( @@ -3279,6 +3294,88 @@ static void test_hc_post_strided_split_gpu() { std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } +static void test_moe_id_alignment_q5_gpu() { + std::fprintf(stderr, " test_moe_id_alignment_q5_gpu ..."); + ggml_backend_t backend = ggml_backend_cuda_init(0); + if (!backend) { + std::fprintf(stderr, " skipped (no GPU backend)\n"); + return; + } + + constexpr int n_routes = 4; + constexpr int n_tokens = 5; + const std::vector ids = { + 10, 20, 30, 40, + 20, 30, 40, 10, + 30, 40, 10, 20, + 40, 10, 20, 30, + 10, 30, 20, 40, + }; + + ggml_context * ctx = make_test_context(1u << 20); + TEST_ASSERT_MSG(ctx != nullptr, "ggml_init failed"); + if (!ctx) { + ggml_backend_free(backend); + std::fprintf(stderr, " FAIL\n"); + return; + } + + ggml_tensor * input = ggml_new_tensor_2d( + ctx, GGML_TYPE_I32, n_routes, n_tokens); + ggml_tensor * aligned = ggml_ds4_moe_align_ids(ctx, input); + ggml_set_output(aligned); + TEST_ASSERT_MSG(ggml_backend_supports_op(backend, aligned), + "GPU rejected MoE ID alignment"); + + ggml_cgraph * graph = ggml_new_graph_custom(ctx, 16, false); + ggml_build_forward_expand(graph, aligned); + ggml_gallocr_t alloc = ggml_gallocr_new( + ggml_backend_get_default_buffer_type(backend)); + const bool allocated = ggml_gallocr_alloc_graph(alloc, graph); + TEST_ASSERT_MSG(allocated, "MoE ID alignment graph allocation failed"); + if (allocated) { + ggml_backend_tensor_set( + input, ids.data(), 0, ids.size() * sizeof(int32_t)); + const bool computed = + ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS; + TEST_ASSERT_MSG(computed, "MoE ID alignment graph compute failed"); + if (computed) { + std::vector actual(ids.size()); + ggml_backend_tensor_get( + aligned, actual.data(), 0, actual.size() * sizeof(int32_t)); + for (int token = 0; token < n_tokens; ++token) { + bool seen_route[n_routes] = {}; + for (int slot = 0; slot < n_routes; ++slot) { + const uint32_t encoded = (uint32_t) + actual[(size_t) token * n_routes + slot]; + const int original_route = (int) ((encoded >> 16) & 0xffu); + const int expert = (int) (encoded & 0xffffu); + TEST_ASSERT_MSG((encoded & 0xff000000u) == 0x5a000000u, + "aligned expert metadata magic mismatch"); + TEST_ASSERT_MSG(original_route >= 0 && + original_route < n_routes, + "aligned original route is out of range"); + if (original_route >= 0 && original_route < n_routes) { + TEST_ASSERT_MSG(!seen_route[original_route], + "alignment did not preserve a route permutation"); + seen_route[original_route] = true; + TEST_ASSERT_MSG( + expert == ids[(size_t) token * n_routes + original_route], + "alignment metadata changed the selected expert"); + } + TEST_ASSERT_MSG(expert == ids[(size_t) slot], + "q=5 repeated experts were not slot-aligned"); + } + } + } + } + + ggml_gallocr_free(alloc); + ggml_free(ctx); + ggml_backend_free(backend); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + static void test_cpu_hc_sinkhorn_ref(float * out, const float * mix, const float * scale, const float * base, int n_hc, int iters, float eps) { const float pre_scale = scale[0]; @@ -4069,6 +4166,7 @@ int main() { test_ds4_topk_block_radix_gpu(); test_ds4_flash_attention_inverse_rope_fallback_gpu(); test_hc_post_strided_split_gpu(); + test_moe_id_alignment_q5_gpu(); test_hc_pre_kernel_gpu(); test_layer_range_rejects_stale_hc_boundary(); test_hc_scratch_per_device(); diff --git a/server/tests/test_summarize_ds4_server_decode.py b/server/tests/test_summarize_ds4_server_decode.py new file mode 100644 index 000000000..0f4a0924d --- /dev/null +++ b/server/tests/test_summarize_ds4_server_decode.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).parents[1] / "scripts" / "summarize_ds4_server_decode.py" +SPEC = importlib.util.spec_from_file_location("summarize_ds4_server_decode", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +class ServerDecodeSummaryTest(unittest.TestCase): + def test_groups_warmups_and_measured_records(self) -> None: + rows = [] + for rate in (50.0, 70.0, 72.0, 40.0, 60.0, 64.0): + rows.append( + "[deepseek4] DSpark decode: 128 tok in 2.000s " + f"({rate:.1f} tok/s) accept_rate=0.96" + ) + records = MODULE.parse_records("\n".join(rows)) + # Repeated contexts are distinct sweep legs (for example the final 2K + # request after a 16K burn-in) and must not overwrite one another. + payload = MODULE.summarize_records(records, [2048, 2048], 1, 2, 128) + groups = payload["groups"] + self.assertEqual(groups[0]["target_context"], 2048) + self.assertEqual(groups[0]["server_decode_tok_s_median"], 71.0) + self.assertEqual(groups[1]["target_context"], 2048) + self.assertEqual(groups[1]["server_decode_tok_s_median"], 62.0) + self.assertEqual(groups[0]["n"], 2) + + def test_rejects_missing_or_short_records(self) -> None: + record = MODULE.DecodeRecord(64, 1.0, 64.0, 1.0) + with self.assertRaisesRegex(ValueError, "found 1.*expected 2"): + MODULE.summarize_records([record], [2048], 0, 2, 64) + with self.assertRaisesRegex(ValueError, "completion lengths"): + MODULE.summarize_records([record], [2048], 0, 1, 128) + + +if __name__ == "__main__": + unittest.main()